fix: not using whole words

This commit is contained in:
tdgao
2026-08-26 16:26:06 -06:00
parent ae4519cc26
commit 31e405e73c
2 changed files with 48 additions and 6 deletions
@@ -244,6 +244,33 @@ function getDuplicateThresholds(terms: readonly string[]): Map<string, number> {
return thresholds
}
function isWordCharacter(character: string | undefined): boolean {
return character !== undefined && /^[\p{L}\p{M}\p{N}_]$/u.test(character)
}
function getCharacterBefore(text: string, index: number): string | undefined {
if (index <= 0) return undefined
const codePoint = text.codePointAt(index - 1)
if (codePoint === undefined) return undefined
if (codePoint >= 0xdc00 && codePoint <= 0xdfff && index > 1) {
return text.slice(index - 2, index)
}
return text[index - 1]
}
function getCharacterAt(text: string, index: number): string | undefined {
const codePoint = text.codePointAt(index)
return codePoint === undefined ? undefined : String.fromCodePoint(codePoint)
}
function isWholeWordMatch(text: string, start: number, end: number): boolean {
return (
!isWordCharacter(getCharacterBefore(text, start)) && !isWordCharacter(getCharacterAt(text, end))
)
}
export function createProfanityValidator(
config: ProfanityConfig = DEFAULT_PROFANITY_CONFIG,
): ProfanityValidator {
@@ -273,13 +300,20 @@ export function createProfanityValidator(
for (const match of matcher.getAllMatches(text, true)) {
const profanityPattern = entries[match.termId]
if (!profanityPattern || match.startIndex < (matches.at(-1)?.end ?? 0)) continue
const end = match.endIndex + 1
if (
!profanityPattern ||
!isWholeWordMatch(text, match.startIndex, end) ||
match.startIndex < (matches.at(-1)?.end ?? 0)
) {
continue
}
matches.push({
...profanityPattern,
rawText: text.slice(match.startIndex, match.endIndex + 1),
rawText: text.slice(match.startIndex, end),
start: match.startIndex,
end: match.endIndex + 1,
end,
})
}
@@ -23,7 +23,7 @@ for (const [form, input] of blockedForms) {
})
}
test('does not use term exceptions', () => {
test('matches whole words without term exceptions', () => {
const validator = createProfanityValidator({
patterns: {
bad: { kind: 'profanity' },
@@ -31,9 +31,10 @@ test('does not use term exceptions', () => {
})
assert.equal(validator.findFirst('not bad word')?.term, 'bad')
assert.equal(validator.findFirst('notbadword'), undefined)
})
test('uses the first match when one configured term prefixes another', () => {
test('uses the whole-word match when one configured term prefixes another', () => {
const validator = createProfanityValidator({
patterns: {
bad: { kind: 'profanity' },
@@ -41,7 +42,14 @@ test('uses the first match when one configured term prefixes another', () => {
},
})
assert.equal(validator.findFirst('badword')?.term, 'bad')
assert.equal(validator.findFirst('badword')?.term, 'badword')
})
test('does not join ordinary words or match inside larger words', () => {
assert.equal(validateProfanity('pause menu').valid, true)
assert.equal(validateProfanity('accumulate').valid, true)
assert.equal(validateProfanity('cum').valid, false)
assert.equal(validateProfanity('s e m e n').valid, false)
})
test('rejects any uncensored configured profanity', () => {