feat: clean up link detection in summary and description

This commit is contained in:
tdgao
2026-08-28 12:10:19 -06:00
parent 18c1f31514
commit d7442e8760
15 changed files with 212 additions and 162 deletions
@@ -1,7 +1,7 @@
import {
extractProjectLinks,
extractDescriptionLinks,
type FieldValidationMessage,
findBlockedProjectContentLink,
findBannedDescriptionLink,
type LinkCheckContext,
type LinkCheckResult,
validateLink,
@@ -92,12 +92,12 @@ export function useProjectDescriptionValidation(
linkValidation.value = null
if (import.meta.server) return
if (findBlockedProjectContentLink(text ?? '')) {
if (findBannedDescriptionLink(text ?? '')) {
pending.value = false
return
}
const links = extractProjectLinks(text ?? '')
const links = extractDescriptionLinks(text ?? '')
if (links.length === 0) {
pending.value = false
return
+1
View File
@@ -17,6 +17,7 @@
"@modrinth/api-client": "workspace:*",
"linkify-it": "^5.0.0",
"obscenity": "^0.4.6",
"tlds": "^1.261.0",
"vue": "^3.5.13"
},
"devDependencies": {
@@ -144,7 +144,7 @@
"defaultMessage": "Fix the project summary"
},
"nags.link.description.invalid-url": {
"defaultMessage": "The description has an invalid link"
"defaultMessage": "The description has an invalid link: “{fullUrl}”."
},
"nags.link.discord.channel": {
"defaultMessage": "This is a link to a Discord channel, not a server invite."
@@ -263,9 +263,6 @@
"nags.project-name-version.title": {
"defaultMessage": "Fix project name"
},
"nags.project-summary-banned-link.description": {
"defaultMessage": "“{fullUrl}” is not allowed in project summaries."
},
"nags.project-summary-content.title": {
"defaultMessage": "Review the project summary"
},
@@ -1,7 +1,14 @@
import { defineMessages } from '@modrinth/ui/i18n'
import LinkifyIt from 'linkify-it'
import tlds from 'tlds' with { type: 'json' }
import type { Nag, ProjectValidationContext } from '../../types/nags.ts'
import { findBlockedProjectContentLink } from '../../validators/links/detection.ts'
import { URL_SHORTENERS } from '../../validators/links/block-list.ts'
import {
getLinkHostname,
hostnameMatchesDomain,
isIpAddress,
} from '../../validators/links/syntax-checks.ts'
import { evaluateRules } from '../evaluate-rules.ts'
import {
evaluateNonStandardText,
@@ -85,11 +92,46 @@ const messages = defineMessages({
},
})
export const DESCRIPTION_MAX_PROFANITY_COUNT = 1
export const DESCRIPTION_MAX_PROFANITY_COUNT = 2
export const DESCRIPTION_NON_STANDARD_TEXT_FAILURE_THRESHOLD = 0.05
export const MIN_DESCRIPTION_CHARS = 200
export const MAX_HEADER_LENGTH = 80
export const MIN_CHARS_PER_IMAGE = 60
export const BANNED_DESCRIPTION_LINK_DOMAINS = [...URL_SHORTENERS] as const
const descriptionLinkify = new LinkifyIt({
fuzzyEmail: false,
fuzzyIP: true,
fuzzyLink: true,
}).tlds(tlds)
export function extractDescriptionLinks(description: string): string[] {
const matches = descriptionLinkify.match(description) ?? []
return [
...new Set(
matches
.map((match) => match.url)
.filter((url) => {
const hostname = getLinkHostname(url)
return hostname === null || !isIpAddress(hostname)
}),
),
]
}
export function findBannedDescriptionLink(description: string): string | null {
for (const url of extractDescriptionLinks(description)) {
const hostname = getLinkHostname(url)
if (
hostname &&
BANNED_DESCRIPTION_LINK_DOMAINS.some((domain) => hostnameMatchesDomain(hostname, domain))
) {
return url
}
}
return null
}
export function analyzeHeaderLength(markdown: string): {
hasLongHeaders: boolean
@@ -199,8 +241,12 @@ export const projectDescriptionValidationRules = {
'project-description-banned-link': {
severity: 'error',
evaluate: (description) => {
const blockedLink = findBlockedProjectContentLink(description ?? '')
return blockedLink ? { valid: false, values: { fullUrl: blockedLink.url } } : { valid: true }
const bannedLink = findBannedDescriptionLink(description ?? '')
if (bannedLink) {
return { valid: false, values: { fullUrl: bannedLink } }
} else {
return { valid: true }
}
},
presentation: {
message: messages.bannedLink,
@@ -213,9 +259,11 @@ export const projectDescriptionValidationRules = {
const normalized = normalizeProjectFieldText(description ?? '')
if (!normalized) return { valid: true }
const length = countText(normalized)
return length < MIN_DESCRIPTION_CHARS
? { valid: false, values: { length, minChars: MIN_DESCRIPTION_CHARS } }
: { valid: true }
if (length < MIN_DESCRIPTION_CHARS) {
return { valid: false, values: { length, minChars: MIN_DESCRIPTION_CHARS } }
} else {
return { valid: true }
}
},
presentation: {
message: messages.tooShort,
@@ -226,9 +274,11 @@ export const projectDescriptionValidationRules = {
severity: 'warning',
evaluate: (description) => {
const { longHeaders } = analyzeHeaderLength(description ?? '')
return longHeaders.length > 0
? { valid: false, values: { count: longHeaders.length } }
: { valid: true }
if (longHeaders.length > 0) {
return { valid: false, values: { count: longHeaders.length } }
} else {
return { valid: true }
}
},
presentation: {
message: messages.longHeaders,
@@ -1,10 +1,8 @@
import { defineMessages } from '@modrinth/ui/i18n'
import LinkifyIt from 'linkify-it'
import tlds from 'tlds' with { type: 'json' }
import type { Nag, ProjectValidationContext } from '../../types/nags.ts'
import {
containsExplicitHttpProjectLink,
findBlockedProjectContentLink,
} from '../../validators/links/detection.ts'
import { evaluateRules } from '../evaluate-rules.ts'
import {
evaluateNonStandardText,
@@ -49,10 +47,6 @@ const messages = defineMessages({
id: 'nags.project-summary-non-standard-text.description',
defaultMessage: 'Non-standard text characters, such as “₮ɆӾ₮”, are not allowed.',
},
bannedLink: {
id: 'nags.project-summary-banned-link.description',
defaultMessage: '“{fullUrl}” is not allowed in project summaries.',
},
matchesName: {
id: 'project.text-validation.summary-matches-title',
defaultMessage: "A project summary cannot be the same as it's title.",
@@ -76,6 +70,16 @@ export interface ProjectSummaryValidationInput {
name: string | null | undefined
}
const summaryLinkify = new LinkifyIt({
fuzzyEmail: false,
fuzzyIP: true,
fuzzyLink: true,
}).tlds(tlds)
function containsProjectSummaryLinkOrIp(summary: string): boolean {
return summaryLinkify.test(summary)
}
export function projectSummaryMatchesName(summary: string, name: string) {
const normalizedSummary = normalizeProjectFieldText(summary).replace(/\s+/g, '')
const normalizedName = normalizeProjectFieldText(name).replace(/\s+/g, '')
@@ -127,23 +131,12 @@ export const projectSummaryValidationRules = {
nag: { title: messages.fixSummary, ...commonNagPresentation },
},
},
'project-summary-banned-link': {
severity: 'error',
evaluate: ({ summary }) => {
const blockedLink = findBlockedProjectContentLink(summary ?? '')
return blockedLink ? { valid: false, values: { fullUrl: blockedLink.url } } : { valid: true }
},
presentation: {
message: messages.bannedLink,
nag: { title: messages.fixSummary, ...commonNagPresentation },
},
},
'project-summary-matches-title': {
severity: 'error',
evaluate: ({ summary, name }) => ({
valid:
!summary ||
containsExplicitHttpProjectLink(summary) ||
containsProjectSummaryLinkOrIp(summary) ||
!name ||
!projectSummaryMatchesName(summary, name),
}),
@@ -155,7 +148,7 @@ export const projectSummaryValidationRules = {
'summary-too-short': {
severity: 'warning',
evaluate: ({ summary }) => {
if (!summary || containsExplicitHttpProjectLink(summary)) return { valid: true }
if (!summary || containsProjectSummaryLinkOrIp(summary)) return { valid: true }
const length = normalizeProjectFieldText(summary).length
return length < MIN_SUMMARY_CHARS
? { valid: false, values: { length, minChars: MIN_SUMMARY_CHARS } }
@@ -171,7 +164,7 @@ export const projectSummaryValidationRules = {
evaluate: ({ summary }) => ({
valid:
!summary ||
(!hasProjectSummaryFormatting(summary) && !containsExplicitHttpProjectLink(summary)),
(!hasProjectSummaryFormatting(summary) && !containsProjectSummaryLinkOrIp(summary)),
}),
presentation: {
message: messages.specialFormatting,
@@ -3,7 +3,10 @@ import test from 'node:test'
import { evaluateRules } from './evaluate-rules.ts'
import {
BANNED_DESCRIPTION_LINK_DOMAINS,
countText,
extractDescriptionLinks,
findBannedDescriptionLink,
MIN_CHARS_PER_IMAGE,
MIN_DESCRIPTION_CHARS,
validateProjectDescription,
@@ -126,6 +129,46 @@ test('validates summary content from one rule set', () => {
)
})
test('rejects every link and IP address in project summaries', () => {
for (const summary of [
'Visit https://example.dev for more information about this project',
'Visit example.dev for more information about this project',
'Join 127.0.0.1:25565 for more information about this project',
]) {
assert.deepEqual(
validateProjectSummary({ summary, name: 'Project title' }).map(({ code }) => code),
['summary-special-formatting'],
)
}
assert.deepEqual(
validateProjectSummary({
summary: 'Contact hello@example.com for more information about this project',
name: 'Project title',
}),
[],
)
})
test('rejects configured description links and allows other links', () => {
for (const domain of BANNED_DESCRIPTION_LINK_DOMAINS) {
assert.equal(
findBannedDescriptionLink(`Visit https://${domain}/project`),
`https://${domain}/project`,
)
assert.equal(
findBannedDescriptionLink(`Visit subdomain.${domain}/project`),
`http://subdomain.${domain}/project`,
)
}
assert.equal(findBannedDescriptionLink('Visit https://example.dev/project'), null)
assert.equal(findBannedDescriptionLink('Join 127.0.0.1:25565'), null)
assert.deepEqual(extractDescriptionLinks('Join 127.0.0.1:25565 or visit example.dev'), [
'http://example.dev',
])
})
test('validates description requirements and simultaneous recommendations', () => {
assert.deepEqual(
validateProjectDescription(' ').map(({ code }) => code),
@@ -1,24 +1,23 @@
export const PROJECT_LINK_BLOCK_LIST = {
urlShorteners: ['bit.ly', 'adf.ly', 'tinyurl.com', 'short.io', 'is.gd'],
external: [
{ label: 'Twitter', domains: ['twitter.com', 'x.com'] },
{ label: 'Instagram', domains: ['instagram.com'] },
{ label: 'Facebook', domains: ['facebook.com'] },
{ label: 'TikTok', domains: ['tiktok.com'] },
{ label: 'Telegram', domains: ['telegram.org', 't.me'] },
{ label: 'Bilibili', domains: ['bilibili.com'] },
{ label: 'Bluesky', domains: ['bsky.app'] },
{ label: 'Twitch', domains: ['twitch.tv'] },
{ label: 'Reddit', domains: ['reddit.com', 'redd.it'] },
{ label: 'Modrinth', domains: ['modrinth.com'] },
{ label: 'Minecraft', domains: ['minecraft.net'] },
{
label: 'Mod distribution platform',
domains: ['curseforge.com', 'planetminecraft.com', '9minecraft.net', 'mcmod.cn'],
},
{
label: 'AI mod generation platform',
domains: ['creativemode.net', 'orcaclient.com', 'autoforged.cn'],
},
],
} as const
export const URL_SHORTENERS = ['bit.ly', 'adf.ly', 'tinyurl.com', 'short.io', 'is.gd'] as const
export const EXTERNAL_LINKS_BLOCK_LIST = [
{ label: 'Twitter', domains: ['twitter.com', 'x.com'] },
{ label: 'Instagram', domains: ['instagram.com'] },
{ label: 'Facebook', domains: ['facebook.com'] },
{ label: 'TikTok', domains: ['tiktok.com'] },
{ label: 'Telegram', domains: ['telegram.org', 't.me'] },
{ label: 'Bilibili', domains: ['bilibili.com'] },
{ label: 'Bluesky', domains: ['bsky.app'] },
{ label: 'Twitch', domains: ['twitch.tv'] },
{ label: 'Reddit', domains: ['reddit.com', 'redd.it'] },
{ label: 'Modrinth', domains: ['modrinth.com'] },
{ label: 'Minecraft', domains: ['minecraft.net'] },
{
label: 'Mod distribution platform',
domains: ['curseforge.com', 'planetminecraft.com', '9minecraft.net', 'mcmod.cn'],
},
{
label: 'AI mod generation platform',
domains: ['creativemode.net', 'orcaclient.com', 'autoforged.cn'],
},
] as const
@@ -1,34 +0,0 @@
import LinkifyIt from 'linkify-it'
import { getBlockedProjectContentLink } from './syntax-checks.ts'
const linkify = new LinkifyIt({
fuzzyEmail: false,
fuzzyIP: true,
fuzzyLink: true,
})
export function extractProjectLinks(text: string) {
const matches = linkify.match(text) ?? []
return [...new Set(matches.map((match) => match.url))]
}
export function containsProjectLinkOrIp(text: string) {
return linkify.test(text)
}
export function containsExplicitHttpProjectLink(text: string) {
return (linkify.match(text) ?? []).some((match) => {
const schema = match.schema.toLowerCase()
return schema === 'http:' || schema === 'https:'
})
}
export function findBlockedProjectContentLink(text: string) {
for (const url of extractProjectLinks(text)) {
const blockedLink = getBlockedProjectContentLink(url)
if (blockedLink) return blockedLink
}
return null
}
@@ -1,20 +1,12 @@
export { PROJECT_LINK_BLOCK_LIST } from './block-list.ts'
export {
containsExplicitHttpProjectLink,
containsProjectLinkOrIp,
extractProjectLinks,
findBlockedProjectContentLink,
} from './detection.ts'
export { EXTERNAL_LINKS_BLOCK_LIST, URL_SHORTENERS } from './block-list.ts'
export { PROJECT_LINK_DOMAIN_LIST } from './domain-list.ts'
export {
getBlockedProjectContentLink,
getBlockedProjectExternalLink,
getLinkHostname,
hostnameMatchesDomain,
isCommonProjectLink,
isDiscordLink,
isInappropriateLicenseLink,
isLinkShortener,
} from './syntax-checks.ts'
export type {
BlockedProjectLink,
@@ -1,4 +1,4 @@
import { PROJECT_LINK_BLOCK_LIST } from './block-list.ts'
import { EXTERNAL_LINKS_BLOCK_LIST, URL_SHORTENERS } from './block-list.ts'
import { PROJECT_LINK_DOMAIN_LIST } from './domain-list.ts'
import type {
BlockedProjectLink,
@@ -304,10 +304,6 @@ export function isDiscordLink(url: string | null | undefined): boolean {
return isCommonProjectLink(url, 'discord')
}
export function isLinkShortener(url: string | null | undefined): boolean {
return isLinkFromDomains(url, PROJECT_LINK_BLOCK_LIST.urlShorteners)
}
export function isInappropriateLicenseLink(url: string | null | undefined): boolean {
return isLinkFromDomains(url, PROJECT_LINK_DOMAIN_LIST.inappropriateLicense)
}
@@ -319,34 +315,23 @@ export function hasFieldSpecificDescendant(node: LinkCheckNode): boolean {
)
}
function isIpAddress(hostname: string): boolean {
export function isIpAddress(hostname: string): boolean {
const strippedHostname = hostname.replace(/^\[|]$/g, '')
return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(strippedHostname) || strippedHostname.includes(':')
}
function getBlockedProjectLink(url: string, includeExternal: boolean): BlockedProjectLink | null {
export function getBlockedProjectExternalLink(url: string): BlockedProjectLink | null {
const hostname = getLinkHostname(url)
if (!hostname) return null
if (isIpAddress(hostname)) return { label: 'IP address', url }
if (
PROJECT_LINK_BLOCK_LIST.urlShorteners.some((domain) => hostnameMatchesDomain(hostname, domain))
) {
if (URL_SHORTENERS.some((domain) => hostnameMatchesDomain(hostname, domain))) {
return { label: 'URL shortener', url }
}
if (!includeExternal) return null
const entry = PROJECT_LINK_BLOCK_LIST.external.find(({ domains }) =>
const entry = EXTERNAL_LINKS_BLOCK_LIST.find(({ domains }) =>
domains.some((domain) => hostnameMatchesDomain(hostname, domain)),
)
return entry ? { label: entry.label, url } : null
}
export function getBlockedProjectContentLink(url: string): BlockedProjectLink | null {
return getBlockedProjectLink(url, false)
}
export function getBlockedProjectExternalLink(url: string): BlockedProjectLink | null {
return getBlockedProjectLink(url, true)
}
@@ -2,14 +2,13 @@ import assert from 'node:assert/strict'
import test from 'node:test'
import {
getBlockedProjectContentLink,
EXTERNAL_LINKS_BLOCK_LIST,
getBlockedProjectExternalLink,
getLinkHostname,
isCommonProjectLink,
isDiscordLink,
isInappropriateLicenseLink,
isLinkShortener,
PROJECT_LINK_BLOCK_LIST,
URL_SHORTENERS,
validateLink,
validateLinkSyntax,
} from './index.ts'
@@ -36,7 +35,8 @@ test('uses a description-specific message for invalid content links', async () =
})
assert.equal(result?.message?.id, 'nags.link.description.invalid-url')
assert.equal(result?.message?.defaultMessage, 'The description has an invalid link')
assert.equal(result?.message?.defaultMessage, 'The description has an invalid link: “{fullUrl}”.')
assert.deepEqual(result?.values, { fullUrl: 'http://example.dev/project' })
})
test('matches recognized hosts case-insensitively', async () => {
@@ -70,20 +70,20 @@ test('allows structured link types in general content', async () => {
assert.equal(result?.severity, 'valid')
})
test('allows unrecognized valid links but keeps global restrictions in general content', async () => {
test('allows unrecognized valid links in general content', async () => {
const allowed = await validateLink({
field: 'description',
url: 'https://docs.example.dev/project',
generalContent: true,
})
const blocked = await validateLink({
const shortener = await validateLink({
field: 'description',
url: 'https://bit.ly/project',
generalContent: true,
})
assert.equal(allowed?.severity, 'valid')
assert.equal(blocked?.severity, 'error')
assert.equal(shortener?.severity, 'valid')
})
test('applies the external-link blocklist only outside general content', async () => {
@@ -126,11 +126,7 @@ test('compares recognized license URLs with the selected license', async () => {
})
test('blocks every configured URL shortener and its subdomains', () => {
for (const domain of PROJECT_LINK_BLOCK_LIST.urlShorteners) {
assert.deepEqual(getBlockedProjectContentLink(`https://${domain}/project`), {
label: 'URL shortener',
url: `https://${domain}/project`,
})
for (const domain of URL_SHORTENERS) {
assert.equal(
getBlockedProjectExternalLink(`https://subdomain.${domain}/project`)?.label,
'URL shortener',
@@ -139,7 +135,7 @@ test('blocks every configured URL shortener and its subdomains', () => {
})
test('blocks every configured external domain and its subdomains', () => {
for (const { label, domains } of PROJECT_LINK_BLOCK_LIST.external) {
for (const { label, domains } of EXTERNAL_LINKS_BLOCK_LIST) {
for (const domain of domains) {
assert.deepEqual(getBlockedProjectExternalLink(`https://${domain}/project`), {
label,
@@ -153,8 +149,7 @@ test('blocks every configured external domain and its subdomains', () => {
}
})
test('allows external-only blocklist entries in project content', () => {
assert.equal(getBlockedProjectContentLink('https://social.modrinth.com/project'), null)
test('blocks configured external links', () => {
assert.equal(
getBlockedProjectExternalLink('https://social.modrinth.com/project')?.label,
'Modrinth',
@@ -162,12 +157,10 @@ test('allows external-only blocklist entries in project content', () => {
})
test('blocks IP-address URLs without blocking domain lookalikes', () => {
assert.equal(getBlockedProjectContentLink('http://127.0.0.1:25565')?.label, 'IP address')
assert.equal(getBlockedProjectContentLink('https://[2001:db8::1]')?.label, 'IP address')
assert.equal(getBlockedProjectExternalLink('http://127.0.0.1:25565')?.label, 'IP address')
assert.equal(getBlockedProjectContentLink('https://modrinth.com.example.dev'), null)
assert.equal(getBlockedProjectExternalLink('https://[2001:db8::1]')?.label, 'IP address')
assert.equal(getBlockedProjectExternalLink('https://modrinth.com.example.dev'), null)
assert.equal(getBlockedProjectContentLink('not a URL'), null)
assert.equal(getBlockedProjectExternalLink('not a URL'), null)
})
test('matches classified domains exactly or by subdomain', () => {
@@ -177,8 +170,6 @@ test('matches classified domains exactly or by subdomain', () => {
assert.equal(isCommonProjectLink('https://github.com.example.com/modrinth/code', 'source'), false)
assert.equal(isDiscordLink('https://discord.gg/modrinth'), true)
assert.equal(isDiscordLink('https://discord.gg.example.com/modrinth'), false)
assert.equal(isLinkShortener('https://bit.ly/modrinth'), true)
assert.equal(isLinkShortener('https://bit.ly.example.com/modrinth'), false)
assert.equal(isInappropriateLicenseLink('https://youtube.com/watch?v=example'), true)
assert.equal(isInappropriateLicenseLink('https://youtube.com.evil.dev/license'), false)
})
@@ -12,7 +12,6 @@ import {
anchored,
check,
fallback,
getBlockedProjectContentLink,
getBlockedProjectExternalLink,
hasFieldSpecificDescendant,
matchesField,
@@ -63,7 +62,6 @@ const coreMessages = defineMessages({
},
})
//TODO: we should probably just let you not provide https but backend currently requires it
const invalidUrlMessage = defineMessage({
id: 'nags.link.invalid-url',
defaultMessage: 'This URL is invalid',
@@ -71,7 +69,7 @@ const invalidUrlMessage = defineMessage({
const invalidDescriptionUrlMessage = defineMessage({
id: 'nags.link.description.invalid-url',
defaultMessage: 'The description has an invalid link',
defaultMessage: 'The description has an invalid link: “{fullUrl}”.',
})
const checks = check(validUrlPrefix).message(invalidUrlMessage).transparent()
@@ -97,11 +95,13 @@ function prepareMatchedLinkValidation(
const build = matched.unrecognizedSeverity === 'warn' ? warn : error
if (matched.unrecognizedMessage && isLeaf) {
const message =
const isInvalidDescriptionUrl =
context.field === 'description' && matched.unrecognizedMessage.id === invalidUrlMessage.id
? invalidDescriptionUrlMessage
: matched.unrecognizedMessage
return build(message, { label: matched.label })
const message = isInvalidDescriptionUrl
? invalidDescriptionUrlMessage
: matched.unrecognizedMessage
const values = isInvalidDescriptionUrl ? { fullUrl: context.url } : { label: matched.label }
return build(message, values)
}
if (expectedChild) {
@@ -121,12 +121,10 @@ function prepareMatchedLinkValidation(
return () => matched.verifyMatch!(match, context)
}
function getBlockedLinkResult(context: LinkCheckContext): LinkCheckResult | undefined {
function getBlockedExternalLinkResult(context: LinkCheckContext): LinkCheckResult | undefined {
const url = context.url
if (!url) return
const blockedLink = context.generalContent
? getBlockedProjectContentLink(url)
: getBlockedProjectExternalLink(url)
if (!url || context.generalContent) return
const blockedLink = getBlockedProjectExternalLink(url)
return blockedLink ? error(coreMessages.neverValid, { label: blockedLink.label }) : undefined
}
@@ -134,7 +132,7 @@ export function validateLinkSyntax(context: LinkCheckContext): LinkCheckResult |
const url = context.url
if (!url) return
const blockedResult = getBlockedLinkResult(context)
const blockedResult = getBlockedExternalLinkResult(context)
if (blockedResult) return blockedResult
const normalizedUrl = url.replace(/^(https:\/\/)www\./i, '$1')
@@ -155,7 +153,7 @@ export async function validateLink(
const url = context.url
if (!url) return
const blockedResult = getBlockedLinkResult(context)
const blockedResult = getBlockedExternalLinkResult(context)
if (blockedResult) return blockedResult
const normalizedUrl = url.replace(/^(https:\/\/)www\./i, '$1')
@@ -16,6 +16,7 @@ export interface ProfanityPattern {
export interface ProfanityConfig {
patterns: Readonly<Record<string, ProfanityPattern>>
allowlist?: readonly string[]
}
export interface ProfanityMatch {
@@ -223,8 +224,11 @@ export const DEFAULT_PROFANITY_PATTERNS: Readonly<Record<string, ProfanityPatter
}),
)
export const DEFAULT_PROFANITY_ALLOWLIST = ['Кооп'] as const
export const DEFAULT_PROFANITY_CONFIG: ProfanityConfig = {
patterns: DEFAULT_PROFANITY_PATTERNS,
allowlist: DEFAULT_PROFANITY_ALLOWLIST,
}
function getDuplicateThresholds(terms: readonly string[]): Map<string, number> {
@@ -312,6 +316,7 @@ function isWholeWordMatch(text: string, start: number, end: number): boolean {
export function createProfanityValidator(
config: ProfanityConfig = DEFAULT_PROFANITY_CONFIG,
): ProfanityValidator {
const allowlist = new Set(config.allowlist?.map((term) => term.normalize('NFC').toLowerCase()))
const entries = Object.entries(config.patterns).map(([rawTerm, pattern]) => {
const term = rawTerm.toLowerCase()
if (!term || !/^[a-z]+$/.test(term)) {
@@ -358,6 +363,7 @@ export function createProfanityValidator(
const matchKey = `${match.termId}:${match.startIndex}:${match.endIndex}`
if (
!profanityPattern ||
allowlist.has(rawText.normalize('NFC').toLowerCase()) ||
(!strictMatches.has(matchKey) && !isCharacterByCharacterObfuscation(rawText)) ||
!isWholeWordMatch(text, match.startIndex, end) ||
match.startIndex < (matches.at(-1)?.end ?? 0)
@@ -68,6 +68,26 @@ test('allows redacted profanity when the removed letters cannot reconstruct a te
assert.equal(validateProfanity('f.u.c.k').valid, false)
})
test('allows exact allowlisted terms case-insensitively', () => {
const validator = createProfanityValidator({
patterns: {
koon: { kind: 'slur' },
},
allowlist: ['Кооп'],
})
assert.equal(validator.validate('Кооп').valid, true)
assert.equal(validator.validate('кооп').valid, true)
assert.equal(validator.validate('К.о.о.п').valid, false)
assert.equal(validator.validate('koon').valid, false)
})
test('allows default allowlisted terms', () => {
assert.equal(validateProfanity('Кооп').valid, true)
assert.equal(validateProfanity('кооп').valid, true)
assert.equal(validateProfanity('A Кооп project').valid, true)
})
test('classifies slurs separately from other profanity', () => {
const validator = createProfanityValidator({
patterns: {
+9
View File
@@ -563,6 +563,9 @@ importers:
obscenity:
specifier: ^0.4.6
version: 0.4.6
tlds:
specifier: ^1.261.0
version: 1.261.0
vue:
specifier: ^3.5.13
version: 3.5.27(typescript@5.9.3)
@@ -9459,6 +9462,10 @@ packages:
resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==}
engines: {node: '>=14.0.0'}
tlds@1.261.0:
resolution: {integrity: sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA==}
hasBin: true
to-regex-range@5.0.1:
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
engines: {node: '>=8.0'}
@@ -20408,6 +20415,8 @@ snapshots:
tinyspy@4.0.4: {}
tlds@1.261.0: {}
to-regex-range@5.0.1:
dependencies:
is-number: 7.0.0