From d7442e8760a96a5d5df5770838e294e66c0f46a0 Mon Sep 17 00:00:00 2001 From: tdgao Date: Fri, 28 Aug 2026 12:10:19 -0600 Subject: [PATCH] feat: clean up link detection in summary and description --- .../composables/project-field-validation.ts | 8 +-- packages/moderation/package.json | 1 + .../moderation/src/locales/en-US/index.json | 5 +- .../src/validation-rules/rules/description.ts | 70 ++++++++++++++++--- .../src/validation-rules/rules/summary.ts | 37 ++++------ .../moderation/src/validation-rules/tests.ts | 43 ++++++++++++ .../src/validators/links/block-list.ts | 47 ++++++------- .../src/validators/links/detection.ts | 34 --------- .../moderation/src/validators/links/index.ts | 10 +-- .../src/validators/links/syntax-checks.ts | 25 ++----- .../moderation/src/validators/links/tests.ts | 33 ++++----- .../src/validators/links/validation.ts | 26 ++++--- .../src/validators/profanity/index.ts | 6 ++ .../src/validators/profanity/tests.ts | 20 ++++++ pnpm-lock.yaml | 9 +++ 15 files changed, 212 insertions(+), 162 deletions(-) delete mode 100644 packages/moderation/src/validators/links/detection.ts diff --git a/apps/frontend/src/composables/project-field-validation.ts b/apps/frontend/src/composables/project-field-validation.ts index b99eaedcd9..fd45186a4b 100644 --- a/apps/frontend/src/composables/project-field-validation.ts +++ b/apps/frontend/src/composables/project-field-validation.ts @@ -1,7 +1,7 @@ import { - extractProjectLinks, + extractDescriptionLinks, type FieldValidationMessage, - findBlockedProjectContentLink, + findBannedDescriptionLink, type LinkCheckContext, type LinkCheckResult, validateLink, @@ -92,12 +92,12 @@ export function useProjectDescriptionValidation( linkValidation.value = null if (import.meta.server) return - if (findBlockedProjectContentLink(text ?? '')) { + if (findBannedDescriptionLink(text ?? '')) { pending.value = false return } - const links = extractProjectLinks(text ?? '') + const links = extractDescriptionLinks(text ?? '') if (links.length === 0) { pending.value = false return diff --git a/packages/moderation/package.json b/packages/moderation/package.json index b426910e04..fea0e3f07b 100644 --- a/packages/moderation/package.json +++ b/packages/moderation/package.json @@ -17,6 +17,7 @@ "@modrinth/api-client": "workspace:*", "linkify-it": "^5.0.0", "obscenity": "^0.4.6", + "tlds": "^1.261.0", "vue": "^3.5.13" }, "devDependencies": { diff --git a/packages/moderation/src/locales/en-US/index.json b/packages/moderation/src/locales/en-US/index.json index ff66b3fc2a..0893b7973d 100644 --- a/packages/moderation/src/locales/en-US/index.json +++ b/packages/moderation/src/locales/en-US/index.json @@ -144,7 +144,7 @@ "defaultMessage": "Fix the project summary" }, "nags.link.description.invalid-url": { - "defaultMessage": "The description has an invalid link" + "defaultMessage": "The description has an invalid link: “{fullUrl}”." }, "nags.link.discord.channel": { "defaultMessage": "This is a link to a Discord channel, not a server invite." @@ -263,9 +263,6 @@ "nags.project-name-version.title": { "defaultMessage": "Fix project name" }, - "nags.project-summary-banned-link.description": { - "defaultMessage": "“{fullUrl}” is not allowed in project summaries." - }, "nags.project-summary-content.title": { "defaultMessage": "Review the project summary" }, diff --git a/packages/moderation/src/validation-rules/rules/description.ts b/packages/moderation/src/validation-rules/rules/description.ts index 3518988327..301267cf22 100644 --- a/packages/moderation/src/validation-rules/rules/description.ts +++ b/packages/moderation/src/validation-rules/rules/description.ts @@ -1,7 +1,14 @@ import { defineMessages } from '@modrinth/ui/i18n' +import LinkifyIt from 'linkify-it' +import tlds from 'tlds' with { type: 'json' } import type { Nag, ProjectValidationContext } from '../../types/nags.ts' -import { findBlockedProjectContentLink } from '../../validators/links/detection.ts' +import { URL_SHORTENERS } from '../../validators/links/block-list.ts' +import { + getLinkHostname, + hostnameMatchesDomain, + isIpAddress, +} from '../../validators/links/syntax-checks.ts' import { evaluateRules } from '../evaluate-rules.ts' import { evaluateNonStandardText, @@ -85,11 +92,46 @@ const messages = defineMessages({ }, }) -export const DESCRIPTION_MAX_PROFANITY_COUNT = 1 +export const DESCRIPTION_MAX_PROFANITY_COUNT = 2 export const DESCRIPTION_NON_STANDARD_TEXT_FAILURE_THRESHOLD = 0.05 export const MIN_DESCRIPTION_CHARS = 200 export const MAX_HEADER_LENGTH = 80 export const MIN_CHARS_PER_IMAGE = 60 +export const BANNED_DESCRIPTION_LINK_DOMAINS = [...URL_SHORTENERS] as const + +const descriptionLinkify = new LinkifyIt({ + fuzzyEmail: false, + fuzzyIP: true, + fuzzyLink: true, +}).tlds(tlds) + +export function extractDescriptionLinks(description: string): string[] { + const matches = descriptionLinkify.match(description) ?? [] + return [ + ...new Set( + matches + .map((match) => match.url) + .filter((url) => { + const hostname = getLinkHostname(url) + return hostname === null || !isIpAddress(hostname) + }), + ), + ] +} + +export function findBannedDescriptionLink(description: string): string | null { + for (const url of extractDescriptionLinks(description)) { + const hostname = getLinkHostname(url) + if ( + hostname && + BANNED_DESCRIPTION_LINK_DOMAINS.some((domain) => hostnameMatchesDomain(hostname, domain)) + ) { + return url + } + } + + return null +} export function analyzeHeaderLength(markdown: string): { hasLongHeaders: boolean @@ -199,8 +241,12 @@ export const projectDescriptionValidationRules = { 'project-description-banned-link': { severity: 'error', evaluate: (description) => { - const blockedLink = findBlockedProjectContentLink(description ?? '') - return blockedLink ? { valid: false, values: { fullUrl: blockedLink.url } } : { valid: true } + const bannedLink = findBannedDescriptionLink(description ?? '') + if (bannedLink) { + return { valid: false, values: { fullUrl: bannedLink } } + } else { + return { valid: true } + } }, presentation: { message: messages.bannedLink, @@ -213,9 +259,11 @@ export const projectDescriptionValidationRules = { const normalized = normalizeProjectFieldText(description ?? '') if (!normalized) return { valid: true } const length = countText(normalized) - return length < MIN_DESCRIPTION_CHARS - ? { valid: false, values: { length, minChars: MIN_DESCRIPTION_CHARS } } - : { valid: true } + if (length < MIN_DESCRIPTION_CHARS) { + return { valid: false, values: { length, minChars: MIN_DESCRIPTION_CHARS } } + } else { + return { valid: true } + } }, presentation: { message: messages.tooShort, @@ -226,9 +274,11 @@ export const projectDescriptionValidationRules = { severity: 'warning', evaluate: (description) => { const { longHeaders } = analyzeHeaderLength(description ?? '') - return longHeaders.length > 0 - ? { valid: false, values: { count: longHeaders.length } } - : { valid: true } + if (longHeaders.length > 0) { + return { valid: false, values: { count: longHeaders.length } } + } else { + return { valid: true } + } }, presentation: { message: messages.longHeaders, diff --git a/packages/moderation/src/validation-rules/rules/summary.ts b/packages/moderation/src/validation-rules/rules/summary.ts index 29521a5d1b..7eb644017d 100644 --- a/packages/moderation/src/validation-rules/rules/summary.ts +++ b/packages/moderation/src/validation-rules/rules/summary.ts @@ -1,10 +1,8 @@ import { defineMessages } from '@modrinth/ui/i18n' +import LinkifyIt from 'linkify-it' +import tlds from 'tlds' with { type: 'json' } import type { Nag, ProjectValidationContext } from '../../types/nags.ts' -import { - containsExplicitHttpProjectLink, - findBlockedProjectContentLink, -} from '../../validators/links/detection.ts' import { evaluateRules } from '../evaluate-rules.ts' import { evaluateNonStandardText, @@ -49,10 +47,6 @@ const messages = defineMessages({ id: 'nags.project-summary-non-standard-text.description', defaultMessage: 'Non-standard text characters, such as “₮ɆӾ₮”, are not allowed.', }, - bannedLink: { - id: 'nags.project-summary-banned-link.description', - defaultMessage: '“{fullUrl}” is not allowed in project summaries.', - }, matchesName: { id: 'project.text-validation.summary-matches-title', defaultMessage: "A project summary cannot be the same as it's title.", @@ -76,6 +70,16 @@ export interface ProjectSummaryValidationInput { name: string | null | undefined } +const summaryLinkify = new LinkifyIt({ + fuzzyEmail: false, + fuzzyIP: true, + fuzzyLink: true, +}).tlds(tlds) + +function containsProjectSummaryLinkOrIp(summary: string): boolean { + return summaryLinkify.test(summary) +} + export function projectSummaryMatchesName(summary: string, name: string) { const normalizedSummary = normalizeProjectFieldText(summary).replace(/\s+/g, '') const normalizedName = normalizeProjectFieldText(name).replace(/\s+/g, '') @@ -127,23 +131,12 @@ export const projectSummaryValidationRules = { nag: { title: messages.fixSummary, ...commonNagPresentation }, }, }, - 'project-summary-banned-link': { - severity: 'error', - evaluate: ({ summary }) => { - const blockedLink = findBlockedProjectContentLink(summary ?? '') - return blockedLink ? { valid: false, values: { fullUrl: blockedLink.url } } : { valid: true } - }, - presentation: { - message: messages.bannedLink, - nag: { title: messages.fixSummary, ...commonNagPresentation }, - }, - }, 'project-summary-matches-title': { severity: 'error', evaluate: ({ summary, name }) => ({ valid: !summary || - containsExplicitHttpProjectLink(summary) || + containsProjectSummaryLinkOrIp(summary) || !name || !projectSummaryMatchesName(summary, name), }), @@ -155,7 +148,7 @@ export const projectSummaryValidationRules = { 'summary-too-short': { severity: 'warning', evaluate: ({ summary }) => { - if (!summary || containsExplicitHttpProjectLink(summary)) return { valid: true } + if (!summary || containsProjectSummaryLinkOrIp(summary)) return { valid: true } const length = normalizeProjectFieldText(summary).length return length < MIN_SUMMARY_CHARS ? { valid: false, values: { length, minChars: MIN_SUMMARY_CHARS } } @@ -171,7 +164,7 @@ export const projectSummaryValidationRules = { evaluate: ({ summary }) => ({ valid: !summary || - (!hasProjectSummaryFormatting(summary) && !containsExplicitHttpProjectLink(summary)), + (!hasProjectSummaryFormatting(summary) && !containsProjectSummaryLinkOrIp(summary)), }), presentation: { message: messages.specialFormatting, diff --git a/packages/moderation/src/validation-rules/tests.ts b/packages/moderation/src/validation-rules/tests.ts index 9a103bd014..c0bb20de24 100644 --- a/packages/moderation/src/validation-rules/tests.ts +++ b/packages/moderation/src/validation-rules/tests.ts @@ -3,7 +3,10 @@ import test from 'node:test' import { evaluateRules } from './evaluate-rules.ts' import { + BANNED_DESCRIPTION_LINK_DOMAINS, countText, + extractDescriptionLinks, + findBannedDescriptionLink, MIN_CHARS_PER_IMAGE, MIN_DESCRIPTION_CHARS, validateProjectDescription, @@ -126,6 +129,46 @@ test('validates summary content from one rule set', () => { ) }) +test('rejects every link and IP address in project summaries', () => { + for (const summary of [ + 'Visit https://example.dev for more information about this project', + 'Visit example.dev for more information about this project', + 'Join 127.0.0.1:25565 for more information about this project', + ]) { + assert.deepEqual( + validateProjectSummary({ summary, name: 'Project title' }).map(({ code }) => code), + ['summary-special-formatting'], + ) + } + + assert.deepEqual( + validateProjectSummary({ + summary: 'Contact hello@example.com for more information about this project', + name: 'Project title', + }), + [], + ) +}) + +test('rejects configured description links and allows other links', () => { + for (const domain of BANNED_DESCRIPTION_LINK_DOMAINS) { + assert.equal( + findBannedDescriptionLink(`Visit https://${domain}/project`), + `https://${domain}/project`, + ) + assert.equal( + findBannedDescriptionLink(`Visit subdomain.${domain}/project`), + `http://subdomain.${domain}/project`, + ) + } + + assert.equal(findBannedDescriptionLink('Visit https://example.dev/project'), null) + assert.equal(findBannedDescriptionLink('Join 127.0.0.1:25565'), null) + assert.deepEqual(extractDescriptionLinks('Join 127.0.0.1:25565 or visit example.dev'), [ + 'http://example.dev', + ]) +}) + test('validates description requirements and simultaneous recommendations', () => { assert.deepEqual( validateProjectDescription(' ').map(({ code }) => code), diff --git a/packages/moderation/src/validators/links/block-list.ts b/packages/moderation/src/validators/links/block-list.ts index 521f34a773..def4ba728a 100644 --- a/packages/moderation/src/validators/links/block-list.ts +++ b/packages/moderation/src/validators/links/block-list.ts @@ -1,24 +1,23 @@ -export const PROJECT_LINK_BLOCK_LIST = { - urlShorteners: ['bit.ly', 'adf.ly', 'tinyurl.com', 'short.io', 'is.gd'], - external: [ - { label: 'Twitter', domains: ['twitter.com', 'x.com'] }, - { label: 'Instagram', domains: ['instagram.com'] }, - { label: 'Facebook', domains: ['facebook.com'] }, - { label: 'TikTok', domains: ['tiktok.com'] }, - { label: 'Telegram', domains: ['telegram.org', 't.me'] }, - { label: 'Bilibili', domains: ['bilibili.com'] }, - { label: 'Bluesky', domains: ['bsky.app'] }, - { label: 'Twitch', domains: ['twitch.tv'] }, - { label: 'Reddit', domains: ['reddit.com', 'redd.it'] }, - { label: 'Modrinth', domains: ['modrinth.com'] }, - { label: 'Minecraft', domains: ['minecraft.net'] }, - { - label: 'Mod distribution platform', - domains: ['curseforge.com', 'planetminecraft.com', '9minecraft.net', 'mcmod.cn'], - }, - { - label: 'AI mod generation platform', - domains: ['creativemode.net', 'orcaclient.com', 'autoforged.cn'], - }, - ], -} as const +export const URL_SHORTENERS = ['bit.ly', 'adf.ly', 'tinyurl.com', 'short.io', 'is.gd'] as const + +export const EXTERNAL_LINKS_BLOCK_LIST = [ + { label: 'Twitter', domains: ['twitter.com', 'x.com'] }, + { label: 'Instagram', domains: ['instagram.com'] }, + { label: 'Facebook', domains: ['facebook.com'] }, + { label: 'TikTok', domains: ['tiktok.com'] }, + { label: 'Telegram', domains: ['telegram.org', 't.me'] }, + { label: 'Bilibili', domains: ['bilibili.com'] }, + { label: 'Bluesky', domains: ['bsky.app'] }, + { label: 'Twitch', domains: ['twitch.tv'] }, + { label: 'Reddit', domains: ['reddit.com', 'redd.it'] }, + { label: 'Modrinth', domains: ['modrinth.com'] }, + { label: 'Minecraft', domains: ['minecraft.net'] }, + { + label: 'Mod distribution platform', + domains: ['curseforge.com', 'planetminecraft.com', '9minecraft.net', 'mcmod.cn'], + }, + { + label: 'AI mod generation platform', + domains: ['creativemode.net', 'orcaclient.com', 'autoforged.cn'], + }, +] as const diff --git a/packages/moderation/src/validators/links/detection.ts b/packages/moderation/src/validators/links/detection.ts deleted file mode 100644 index ac92bdd8f6..0000000000 --- a/packages/moderation/src/validators/links/detection.ts +++ /dev/null @@ -1,34 +0,0 @@ -import LinkifyIt from 'linkify-it' - -import { getBlockedProjectContentLink } from './syntax-checks.ts' - -const linkify = new LinkifyIt({ - fuzzyEmail: false, - fuzzyIP: true, - fuzzyLink: true, -}) - -export function extractProjectLinks(text: string) { - const matches = linkify.match(text) ?? [] - return [...new Set(matches.map((match) => match.url))] -} - -export function containsProjectLinkOrIp(text: string) { - return linkify.test(text) -} - -export function containsExplicitHttpProjectLink(text: string) { - return (linkify.match(text) ?? []).some((match) => { - const schema = match.schema.toLowerCase() - return schema === 'http:' || schema === 'https:' - }) -} - -export function findBlockedProjectContentLink(text: string) { - for (const url of extractProjectLinks(text)) { - const blockedLink = getBlockedProjectContentLink(url) - if (blockedLink) return blockedLink - } - - return null -} diff --git a/packages/moderation/src/validators/links/index.ts b/packages/moderation/src/validators/links/index.ts index 990e9cd2bf..4109419cb6 100644 --- a/packages/moderation/src/validators/links/index.ts +++ b/packages/moderation/src/validators/links/index.ts @@ -1,20 +1,12 @@ -export { PROJECT_LINK_BLOCK_LIST } from './block-list.ts' -export { - containsExplicitHttpProjectLink, - containsProjectLinkOrIp, - extractProjectLinks, - findBlockedProjectContentLink, -} from './detection.ts' +export { EXTERNAL_LINKS_BLOCK_LIST, URL_SHORTENERS } from './block-list.ts' export { PROJECT_LINK_DOMAIN_LIST } from './domain-list.ts' export { - getBlockedProjectContentLink, getBlockedProjectExternalLink, getLinkHostname, hostnameMatchesDomain, isCommonProjectLink, isDiscordLink, isInappropriateLicenseLink, - isLinkShortener, } from './syntax-checks.ts' export type { BlockedProjectLink, diff --git a/packages/moderation/src/validators/links/syntax-checks.ts b/packages/moderation/src/validators/links/syntax-checks.ts index 2ffe5c17d1..fecd7583d7 100644 --- a/packages/moderation/src/validators/links/syntax-checks.ts +++ b/packages/moderation/src/validators/links/syntax-checks.ts @@ -1,4 +1,4 @@ -import { PROJECT_LINK_BLOCK_LIST } from './block-list.ts' +import { EXTERNAL_LINKS_BLOCK_LIST, URL_SHORTENERS } from './block-list.ts' import { PROJECT_LINK_DOMAIN_LIST } from './domain-list.ts' import type { BlockedProjectLink, @@ -304,10 +304,6 @@ export function isDiscordLink(url: string | null | undefined): boolean { return isCommonProjectLink(url, 'discord') } -export function isLinkShortener(url: string | null | undefined): boolean { - return isLinkFromDomains(url, PROJECT_LINK_BLOCK_LIST.urlShorteners) -} - export function isInappropriateLicenseLink(url: string | null | undefined): boolean { return isLinkFromDomains(url, PROJECT_LINK_DOMAIN_LIST.inappropriateLicense) } @@ -319,34 +315,23 @@ export function hasFieldSpecificDescendant(node: LinkCheckNode): boolean { ) } -function isIpAddress(hostname: string): boolean { +export function isIpAddress(hostname: string): boolean { const strippedHostname = hostname.replace(/^\[|]$/g, '') return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(strippedHostname) || strippedHostname.includes(':') } -function getBlockedProjectLink(url: string, includeExternal: boolean): BlockedProjectLink | null { +export function getBlockedProjectExternalLink(url: string): BlockedProjectLink | null { const hostname = getLinkHostname(url) if (!hostname) return null if (isIpAddress(hostname)) return { label: 'IP address', url } - if ( - PROJECT_LINK_BLOCK_LIST.urlShorteners.some((domain) => hostnameMatchesDomain(hostname, domain)) - ) { + if (URL_SHORTENERS.some((domain) => hostnameMatchesDomain(hostname, domain))) { return { label: 'URL shortener', url } } - if (!includeExternal) return null - const entry = PROJECT_LINK_BLOCK_LIST.external.find(({ domains }) => + const entry = EXTERNAL_LINKS_BLOCK_LIST.find(({ domains }) => domains.some((domain) => hostnameMatchesDomain(hostname, domain)), ) return entry ? { label: entry.label, url } : null } - -export function getBlockedProjectContentLink(url: string): BlockedProjectLink | null { - return getBlockedProjectLink(url, false) -} - -export function getBlockedProjectExternalLink(url: string): BlockedProjectLink | null { - return getBlockedProjectLink(url, true) -} diff --git a/packages/moderation/src/validators/links/tests.ts b/packages/moderation/src/validators/links/tests.ts index 62960230c4..1ae689b158 100644 --- a/packages/moderation/src/validators/links/tests.ts +++ b/packages/moderation/src/validators/links/tests.ts @@ -2,14 +2,13 @@ import assert from 'node:assert/strict' import test from 'node:test' import { - getBlockedProjectContentLink, + EXTERNAL_LINKS_BLOCK_LIST, getBlockedProjectExternalLink, getLinkHostname, isCommonProjectLink, isDiscordLink, isInappropriateLicenseLink, - isLinkShortener, - PROJECT_LINK_BLOCK_LIST, + URL_SHORTENERS, validateLink, validateLinkSyntax, } from './index.ts' @@ -36,7 +35,8 @@ test('uses a description-specific message for invalid content links', async () = }) assert.equal(result?.message?.id, 'nags.link.description.invalid-url') - assert.equal(result?.message?.defaultMessage, 'The description has an invalid link') + assert.equal(result?.message?.defaultMessage, 'The description has an invalid link: “{fullUrl}”.') + assert.deepEqual(result?.values, { fullUrl: 'http://example.dev/project' }) }) test('matches recognized hosts case-insensitively', async () => { @@ -70,20 +70,20 @@ test('allows structured link types in general content', async () => { assert.equal(result?.severity, 'valid') }) -test('allows unrecognized valid links but keeps global restrictions in general content', async () => { +test('allows unrecognized valid links in general content', async () => { const allowed = await validateLink({ field: 'description', url: 'https://docs.example.dev/project', generalContent: true, }) - const blocked = await validateLink({ + const shortener = await validateLink({ field: 'description', url: 'https://bit.ly/project', generalContent: true, }) assert.equal(allowed?.severity, 'valid') - assert.equal(blocked?.severity, 'error') + assert.equal(shortener?.severity, 'valid') }) test('applies the external-link blocklist only outside general content', async () => { @@ -126,11 +126,7 @@ test('compares recognized license URLs with the selected license', async () => { }) test('blocks every configured URL shortener and its subdomains', () => { - for (const domain of PROJECT_LINK_BLOCK_LIST.urlShorteners) { - assert.deepEqual(getBlockedProjectContentLink(`https://${domain}/project`), { - label: 'URL shortener', - url: `https://${domain}/project`, - }) + for (const domain of URL_SHORTENERS) { assert.equal( getBlockedProjectExternalLink(`https://subdomain.${domain}/project`)?.label, 'URL shortener', @@ -139,7 +135,7 @@ test('blocks every configured URL shortener and its subdomains', () => { }) test('blocks every configured external domain and its subdomains', () => { - for (const { label, domains } of PROJECT_LINK_BLOCK_LIST.external) { + for (const { label, domains } of EXTERNAL_LINKS_BLOCK_LIST) { for (const domain of domains) { assert.deepEqual(getBlockedProjectExternalLink(`https://${domain}/project`), { label, @@ -153,8 +149,7 @@ test('blocks every configured external domain and its subdomains', () => { } }) -test('allows external-only blocklist entries in project content', () => { - assert.equal(getBlockedProjectContentLink('https://social.modrinth.com/project'), null) +test('blocks configured external links', () => { assert.equal( getBlockedProjectExternalLink('https://social.modrinth.com/project')?.label, 'Modrinth', @@ -162,12 +157,10 @@ test('allows external-only blocklist entries in project content', () => { }) test('blocks IP-address URLs without blocking domain lookalikes', () => { - assert.equal(getBlockedProjectContentLink('http://127.0.0.1:25565')?.label, 'IP address') - assert.equal(getBlockedProjectContentLink('https://[2001:db8::1]')?.label, 'IP address') assert.equal(getBlockedProjectExternalLink('http://127.0.0.1:25565')?.label, 'IP address') - assert.equal(getBlockedProjectContentLink('https://modrinth.com.example.dev'), null) + assert.equal(getBlockedProjectExternalLink('https://[2001:db8::1]')?.label, 'IP address') assert.equal(getBlockedProjectExternalLink('https://modrinth.com.example.dev'), null) - assert.equal(getBlockedProjectContentLink('not a URL'), null) + assert.equal(getBlockedProjectExternalLink('not a URL'), null) }) test('matches classified domains exactly or by subdomain', () => { @@ -177,8 +170,6 @@ test('matches classified domains exactly or by subdomain', () => { assert.equal(isCommonProjectLink('https://github.com.example.com/modrinth/code', 'source'), false) assert.equal(isDiscordLink('https://discord.gg/modrinth'), true) assert.equal(isDiscordLink('https://discord.gg.example.com/modrinth'), false) - assert.equal(isLinkShortener('https://bit.ly/modrinth'), true) - assert.equal(isLinkShortener('https://bit.ly.example.com/modrinth'), false) assert.equal(isInappropriateLicenseLink('https://youtube.com/watch?v=example'), true) assert.equal(isInappropriateLicenseLink('https://youtube.com.evil.dev/license'), false) }) diff --git a/packages/moderation/src/validators/links/validation.ts b/packages/moderation/src/validators/links/validation.ts index 6e49527d15..76af9339a4 100644 --- a/packages/moderation/src/validators/links/validation.ts +++ b/packages/moderation/src/validators/links/validation.ts @@ -12,7 +12,6 @@ import { anchored, check, fallback, - getBlockedProjectContentLink, getBlockedProjectExternalLink, hasFieldSpecificDescendant, matchesField, @@ -63,7 +62,6 @@ const coreMessages = defineMessages({ }, }) -//TODO: we should probably just let you not provide https but backend currently requires it const invalidUrlMessage = defineMessage({ id: 'nags.link.invalid-url', defaultMessage: 'This URL is invalid', @@ -71,7 +69,7 @@ const invalidUrlMessage = defineMessage({ const invalidDescriptionUrlMessage = defineMessage({ id: 'nags.link.description.invalid-url', - defaultMessage: 'The description has an invalid link', + defaultMessage: 'The description has an invalid link: “{fullUrl}”.', }) const checks = check(validUrlPrefix).message(invalidUrlMessage).transparent() @@ -97,11 +95,13 @@ function prepareMatchedLinkValidation( const build = matched.unrecognizedSeverity === 'warn' ? warn : error if (matched.unrecognizedMessage && isLeaf) { - const message = + const isInvalidDescriptionUrl = context.field === 'description' && matched.unrecognizedMessage.id === invalidUrlMessage.id - ? invalidDescriptionUrlMessage - : matched.unrecognizedMessage - return build(message, { label: matched.label }) + const message = isInvalidDescriptionUrl + ? invalidDescriptionUrlMessage + : matched.unrecognizedMessage + const values = isInvalidDescriptionUrl ? { fullUrl: context.url } : { label: matched.label } + return build(message, values) } if (expectedChild) { @@ -121,12 +121,10 @@ function prepareMatchedLinkValidation( return () => matched.verifyMatch!(match, context) } -function getBlockedLinkResult(context: LinkCheckContext): LinkCheckResult | undefined { +function getBlockedExternalLinkResult(context: LinkCheckContext): LinkCheckResult | undefined { const url = context.url - if (!url) return - const blockedLink = context.generalContent - ? getBlockedProjectContentLink(url) - : getBlockedProjectExternalLink(url) + if (!url || context.generalContent) return + const blockedLink = getBlockedProjectExternalLink(url) return blockedLink ? error(coreMessages.neverValid, { label: blockedLink.label }) : undefined } @@ -134,7 +132,7 @@ export function validateLinkSyntax(context: LinkCheckContext): LinkCheckResult | const url = context.url if (!url) return - const blockedResult = getBlockedLinkResult(context) + const blockedResult = getBlockedExternalLinkResult(context) if (blockedResult) return blockedResult const normalizedUrl = url.replace(/^(https:\/\/)www\./i, '$1') @@ -155,7 +153,7 @@ export async function validateLink( const url = context.url if (!url) return - const blockedResult = getBlockedLinkResult(context) + const blockedResult = getBlockedExternalLinkResult(context) if (blockedResult) return blockedResult const normalizedUrl = url.replace(/^(https:\/\/)www\./i, '$1') diff --git a/packages/moderation/src/validators/profanity/index.ts b/packages/moderation/src/validators/profanity/index.ts index 1f7bbac132..c1e478b50e 100644 --- a/packages/moderation/src/validators/profanity/index.ts +++ b/packages/moderation/src/validators/profanity/index.ts @@ -16,6 +16,7 @@ export interface ProfanityPattern { export interface ProfanityConfig { patterns: Readonly> + allowlist?: readonly string[] } export interface ProfanityMatch { @@ -223,8 +224,11 @@ export const DEFAULT_PROFANITY_PATTERNS: Readonly { @@ -312,6 +316,7 @@ function isWholeWordMatch(text: string, start: number, end: number): boolean { export function createProfanityValidator( config: ProfanityConfig = DEFAULT_PROFANITY_CONFIG, ): ProfanityValidator { + const allowlist = new Set(config.allowlist?.map((term) => term.normalize('NFC').toLowerCase())) const entries = Object.entries(config.patterns).map(([rawTerm, pattern]) => { const term = rawTerm.toLowerCase() if (!term || !/^[a-z]+$/.test(term)) { @@ -358,6 +363,7 @@ export function createProfanityValidator( const matchKey = `${match.termId}:${match.startIndex}:${match.endIndex}` if ( !profanityPattern || + allowlist.has(rawText.normalize('NFC').toLowerCase()) || (!strictMatches.has(matchKey) && !isCharacterByCharacterObfuscation(rawText)) || !isWholeWordMatch(text, match.startIndex, end) || match.startIndex < (matches.at(-1)?.end ?? 0) diff --git a/packages/moderation/src/validators/profanity/tests.ts b/packages/moderation/src/validators/profanity/tests.ts index 21d9fc4bac..1e777d48c0 100644 --- a/packages/moderation/src/validators/profanity/tests.ts +++ b/packages/moderation/src/validators/profanity/tests.ts @@ -68,6 +68,26 @@ test('allows redacted profanity when the removed letters cannot reconstruct a te assert.equal(validateProfanity('f.u.c.k').valid, false) }) +test('allows exact allowlisted terms case-insensitively', () => { + const validator = createProfanityValidator({ + patterns: { + koon: { kind: 'slur' }, + }, + allowlist: ['Кооп'], + }) + + assert.equal(validator.validate('Кооп').valid, true) + assert.equal(validator.validate('кооп').valid, true) + assert.equal(validator.validate('К.о.о.п').valid, false) + assert.equal(validator.validate('koon').valid, false) +}) + +test('allows default allowlisted terms', () => { + assert.equal(validateProfanity('Кооп').valid, true) + assert.equal(validateProfanity('кооп').valid, true) + assert.equal(validateProfanity('A Кооп project').valid, true) +}) + test('classifies slurs separately from other profanity', () => { const validator = createProfanityValidator({ patterns: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cca25cd36c..7cc103e2a4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -563,6 +563,9 @@ importers: obscenity: specifier: ^0.4.6 version: 0.4.6 + tlds: + specifier: ^1.261.0 + version: 1.261.0 vue: specifier: ^3.5.13 version: 3.5.27(typescript@5.9.3) @@ -9459,6 +9462,10 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} + tlds@1.261.0: + resolution: {integrity: sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA==} + hasBin: true + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -20408,6 +20415,8 @@ snapshots: tinyspy@4.0.4: {} + tlds@1.261.0: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0