diff --git a/packages/moderation/src/validation-rules/rules/description.ts b/packages/moderation/src/validation-rules/rules/description.ts index b82cd9da7b..e9f2ddafdc 100644 --- a/packages/moderation/src/validation-rules/rules/description.ts +++ b/packages/moderation/src/validation-rules/rules/description.ts @@ -14,7 +14,7 @@ import { import { validateSpam } from '../../validators/spam/index.ts' import { evaluateRules } from '../evaluate-rules.ts' import { - evaluateEnglishText, + evaluateEnglishTextBlocks, evaluateNonStandardText, evaluateProfanity, evaluateSlur, @@ -189,6 +189,16 @@ export function extractDescriptionText(markdown: string): string { return withoutMarkdownSyntax.replace(/\s+/g, ' ').trim() } +export function extractDescriptionTextBlocks(markdown: string): string[] { + if (!markdown) return [] + + return markdown + .replace(/```[\s\S]*?```/g, '') + .split(/\n\s*\n+/) + .map(extractDescriptionText) + .filter(Boolean) +} + export function countText(markdown: string): number { return extractDescriptionText(markdown).length } @@ -248,12 +258,13 @@ export const projectDescriptionValidationRules = { 'project-description-non-english': { severity: 'warning', evaluate: (description) => { - const text = extractDescriptionText(description ?? '') + const blocks = extractDescriptionTextBlocks(description ?? '') + const text = blocks.join(' ') if (text.length < MIN_DESCRIPTION_CHARS || !validateSpam(text).valid) { return { valid: true } } - return evaluateEnglishText(text) + return evaluateEnglishTextBlocks(blocks) }, presentation: { message: messages.nonEnglish, diff --git a/packages/moderation/src/validation-rules/text.ts b/packages/moderation/src/validation-rules/text.ts index 93a971670b..8724dbd789 100644 --- a/packages/moderation/src/validation-rules/text.ts +++ b/packages/moderation/src/validation-rules/text.ts @@ -1,5 +1,9 @@ import type { ProjectValidationContext } from '../types/nags.ts' -import { validateEnglishSummaryText, validateEnglishText } from '../validators/language/index.ts' +import { + validateEnglishSummaryText, + validateEnglishText, + validateEnglishTextBlocks, +} from '../validators/language/index.ts' import { getNonStandardTextRatio, validateNonStandardText, @@ -53,6 +57,11 @@ export function evaluateEnglishText(text: string): ValidationRuleEvaluation { return { valid: result.valid } } +export function evaluateEnglishTextBlocks(blocks: string[]): ValidationRuleEvaluation { + const result = validateEnglishTextBlocks(blocks) + return { valid: result.valid } +} + export function evaluateEnglishSummaryText(text: string): ValidationRuleEvaluation { const result = validateEnglishSummaryText(text) return { valid: result.valid } diff --git a/packages/moderation/src/validators/language/index.ts b/packages/moderation/src/validators/language/index.ts index 0c50e568a7..7c3e62f9fe 100644 --- a/packages/moderation/src/validators/language/index.ts +++ b/packages/moderation/src/validators/language/index.ts @@ -8,11 +8,26 @@ export interface LanguageDetection { export interface EnglishTextResult { valid: boolean detections: LanguageDetection[] + reasons: EnglishTextFailureReason[] +} + +export type EnglishTextFailureReason = 'insufficient-english-chunk-coverage' + +export interface LanguageChunkAnalysis { + totalChunks: number + englishChunks: number + nonEnglishChunks: number + ambiguousChunks: number + englishChunkPercentage: number | null } export const MIN_LANGUAGE_DETECTION_WORDS = 8 export const MIN_LANGUAGE_DETECTION_CHARACTERS = 35 -export const MIN_ENGLISH_SCORE = 0.45 +export const MIN_ENGLISH_SCORE = 0.8 +export const MIN_ENGLISH_SUMMARY_SCORE = 0.5 +export const LANGUAGE_CHUNK_WORDS = 24 +export const LANGUAGE_CHUNK_STRIDE_WORDS = 12 +export const MIN_ENGLISH_CHUNK_PERCENTAGE = 0.3 const wordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) const characterSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' }) @@ -27,30 +42,104 @@ function hasEnoughCharacters(text: string): boolean { return false } -function hasEnoughWords(text: string): boolean { - let wordCount = 0 - - for (const { isWordLike } of wordSegmenter.segment(text)) { - if (isWordLike && ++wordCount >= MIN_LANGUAGE_DETECTION_WORDS) return true - } - - return false +function getWords(text: string): string[] { + return [...wordSegmenter.segment(text)] + .filter(({ isWordLike }) => isWordLike) + .map(({ segment }) => segment) } -// TODO, the algorithm can still miss cases where english translation is provided and text primarily in another language -export function validateEnglishText(text: string): EnglishTextResult { - if (!hasEnoughCharacters(text) || !hasEnoughWords(text)) { - return { valid: true, detections: [] } +function getWordWindows(block: string): string[] { + const words = getWords(block) + if (words.length < MIN_LANGUAGE_DETECTION_WORDS) return [] + if (words.length <= LANGUAGE_CHUNK_WORDS) { + const window = words.join(' ') + return hasEnoughCharacters(window) ? [window] : [] } - const results = francAll(text) - const englishScore = results.find(([language]) => language === 'eng')?.[1] ?? 0 - const detections = results.map(([language, accuracy]) => ({ language, accuracy })) + const starts = new Set() + for ( + let start = 0; + start + MIN_LANGUAGE_DETECTION_WORDS <= words.length; + start += LANGUAGE_CHUNK_STRIDE_WORDS + ) { + starts.add(start) + } + starts.add(words.length - LANGUAGE_CHUNK_WORDS) + + return [...starts] + .sort((left, right) => left - right) + .map((start) => words.slice(start, start + LANGUAGE_CHUNK_WORDS).join(' ')) + .filter(hasEnoughCharacters) +} + +export function analyzeLanguageChunks(blocks: string[]): LanguageChunkAnalysis { + let englishChunks = 0 + let nonEnglishChunks = 0 + let ambiguousChunks = 0 + + for (const chunk of blocks.flatMap(getWordWindows)) { + const results = francAll(chunk) + const primaryLanguage = results[0]?.[0] + const englishScore = results.find(([language]) => language === 'eng')?.[1] ?? 0 + + if (primaryLanguage === 'eng') englishChunks++ + else if (primaryLanguage && englishScore < MIN_ENGLISH_SCORE) nonEnglishChunks++ + else ambiguousChunks++ + } + + const totalChunks = englishChunks + nonEnglishChunks + ambiguousChunks + const classifiedChunks = englishChunks + nonEnglishChunks + return { + totalChunks, + englishChunks, + nonEnglishChunks, + ambiguousChunks, + englishChunkPercentage: classifiedChunks === 0 ? null : englishChunks / classifiedChunks, + } +} + +export function validateEnglishTextBlocks(blocks: string[]): EnglishTextResult { + const normalizedBlocks = blocks.map((block) => block.trim()).filter(Boolean) + const text = normalizedBlocks.join('\n') + const chunkAnalysis = analyzeLanguageChunks(normalizedBlocks) + const valid = + chunkAnalysis.englishChunkPercentage === null || + chunkAnalysis.englishChunkPercentage >= MIN_ENGLISH_CHUNK_PERCENTAGE + const detections = + getWords(text).length >= MIN_LANGUAGE_DETECTION_WORDS && hasEnoughCharacters(text) + ? francAll(text).map(([language, accuracy]) => ({ language, accuracy })) + : [] return { - valid: englishScore > MIN_ENGLISH_SCORE, + valid, detections, + reasons: valid ? [] : ['insufficient-english-chunk-coverage'], } } -export const validateEnglishSummaryText = validateEnglishText +export function validateEnglishText(text: string): EnglishTextResult { + return validateEnglishTextBlocks(text.split(/\n+/)) +} + +export function validateEnglishSummaryText(text: string): EnglishTextResult { + const normalizedText = text.trim() + if ( + getWords(normalizedText).length < MIN_LANGUAGE_DETECTION_WORDS || + !hasEnoughCharacters(normalizedText) + ) { + return { valid: true, detections: [], reasons: [] } + } + + const detections = francAll(normalizedText).map(([language, accuracy]) => ({ + language, + accuracy, + })) + const englishScore = detections.find(({ language }) => language === 'eng')?.accuracy ?? 0 + const valid = englishScore >= MIN_ENGLISH_SUMMARY_SCORE + + return { + valid, + detections, + reasons: valid ? [] : ['insufficient-english-chunk-coverage'], + } +} diff --git a/packages/moderation/src/validators/language/tests.ts b/packages/moderation/src/validators/language/tests.ts index d5973d5b27..dc0fcc6557 100644 --- a/packages/moderation/src/validators/language/tests.ts +++ b/packages/moderation/src/validators/language/tests.ts @@ -4,131 +4,143 @@ import test from 'node:test' import { francAll } from 'franc-min' import { + analyzeLanguageChunks, + LANGUAGE_CHUNK_STRIDE_WORDS, + LANGUAGE_CHUNK_WORDS, + MIN_ENGLISH_CHUNK_PERCENTAGE, MIN_ENGLISH_SCORE, + MIN_ENGLISH_SUMMARY_SCORE, MIN_LANGUAGE_DETECTION_CHARACTERS, MIN_LANGUAGE_DETECTION_WORDS, validateEnglishSummaryText, validateEnglishText, + validateEnglishTextBlocks, } from './index.ts' -test('accepts text when franc scores English above the minimum score', () => { - const text = - 'This project adds configurable caves, useful tools, and polished world generation for every player.' - const result = validateEnglishText(text) - const english = result.detections.find(({ language }) => language === 'eng') - const alternative = result.detections.find(({ language }) => language !== 'eng') +const english = + 'This project adds useful tools, configurable settings, and clear documentation for every player.' +const russian = + 'Этот проект добавляет новые инструменты и значительно улучшает игровой процесс для всех игроков.' + +test('accepts English text and retains whole-text language diagnostics', () => { + const result = validateEnglishText(english) + const englishDetection = result.detections.find(({ language }) => language === 'eng') assert.equal(result.valid, true) - assert.ok(english) - assert.ok(alternative) - assert.ok(english.accuracy > MIN_ENGLISH_SCORE) + assert.ok(englishDetection) + assert.ok(englishDetection.accuracy > MIN_ENGLISH_SCORE) + assert.deepEqual(result.reasons, []) assert.deepEqual( result.detections, - francAll(text).map(([language, accuracy]) => ({ language, accuracy })), + francAll(english).map(([language, accuracy]) => ({ language, accuracy })), ) }) -test('accepts mixed English and Chinese text above the minimum score', () => { - const result = validateEnglishText( - 'A super light QQ bot for minecraft server and QQ group exchange msgs | 超轻量的QQ-MC群服插件', - ) - - assert.equal(result.valid, true) -}) - -test('rejects text when franc scores English below the minimum score', () => { +test('rejects text containing only confidently non-English chunks', () => { for (const text of [ - 'Чистый модпак для комфортной игры с друзьями, новыми заданиями и значительно улучшенной производительностью.', + russian, 'これは新しい洞窟と構造物を追加し、すべてのプレイヤーの世界生成を改善するプロジェクトです。', + 'Um modpack focado em desempenho, imersão e exploração, mantendo a experiência próxima ao jogo original.', ]) { - assert.equal(validateEnglishText(text).valid, false, text) + const result = validateEnglishText(text) + + assert.equal(result.valid, false, text) + assert.deepEqual(result.reasons, ['insufficient-english-chunk-coverage']) } }) -test('validates production project text using the English score threshold', () => { - const cases = [ - { - text: 'This mod adds some new things about turtles to Minecraft这个模组为Minecraft增加了一些关于乌龟的新东西', - valid: true, - }, - { - text: 'Tenhle Project má super mody ktere zlepší kvalitu hraní PVP', - valid: true, - }, - { - text: '此插件修复了Authme在lophine服务端上的登录漏洞 修复了玩家退出时SQL数据库的Logged依然为1的问题', - valid: false, - }, - { - text: 'Мод добавляет рубин — новый драгоценный камень. Добывайте рубиновую руду, кристаллизующуюся в толще камня, а закалив четыре рубина четырьмя незеритовыми ломами - можно будет сделать меч, кирку, броню и крюк захвата', - valid: false, - }, - { - text: 'Um modpack Fabric focado em desempenho, imersão e exploração, mantendo a experiência próxima ao Minecraft Vanilla. O objetivo é melhorar o visual, os sons, a geração de mundo e a qualidade de vida do jogo sem adicionar sistemas complexos.', - valid: true, - }, - { - text: 'A Create Tacz Warfare Modpack for the Server Create: Warfare', - valid: true, - }, - { - text: '一个集成了全息字和占位符创建的插件 A Plugin Integrating Holograms and Placeholder Support', - valid: true, - }, - { - text: "A modpack that adds stuff from TaCZ guns, to shaders, to curios slots, and even create! And also, Superb Warfare, in case TaCZ isn't for you!", - valid: true, - }, - { - text: 'Leichtes Client-Modpack für entspannte Feierabend-Sessions, bessere Performance, praktische QoL-Mods und ein aufgeräumtes Spielgefühl ohne unnötigen Ballast.', - valid: true, - }, - { - text: 'A super light QQ bot for minecraft server and QQ group exchange msgs | 超轻量的QQ-MC群服插件', - valid: true, - }, - { - text: 'Integrates MCP into minecraft, made for mapmakers and complex command block logic and datapack making.', - valid: true, - }, +test('accepts bilingual text when English chunks are 30% of classified chunks', () => { + const blocks = [ + english, + english, + english, + russian, + russian, + russian, + russian, + russian, + russian, + russian, ] + const analysis = analyzeLanguageChunks(blocks) + const result = validateEnglishTextBlocks(blocks) - for (const { text, valid } of cases) { - assert.equal(validateEnglishText(text).valid, valid, text) - } + assert.equal(analysis.englishChunks, 3) + assert.equal(analysis.nonEnglishChunks, 7) + assert.equal(analysis.englishChunkPercentage, 0.3) + assert.equal(MIN_ENGLISH_CHUNK_PERCENTAGE, 0.3) + assert.equal(result.valid, true) + assert.deepEqual(result.reasons, []) }) -test('skips language detection for production text below the word minimum', () => { - for (const text of [ - 'You can chat Gemini AI in Minecraft', - 'Create Mods X Zombie Apolcalypse', - 'Open-world zombie survival modpack', - "BIG-GOOSE Minecraft server's modpack", - ]) { - assert.deepEqual(validateEnglishText(text), { valid: true, detections: [] }, text) - } -}) - -test('skips language detection for text below the character minimum', () => { - const result = validateEnglishText('one two three four a b c d') - - assert.deepEqual(result, { valid: true, detections: [] }) - assert.equal(MIN_LANGUAGE_DETECTION_CHARACTERS, 35) -}) - -test('validates production text meeting both signal minimums', () => { - const text = '𝗔𝗶𝗺𝗶𝗻𝗴 𝘁𝗼 𝗲𝗻𝗵𝗮𝗻𝗰𝗲 𝗠𝗶𝗻𝗲𝗰𝗿𝗮𝗳𝘁 𝘄𝗵𝗶𝗹𝗲 𝗿𝗲𝘁𝗮𝗶𝗻𝗶𝗻𝗴 𝘁𝗵𝗮𝘁 𝗩𝗮𝗻𝗶𝗹𝗹𝗮 𝗳𝗲𝗲𝗹!' - const result = validateEnglishText(text) +test('rejects mixed-language text when English chunks are below 30% of classified chunks', () => { + const blocks = [english, english, russian, russian, russian, russian, russian] + const analysis = analyzeLanguageChunks(blocks) + const result = validateEnglishTextBlocks(blocks) + assert.equal(analysis.englishChunks, 2) + assert.equal(analysis.nonEnglishChunks, 5) + assert.ok((analysis.englishChunkPercentage ?? 0) < MIN_ENGLISH_CHUNK_PERCENTAGE) assert.equal(result.valid, false) - assert.ok(result.detections.length > 0) + assert.deepEqual(result.reasons, ['insufficient-english-chunk-coverage']) +}) + +test('uses overlapping 24-word windows with a 12-word stride', () => { + const text = Array.from({ length: 48 }, (_, index) => `word${index}`).join(' ') + const analysis = analyzeLanguageChunks([text]) + + assert.equal(LANGUAGE_CHUNK_WORDS, 24) + assert.equal(LANGUAGE_CHUNK_STRIDE_WORDS, 12) + assert.equal(analysis.totalChunks, 4) +}) + +test('skips chunks below the minimum word count', () => { + for (const text of [ + 'Minecraft', + 'Minecraft server', + 'This description has only seven English words', + ]) { + assert.deepEqual(validateEnglishText(text), { valid: true, detections: [], reasons: [] }) + } + assert.equal(MIN_LANGUAGE_DETECTION_WORDS, 8) }) -test('uses the same validation for summaries', () => { - assert.equal(validateEnglishSummaryText, validateEnglishText) +test('skips chunks below the minimum character count', () => { + assert.deepEqual(validateEnglishText('a b c d e f g h'), { + valid: true, + detections: [], + reasons: [], + }) + assert.equal(MIN_LANGUAGE_DETECTION_CHARACTERS, 35) +}) + +test('accepts summaries with an English score of at least 50% without changing description logic', () => { + const text = + 'Um modpack Fabric focado em desempenho, imersão e exploração, mantendo a experiência próxima ao Minecraft Vanilla.' + const summaryResult = validateEnglishSummaryText(text) + const englishDetection = summaryResult.detections.find(({ language }) => language === 'eng') + + assert.equal(MIN_ENGLISH_SUMMARY_SCORE, 0.5) + assert.ok(englishDetection) + assert.ok(englishDetection.accuracy >= MIN_ENGLISH_SUMMARY_SCORE) + assert.equal(summaryResult.valid, true) + assert.equal(validateEnglishText(text).valid, false) +}) + +test('skips summary detection below its word or character minimum', () => { + for (const text of ['This summary has only seven English words', 'a b c d e f g h']) { + assert.deepEqual(validateEnglishSummaryText(text), { + valid: true, + detections: [], + reasons: [], + }) + } + + assert.equal(MIN_LANGUAGE_DETECTION_WORDS, 8) + assert.equal(MIN_LANGUAGE_DETECTION_CHARACTERS, 35) }) test('allows empty text to be handled by required-field validation', () => { - assert.deepEqual(validateEnglishText(' '), { valid: true, detections: [] }) + assert.deepEqual(validateEnglishText(' '), { valid: true, detections: [], reasons: [] }) })