update 24-48hrs -> within a week to give more realistic estimate

fix: search indexing performance and batching (#6521 )
* Add consume batching delay * maybe fix * max batch size * more logging * parallelize remove tasks * delete/upsert project/version messages * prepare * more logging * log number of docs * try more targeted, homogenous version change ops * ensure only necessary fields are serialized into typesense * disable index background task * wip: script changes * don't facet by project id * fix * fix clippy * batch by document and dedup loaders * fix test * wip: projects/versions collections * clean up SearchBackend interface * cleanup pass * cleanup pass 2 * standardise fn names * cleanup pass * fix compile * factor out filter rewriting * query perf * put categories into the project doc * wip: search filter AST * doc comment * more AST normalization * (temp) convert search request errors into internal errors * implement unary NOT * tombi fmt * fix tests * revert request error * try increase stack size
2026-08-01 13:45:53 +00:00 · 2026-07-26 11:57:49 -07:00 · 2026-07-26 14:55:00 +00:00 · 2026-07-25 18:32:56 -07:00 · 2026-07-26 01:14:21 +00:00 · 2026-07-26 00:47:14 +00:00
56 changed files with 3099 additions and 860 deletions
@@ -1,5 +1,5 @@
 # Windows has stack overflows when calling from Tauri, so we increase the default stack size used by the compiler
-[target.'cfg(windows)']
+[target."cfg(windows)"]
 rustflags = ["-C", "link-args=/STACK:16777220"]

 [target.x86_64-pc-windows-msvc]
@@ -81,7 +81,7 @@ jobs:
      REDIS_CONNECTION_TYPE: multiplexed
      REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005
      # Avoid stack overflows in tests
-      RUST_MIN_STACK: 33554432
+      RUST_MIN_STACK: 67108864

    steps:
      - name: Check out code
@@ -5465,6 +5465,7 @@ dependencies = [
 "bytes",
 "censor",
 "chrono",
+ "chumsky",
 "clap 4.5.48",
 "clickhouse",
 "color-eyre",
@@ -5518,6 +5519,7 @@ dependencies = [
 "serde_with",
 "sha1 0.10.6",
 "sha2",
+ "smallvec",
 "spdx",
 "sqlx",
 "sqlx-tracing",
@@ -59,6 +59,7 @@ bytes = "1.10.1"
 censor = "0.3.0"
 chardetng = "0.1.17"
 chrono = "0.4.42"
+chumsky = "0.9.3"
 cidre = { version = "0.15.0", default-features = false, features = [
  "macos_15_0"
 ] }
@@ -192,6 +193,7 @@ sha1 = "0.10.6"
 sha1_smol = { version = "1.0.1", features = ["std"] }
 sha2 = "0.10.9"
 shlex = "1.3.0"
+smallvec = "1.15.1"
 spdx = "0.12.0"
 sqlx = { version = "0.8.6", default-features = false }
 sqlx-tracing = { path = "packages/sqlx-tracing" }
@@ -55,11 +55,11 @@ core-foundation.workspace = true
 core-graphics.workspace = true
 objc2-app-kit = { workspace = true, features = ["NSWindow"] }

-[target.'cfg(windows)'.dependencies]
+[target."cfg(windows)".dependencies]
 webview2-com.workspace = true
 windows-core.workspace = true

-[target.'cfg(windows)'.dependencies.windows]
+[target."cfg(windows)".dependencies.windows]
 workspace = true
 features = [
  "Win32_Foundation",
@@ -1,8 +1,16 @@
 <script setup lang="ts">
 import type { Labrinth } from '@modrinth/api-client'
-import { FolderSearchIcon, StarIcon, TrashIcon } from '@modrinth/assets'
+import {
+	FolderSearchIcon,
+	RotateCounterClockwiseIcon,
+	SpinnerIcon,
+	StarIcon,
+	TrashIcon,
+} from '@modrinth/assets'
 import {
 	ButtonStyled,
+	Combobox,
+	type ComboboxOption,
 	ConfirmModal,
 	defineMessages,
 	injectModrinthClient,
@@ -12,6 +20,7 @@ import {
 	type TableColumn,
 	useVIntl,
 } from '@modrinth/ui'
+import { renderString } from '@modrinth/utils'
 import { useQueryClient } from '@tanstack/vue-query'
 import { computed, ref, useTemplateRef } from 'vue'

@@ -52,6 +61,10 @@ const messages = defineMessages({
 		id: 'modpack-scan-modal.scanning',
 		defaultMessage: 'Scanning...',
 	},
+	success: {
+		id: 'modpack-scan-modal.success',
+		defaultMessage: 'Success',
+	},
 	failed: {
 		id: 'modpack-scan-modal.failed',
 		defaultMessage: 'Failed',
@@ -66,11 +79,40 @@ const messages = defineMessages({
 	},
 	scanError: {
 		id: 'modpack-scan-modal.scan-error',
-		defaultMessage: 'Some files failed to scan: {error}',
+		defaultMessage: 'Some files failed to scan: \n\n{error}',
 	},
-	clearAllGroups: {
-		id: 'modpack-scan-modal.clear-all-groups',
-		defaultMessage: 'Clear All Groups',
+	batchPlaceholder: {
+		id: 'modpack-scan-modal.batch.placeholder',
+		defaultMessage: 'Batch amount',
+	},
+	batchLabel: {
+		id: 'modpack-scan-modal.batch.label',
+		defaultMessage: '{amount} per batch',
+	},
+	deleteAllGroups: {
+		id: 'project.settings.permissions.delete-all-groups',
+		defaultMessage: 'Delete all groups',
+	},
+	deleteAllGroupsConfirmationTitle: {
+		id: 'project.settings.permissions.delete-all-groups-confirmation.title',
+		defaultMessage: 'Delete all attribution groups?',
+	},
+	deleteAllGroupsConfirmationDescription: {
+		id: 'modpack-scan-modal.delete-all-groups-confirmation.description',
+		defaultMessage:
+			'This will permanently delete all attribution groups for this project and all files inside them. This action cannot be undone.',
+	},
+	deleteAllGroupsConfirmationProceed: {
+		id: 'modpack-scan-modal.delete-all-groups.proceed',
+		defaultMessage: 'Clear',
+	},
+	deleteAllGroupsSuccess: {
+		id: 'modpack-scan-modal.delete-all-groups.success',
+		defaultMessage: 'All groups cleared successfully.',
+	},
+	deleteAllGroupsError: {
+		id: 'modpack-scan-modal.delete-all-groups.error',
+		defaultMessage: 'Failed to clear all groups: {error}',
 	},
 })

@@ -98,23 +140,64 @@ const modalRef = useTemplateRef<InstanceType<typeof NewModal>>('modalRef')
 const clearModalRef = useTemplateRef<InstanceType<typeof ConfirmModal>>('clearModalRef')
 const { formatMessage } = useVIntl()

-const rows = ref<ScanRow[]>([])
+const rows = ref<Record<string, ScanRow>>({})
 const isLoadingVersions = ref(false)
 const isScanning = ref(false)
 const isClearing = ref(false)
 const versionLoadError = ref<string | null>(null)
-const scanError = ref<string | null>(null)
 const requestId = ref(0)
 const scanRequestId = ref(0)

+const DEFAULT_BATCH_AMOUNT = 10
+const batchAmountOptions: ComboboxOption<number>[] = [
+	{ value: 1, label: '1' },
+	{ value: 5, label: '5' },
+	{ value: 10, label: '10' },
+	{ value: 20, label: '20' },
+	{ value: 50, label: '50' },
+]
+const batchAmountValues = batchAmountOptions.map((opt) => opt.value)
+const batchAmountCookie = useCookie<number>('moderation-modpack-scan-batch', {
+	default: () => DEFAULT_BATCH_AMOUNT,
+	maxAge: 60 * 60 * 24 * 365,
+	sameSite: 'lax',
+	path: '/',
+})
+const batchAmount = computed({
+	get() {
+		const value = Number(batchAmountCookie.value)
+		return batchAmountValues.includes(value) ? value : DEFAULT_BATCH_AMOUNT
+	},
+	set(value: number) {
+		batchAmountCookie.value = value
+	},
+})
+
 const columns = computed<TableColumn<ScanTableColumn>[]>(() => [
 	{ key: 'filename', label: formatMessage(messages.packFileName), width: '60%' },
 	{ key: 'newFiles', label: formatMessage(messages.newFiles), align: 'center', width: '20%' },
 	{ key: 'newGroups', label: formatMessage(messages.newGroups), align: 'center', width: '20%' },
 ])

-const scannedCount = computed(() => rows.value.filter((row) => row.scan || row.error).length)
+const scannedCount = computed(
+	() => Object.entries(rows.value).filter(([_, row]) => row.scan || row.error).length,
+)
 const isBusy = computed(() => isLoadingVersions.value || isScanning.value || isClearing.value)
+const titleButtonsDisabled = computed(() => isBusy.value || Object.keys(rows.value).length === 0)
+const rescanButtonsDisabled = computed(() => isLoadingVersions.value || isClearing.value)
+
+const rowErrors = computed(() =>
+	Object.entries(rows.value)
+		.filter(([_, row]) => row.error)
+		.map(([_, row]) => row),
+)
+
+const rowScanError = computed(() => {
+	if (rowErrors.value.length === 0) return undefined
+	return formatMessage(messages.scanError, {
+		error: rowErrors.value.map((r) => `\n - ${r.filename}`).join(''),
+	})
+})

 function getErrorMessage(error: unknown) {
 	if (error instanceof Error) {
@@ -131,12 +214,27 @@ function getErrorMessage(error: unknown) {
 	return String(error)
 }

+async function runWithConcurrency<T>(
+	items: T[],
+	limit: number,
+	task: (item: T) => Promise<void>,
+): Promise<void> {
+	const queue = [...items]
+	const workers = Array.from({ length: limit }, async () => {
+		while (queue.length) {
+			const item = queue.shift()
+			if (item === undefined) return
+			await task(item)
+		}
+	})
+	await Promise.all(workers)
+}
+
 async function fetchAllVersions() {
 	const currentRequestId = ++requestId.value
 	isLoadingVersions.value = true
 	versionLoadError.value = null
-	scanError.value = null
-	rows.value = []
+	rows.value = {}

 	try {
 		const versions = await client.labrinth.versions_v2.getProjectVersions(props.project_id)
@@ -144,17 +242,20 @@ async function fetchAllVersions() {
 			return
 		}

-		rows.value = versions
+		const filteredVersions = versions
 			.flatMap((version) => version.files)
 			.filter((file): file is Labrinth.Versions.v2.VersionFile & { id: string } => Boolean(file.id))
-			.map((file) => ({
-				id: file.id,
-				filename: file.filename,
-				primary: file.primary,
+
+		for (const version of filteredVersions) {
+			rows.value[version.id] = {
+				id: version.id,
+				filename: version.filename,
+				primary: version.primary,
 				isScanning: false,
 				newFiles: undefined,
 				newGroups: undefined,
-			}))
+			}
+		}
 	} catch (error) {
 		if (currentRequestId === requestId.value) {
 			versionLoadError.value = formatMessage(messages.loadVersionsError, {
@@ -168,54 +269,45 @@ async function fetchAllVersions() {
 	}
 }

+async function fetchScan(id: string) {
+	rows.value[id].isScanning = true
+	try {
+		const scan = await client.labrinth.attribution_internal.scanFile(id)
+
+		rows.value[id].scan = scan
+		rows.value[id].newFiles = scan.new_attribution_files
+		rows.value[id].newGroups = scan.new_attribution_groups
+
+		rows.value[id].error = undefined
+	} catch (error) {
+		rows.value[id].error = getErrorMessage(error)
+	} finally {
+		rows.value[id].isScanning = false
+	}
+}
+
 async function fetchAllScans() {
-	if (isBusy.value) {
-		return
-	}
+	if (isBusy.value) return

-	const currentScanRequestId = ++scanRequestId.value
 	isScanning.value = true
-	scanError.value = null
-	rows.value = rows.value.map((row) => ({
-		...row,
-		scan: undefined,
-		isScanning: false,
-		error: undefined,
-		newFiles: undefined,
-		newGroups: undefined,
-	}))
+
+	Object.entries(rows.value).map(([id, row]) => {
+		rows.value[id] = {
+			...row,
+			scan: undefined,
+			isScanning: false,
+			error: undefined,
+			newFiles: undefined,
+			newGroups: undefined,
+		}
+	})

 	try {
-		for (const row of rows.value) {
-			if (currentScanRequestId !== scanRequestId.value) {
-				return
-			}
-
-			row.isScanning = true
-			try {
-				const scan = await client.labrinth.attribution_internal.scanFile(row.id)
-				if (currentScanRequestId !== scanRequestId.value) {
-					return
-				}
-
-				row.scan = scan
-				row.newFiles = scan.new_attribution_files
-				row.newGroups = scan.new_attribution_groups
-			} catch (error) {
-				if (currentScanRequestId !== scanRequestId.value) {
-					return
-				}
-
-				row.error = getErrorMessage(error)
-				scanError.value = formatMessage(messages.scanError, { error: row.error })
-			} finally {
-				row.isScanning = false
-			}
-		}
+		await runWithConcurrency(Object.keys(rows.value), batchAmount.value, async (id: string) => {
+			await fetchScan(id)
+		})
 	} finally {
-		if (currentScanRequestId === scanRequestId.value) {
-			isScanning.value = false
-		}
+		isScanning.value = false
 	}
 }

@@ -242,8 +334,8 @@ async function clearAllGroups() {
 		failed = true
 		addNotification({
 			type: 'error',
-			title: 'An error occurred',
-			text: `Failed to clear all groups: ${getErrorMessage(error)}`,
+			title: formatMessage(messages.failed),
+			text: formatMessage(messages.deleteAllGroupsError, { error: getErrorMessage(error) }),
 		})
 	} finally {
 		isClearing.value = false
@@ -252,8 +344,8 @@ async function clearAllGroups() {
 	if (!failed) {
 		addNotification({
 			type: 'success',
-			title: 'Success',
-			text: 'All groups cleared successfully.',
+			title: formatMessage(messages.success),
+			text: formatMessage(messages.deleteAllGroupsSuccess),
 		})
 	}
 }
@@ -261,7 +353,7 @@ async function clearAllGroups() {
 function show() {
 	scanRequestId.value++
 	isScanning.value = false
-	rows.value = []
+	rows.value = {}
 	void fetchAllVersions()
 	modalRef.value?.show()
 }
@@ -275,9 +367,9 @@ defineExpose({ show, hide })
 <template>
 	<ConfirmModal
 		ref="clearModalRef"
-		title="Clear all permission groups?"
-		description="This will clear **all** groups for this project. This action cannot be undone."
-		proceed-label="Clear"
+		:title="formatMessage(messages.deleteAllGroupsConfirmationTitle)"
+		:description="formatMessage(messages.deleteAllGroupsConfirmationDescription)"
+		:proceed-label="formatMessage(messages.deleteAllGroupsConfirmationProceed)"
 		@proceed="clearAllGroups"
 	/>

@@ -294,27 +386,43 @@ defineExpose({ show, hide })
 					{{
 						formatMessage(messages.title, {
 							scanned: scannedCount,
-							total: rows.length,
+							total: Object.keys(rows).length,
 						})
 					}}
 				</span>
 				<div class="flex items-center gap-2">
+					<Combobox
+						v-model="batchAmount"
+						:options="batchAmountOptions"
+						:disabled="titleButtonsDisabled"
+						:placeholder="formatMessage(messages.batchPlaceholder)"
+					>
+						<template #selected>
+							<span class="flex flex-row gap-2 align-middle font-semibold">
+								<span class="truncate text-contrast">{{
+									formatMessage(messages.batchLabel, { amount: batchAmount })
+								}}</span>
+							</span>
+						</template>
+					</Combobox>
 					<ButtonStyled circular color="red" color-fill="none">
 						<button
-							v-tooltip="formatMessage(messages.clearAllGroups)"
-							:disabled="isBusy || rows.length === 0"
+							v-tooltip="formatMessage(messages.deleteAllGroups)"
+							:disabled="titleButtonsDisabled"
 							@click="showConfirmClearGroups"
 						>
-							<TrashIcon aria-hidden="true" />
+							<TrashIcon v-if="!isClearing" aria-hidden="true" />
+							<SpinnerIcon v-else class="animate-spin" />
 						</button>
 					</ButtonStyled>
 					<ButtonStyled circular>
 						<button
 							v-tooltip="formatMessage(messages.scanAllFiles)"
-							:disabled="isBusy || rows.length === 0"
+							:disabled="titleButtonsDisabled"
 							@click="fetchAllScans"
 						>
-							<FolderSearchIcon aria-hidden="true" />
+							<FolderSearchIcon v-if="!isScanning" aria-hidden="true" />
+							<SpinnerIcon v-else class="animate-spin" />
 						</button>
 					</ButtonStyled>
 				</div>
@@ -323,14 +431,14 @@ defineExpose({ show, hide })

 		<div class="w-full">
 			<div
-				v-if="versionLoadError || scanError"
-				class="mb-3 rounded-xl bg-highlight-red p-3 text-red"
+				v-if="versionLoadError || rowScanError"
+				class="mb-3 rounded-xl bg-highlight-red px-4 py-1 text-red"
 			>
-				{{ versionLoadError || scanError }}
+				<div v-html="renderString((versionLoadError || rowScanError) ?? '')"></div>
 			</div>
 			<Table
 				:columns="columns"
-				:data="rows"
+				:data="Object.entries(rows).map(([_, row]) => row)"
 				row-key="id"
 				:row-below-visible="
 					(row) => Boolean(row.scan?.scanned_file_names && row.scan.scanned_file_names.length > 0)
@@ -345,16 +453,36 @@ defineExpose({ show, hide })
 				</template>
 				<template #cell-newFiles="{ row }">
 					<span v-if="row.isScanning">{{ formatMessage(messages.scanning) }}</span>
-					<span v-else-if="row.error" v-tooltip="row.error" class="text-red">
-						{{ formatMessage(messages.failed) }}
+					<span v-else-if="row.error" v-tooltip="row.error" class="flex justify-center">
+						<ButtonStyled
+							class="justify-self-center"
+							color="red"
+							type="outlined"
+							hover-color-fill="background"
+						>
+							<button :disabled="rescanButtonsDisabled" @click="() => fetchScan(row.id)">
+								<RotateCounterClockwiseIcon />
+								{{ formatMessage(messages.failed) }}
+							</button>
+						</ButtonStyled>
 					</span>
 					<span v-else-if="row.scan">{{ row.scan.new_attribution_files }}</span>
 					<span v-else>{{ formatMessage(messages.notScanned) }}</span>
 				</template>
 				<template #cell-newGroups="{ row }">
 					<span v-if="row.isScanning">{{ formatMessage(messages.scanning) }}</span>
-					<span v-else-if="row.error" v-tooltip="row.error" class="text-red">
-						{{ formatMessage(messages.failed) }}
+					<span v-else-if="row.error" v-tooltip="row.error" class="flex justify-center">
+						<ButtonStyled
+							class="justify-self-center"
+							color="red"
+							type="outlined"
+							hover-color-fill="background"
+						>
+							<button :disabled="rescanButtonsDisabled" @click="() => fetchScan(row.id)">
+								<RotateCounterClockwiseIcon />
+								{{ formatMessage(messages.failed) }}
+							</button>
+						</ButtonStyled>
 					</span>
 					<span v-else-if="row.scan">{{ row.scan.new_attribution_groups }}</span>
 					<span v-else>{{ formatMessage(messages.notScanned) }}</span>
@@ -2972,9 +2972,6 @@
  "moderation.page.technicalReview": {
    "message": "Technische Überprüfung"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Alle Gruppen entfernen"
-  },
  "modpack-scan-modal.failed": {
    "message": "Fehlgeschlagen"
  },
@@ -2972,9 +2972,6 @@
  "moderation.page.technicalReview": {
    "message": "Technische Überprüfung"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Alle Gruppen entfernen"
-  },
  "modpack-scan-modal.failed": {
    "message": "Fehlgeschlagen"
  },
@@ -2972,8 +2972,23 @@
  "moderation.page.technicalReview": {
    "message": "Tech review"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Clear All Groups"
+  "modpack-scan-modal.batch.label": {
+    "message": "{amount} per batch"
+  },
+  "modpack-scan-modal.batch.placeholder": {
+    "message": "Batch amount"
+  },
+  "modpack-scan-modal.delete-all-groups-confirmation.description": {
+    "message": "This will permanently delete all attribution groups for this project and all files inside them. This action cannot be undone."
+  },
+  "modpack-scan-modal.delete-all-groups.error": {
+    "message": "Failed to clear all groups: {error}"
+  },
+  "modpack-scan-modal.delete-all-groups.proceed": {
+    "message": "Clear"
+  },
+  "modpack-scan-modal.delete-all-groups.success": {
+    "message": "All groups cleared successfully."
  },
  "modpack-scan-modal.failed": {
    "message": "Failed"
@@ -3006,11 +3021,14 @@
    "message": "Scan All Files"
  },
  "modpack-scan-modal.scan-error": {
-    "message": "Some files failed to scan: {error}"
+    "message": "Some files failed to scan: \n\n{error}"
  },
  "modpack-scan-modal.scanning": {
    "message": "Scanning..."
  },
+  "modpack-scan-modal.success": {
+    "message": "Success"
+  },
  "modpack-scan-modal.title": {
    "message": "Modpack Scan ({scanned}/{total} Files)"
  },
@@ -3786,7 +3804,10 @@
    "message": "You can still modify your project, it won't affect your position in the queue."
  },
  "project.moderation.admonition.under-review.body.4": {
-    "message": "We aim to review submissions in 24-48 hours, but some projects may face delays. This does not reflect an issue with your submission."
+    "message": "We aim to review submissions in 24–48 hours, but some projects may face delays. This does not reflect an issue with your submission."
+  },
+  "project.moderation.admonition.under-review.body.4.alt-week": {
+    "message": "We aim to review submissions within a week, but some projects may face delays. This does not reflect an issue with your submission."
  },
  "project.moderation.admonition.under-review.body.5": {
    "message": "<emphasis>We appreciate your patience while our moderators work hard to keep Modrinth safe, and look forward to helping you share your content! 💚</emphasis>"
@@ -2954,9 +2954,6 @@
  "moderation.page.technicalReview": {
    "message": "Revue technique"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Effacer tous les groupes"
-  },
  "modpack-scan-modal.failed": {
    "message": "Échec"
  },
@@ -2957,9 +2957,6 @@
  "moderation.page.technicalReview": {
    "message": "Revisione tecnica"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Svuota gruppi"
-  },
  "modpack-scan-modal.failed": {
    "message": "Errore"
  },
@@ -2942,9 +2942,6 @@
  "moderation.page.technicalReview": {
    "message": "기술 리뷰"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "모든 그룹 삭제"
-  },
  "modpack-scan-modal.failed": {
    "message": "실패"
  },
@@ -2939,9 +2939,6 @@
  "moderation.page.technicalReview": {
    "message": "Technische beoordeling"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Alle groepen wissen"
-  },
  "modpack-scan-modal.failed": {
    "message": "Mislukt"
  },
@@ -2942,9 +2942,6 @@
  "moderation.page.technicalReview": {
    "message": "Revisão técnica"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Limpar todos os grupos"
-  },
  "modpack-scan-modal.failed": {
    "message": "Falhou"
  },
@@ -2960,9 +2960,6 @@
  "moderation.page.technicalReview": {
    "message": "Техпроверка"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Очистить все группы"
-  },
  "modpack-scan-modal.failed": {
    "message": "Ошибка"
  },
@@ -2936,9 +2936,6 @@
  "moderation.page.technicalReview": {
    "message": "Технічний огляд"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "Очистити всі групи"
-  },
  "modpack-scan-modal.failed": {
    "message": "Помилка"
  },
@@ -2969,9 +2969,6 @@
  "moderation.page.technicalReview": {
    "message": "技术审查"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "清除所有分组"
-  },
  "modpack-scan-modal.failed": {
    "message": "失败"
  },
@@ -2948,9 +2948,6 @@
  "moderation.page.technicalReview": {
    "message": "技術審查"
  },
-  "modpack-scan-modal.clear-all-groups": {
-    "message": "清除所有群組"
-  },
  "modpack-scan-modal.failed": {
    "message": "失敗"
  },
@@ -72,7 +72,7 @@
 					<h2 id="messages" class="m-0 text-xl font-semibold text-contrast">
 						{{ formatMessage(messages.threadSectionTitle) }}
 					</h2>
-					<div v-if="isStaff(currentMember?.user)" class="flex items-center gap-2">
+					<div v-if="staff" class="flex items-center gap-2">
 						<Toggle id="moderator-see-user-ui-toggle" v-model="moderatorSeeUserUi" small />
 						<label for="moderator-see-user-ui-toggle"> Show member UI </label>
 					</div>
@@ -218,7 +218,10 @@ const {
 } = injectProjectPageContext()

 const canAccess = computed(() => !!currentMember.value)
-const userFacingUiVisible = computed(() => !!currentMember.value && moderatorSeeUserUi.value)
+const staff = computed(() => isStaff(currentMember.value?.user))
+const userFacingUiVisible = computed(
+	() => !!currentMember.value && (!staff.value || moderatorSeeUserUi.value),
+)

 const approvedAdmonitionMessage = computed<MessageDescriptor | null>(() => {
 	switch (project.value?.status) {
@@ -330,10 +333,11 @@ const moderationAdmonition = computed<{
 							defaultMessage:
 								"You can still modify your project, it won't affect your position in the queue.",
 						}),
+						// temp moved 24-48 hr below to keep old translation for future
 						defineMessage({
-							id: 'project.moderation.admonition.under-review.body.4',
+							id: 'project.moderation.admonition.under-review.body.4.alt-week',
 							defaultMessage:
-								'We aim to review submissions in 24-48 hours, but some projects may face delays. This does not reflect an issue with your submission.',
+								'We aim to review submissions within a week, but some projects may face delays. This does not reflect an issue with your submission.',
 						}),
 					],
 				},
@@ -400,6 +404,13 @@ const moderationAdmonition = computed<{
 	return null
 })

+// unused 24-48hr message still defined here for later
+defineMessage({
+	id: 'project.moderation.admonition.under-review.body.4',
+	defaultMessage:
+		'We aim to review submissions in 24–48 hours, but some projects may face delays. This does not reflect an issue with your submission.',
+})
+
 const moderatorSeeUserUi = computed<boolean>({
 	get() {
 		return flags.value.showModeratorProjectMemberUi
@@ -62,6 +62,14 @@
 				</span>
 			</label>
 			<StyledInput id="username-field" v-model="current.username" />
+			<div
+				v-if="current.username.length >= 30"
+				id="bio-character-limit"
+				class="inline-block pl-2"
+				:class="{ 'text-red': current.username.length > 39 }"
+			>
+				{{ current.username.length }}/{{ 39 }}
+			</div>
 			<label for="bio-field">
 				<span class="label__title">{{ formatMessage(messages.bioTitle) }}</span>
 				<span class="label__description">
@@ -69,6 +77,9 @@
 				</span>
 			</label>
 			<StyledInput id="bio-field" v-model="current.bio" multiline />
+			<div id="bio-character-limit" class="pt-2" :class="{ 'text-red': current.bio.length > 160 }">
+				{{ current.bio.length }}/{{ 160 }}
+			</div>
 			<div class="input-group mt-4">
 				<ButtonStyled>
 					<NuxtLink :to="`/user/${auth.user.username}`">
@@ -26,9 +26,12 @@ ELASTICSEARCH_INDEX_PREFIX=labrinth
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=elastic
 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000

 REDIS_MODE=standalone
 REDIS_CONNECTION_TYPE=pooled
@@ -44,9 +44,12 @@ ELASTICSEARCH_USERNAME=
 ELASTICSEARCH_PASSWORD=

 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000

 REDIS_MODE=standalone
 REDIS_CONNECTION_TYPE=pooled
@@ -35,6 +35,7 @@ bitflags = { workspace = true }
 bytes = { workspace = true }
 censor = { workspace = true }
 chrono = { workspace = true, features = ["serde"] }
+chumsky = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 clickhouse = { workspace = true, features = ["time", "uuid"] }
 color-eyre = { workspace = true }
@@ -114,6 +115,7 @@ serde_json = { workspace = true }
 serde_with = { workspace = true }
 sha1 = { workspace = true }
 sha2 = { workspace = true }
+smallvec = { workspace = true }
 spdx = { workspace = true, features = ["text"] }
 sqlx = { workspace = true, features = [
  "chrono",
@@ -18,7 +18,7 @@ use crate::util::anrok;
 use actix_web::web;
 use clap::ValueEnum;
 use eyre::WrapErr;
-use tracing::info;
+use tracing::{info, instrument};
 use xredis::RedisPool;

 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
@@ -50,6 +50,7 @@ pub enum BackgroundTask {

 impl BackgroundTask {
    #[allow(clippy::too_many_arguments)]
+    #[instrument(skip_all, fields(background_task = ?self))]
    pub async fn run(
        self,
        pool: PgPool,
@@ -176,7 +177,7 @@ pub async fn index_search(
    search_backend: web::Data<dyn SearchBackend>,
 ) -> eyre::Result<()> {
    info!("Indexing local database");
-    search_backend.index_projects(ro_pool, redis_pool).await
+    search_backend.rebuild_index(ro_pool, redis_pool).await
 }

 pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {
@@ -43,7 +43,7 @@ macro_rules! vars {
                    )]
                    let $field: Option<$ty> = {
                        let mut default = None::<$ty>;
-                        $( default = Some({ $default }.into()); )?
+                        $( default = Some(<$ty>::from({ $default })); )?

                        match parse_value::<$ty>(stringify!($field), default) {
                            Ok(value) => Some(value),
@@ -208,9 +208,14 @@ vars! {
    // search
    SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
    SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS: u64 = 5u64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE: usize = 1000usize;
    TYPESENSE_URL: String = "http://localhost:8108";
    TYPESENSE_API_KEY: String = "modrinth";
    TYPESENSE_INDEX_PREFIX: String = "labrinth";
+    TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize;
+    TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize;
+    TYPESENSE_USE_CACHE: bool = true;

    // storage
    STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
@@ -131,30 +131,6 @@ pub fn app_setup(
    ));

    if enable_background_tasks {
-        // The interval in seconds at which the local database is indexed
-        // for searching.  Defaults to 1 hour if unset.
-        let local_index_interval =
-            Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
-        let pool_ref = pool.clone();
-        let redis_pool_ref = redis_pool.clone();
-        let search_backend_ref = search_backend.clone();
-        scheduler.run(local_index_interval, move || {
-            let pool_ref = pool_ref.clone();
-            let redis_pool_ref = redis_pool_ref.clone();
-            let search_backend = search_backend_ref.clone();
-            async move {
-                if let Err(err) = background_task::index_search(
-                    pool_ref,
-                    redis_pool_ref,
-                    search_backend,
-                )
-                .await
-                {
-                    warn!("Failed to index search: {err:?}");
-                }
-            }
-        });
-
        // Changes statuses of scheduled projects/versions
        let pool_ref = pool.clone();
        // TODO: Clear cache when these are run
@@ -178,8 +178,9 @@ impl ServerPingQueue {
                        None,
                        &self.redis,
                    );
-                    let queue_search =
-                        self.incremental_search_queue.push(*project_id);
+                    let queue_search = self
+                        .incremental_search_queue
+                        .push_project_change(*project_id);

                    let (clear_cache_result, _) =
                        join(clear_cache, queue_search).await;
@@ -7,7 +7,7 @@ use crate::queue::analytics::AnalyticsQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::search::SearchBackend;
-use crate::search::incremental::consume::reindex_project;
+use crate::search::incremental::consume::reindex_project_document;
 use crate::util::date::get_current_tenths_of_ms;
 use crate::util::error::Context;
 use crate::util::guards::admin_key_guard;
@@ -330,7 +330,7 @@ pub async fn force_reindex(
 ) -> Result<HttpResponse, ApiError> {
    let redis = redis.get_ref();
    search_backend
-        .index_projects(pool.as_ref().clone(), redis.clone())
+        .rebuild_index(pool.as_ref().clone(), redis.clone())
        .await
        .wrap_internal_err("failed to index projects")?;
    Ok(HttpResponse::NoContent().finish())
@@ -355,7 +355,7 @@ pub async fn force_reindex_project(
    search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
    let (project_id,) = path.into_inner();
-    reindex_project(
+    reindex_project_document(
        pool.as_ref(),
        redis.as_ref(),
        search_backend.as_ref(),
@@ -10,7 +10,7 @@ use crate::models::projects::{
 use crate::models::v2::projects::LegacyVersion;
 use crate::queue::session::AuthQueue;
 use crate::routes::{v2_reroute, v3};
-use crate::search::{SearchBackend, SearchState};
+use crate::search::SearchState;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
@@ -488,7 +488,6 @@ pub async fn version_delete(
    pool: web::Data<PgPool>,
    redis: web::Data<RedisPool>,
    session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
    search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
    // Returns NoContent, so we don't need to convert the response
@@ -498,7 +497,6 @@ pub async fn version_delete(
        pool,
        redis,
        session_queue,
-        search_backend,
        search_state,
    )
    .await
@@ -85,7 +85,11 @@ pub async fn clear_project_cache_and_queue_search(
        redis,
    )
    .await?;
-    search_state.queue.push(project_id.into()).await;
+
+    search_state
+        .queue
+        .push_project_change(project_id.into())
+        .await;

    Ok(())
 }
@@ -1053,10 +1057,10 @@ pub async fn project_edit_internal(
        edit: Option<Option<E>>,
        mut component: &mut Option<E::Component>,
        perms: ProjectPermissions,
-    ) -> Result<(), ApiError> {
+    ) -> Result<bool, ApiError> {
        let Some(edit) = edit else {
            // component is not specified in the input JSON - leave alone
-            return Ok(());
+            return Ok(false);
        };

        if !perms.contains(ProjectPermissions::EDIT_DETAILS) {
@@ -1095,10 +1099,13 @@ pub async fn project_edit_internal(
            }
        }

-        Ok(())
+        Ok(true)
    }

-    update(
+    let mut reindex_versions = new_project.categories.is_some()
+        || new_project.additional_categories.is_some();
+
+    reindex_versions |= update(
        &mut transaction,
        id,
        new_project.minecraft_server,
@@ -1106,7 +1113,7 @@ pub async fn project_edit_internal(
        perms,
    )
    .await?;
-    update(
+    reindex_versions |= update(
        &mut transaction,
        id,
        new_project.minecraft_java_server,
@@ -1114,7 +1121,7 @@ pub async fn project_edit_internal(
        perms,
    )
    .await?;
-    update(
+    reindex_versions |= update(
        &mut transaction,
        id,
        new_project.minecraft_bedrock_server,
@@ -1167,14 +1174,31 @@ pub async fn project_edit_internal(

    transaction.commit().await?;

-    clear_project_cache_and_queue_search(
-        &redis,
-        &search_state,
-        project_item.inner.id,
-        project_item.inner.slug,
-        None,
-    )
-    .await?;
+    if reindex_versions {
+        db_models::DBProject::clear_cache(
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+            &redis,
+        )
+        .await?;
+        search_state
+            .queue
+            .push_version_changes(
+                project_item.inner.id.into(),
+                project_item.versions.iter().copied().map(VersionId::from),
+            )
+            .await;
+    } else {
+        clear_project_cache_and_queue_search(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+        )
+        .await?;
+    }

    // Remove no longer searchable projects from search index
    if let (true, Some(false)) = (
@@ -1182,16 +1206,9 @@ pub async fn project_edit_internal(
        new_project.status.map(|status| status.is_searchable()),
    ) {
        search_state
-            .backend
-            .remove_documents(
-                &project_item
-                    .versions
-                    .into_iter()
-                    .map(|x| x.into())
-                    .collect::<Vec<_>>(),
-            )
-            .await
-            .wrap_internal_err("failed to remove documents")?;
+            .queue
+            .push_project_removal(project_item.inner.id.into())
+            .await;
    }

    Ok(HttpResponse::NoContent().body(""))
@@ -1654,7 +1671,7 @@ pub async fn projects_edit(
            };
        }

-        bulk_edit_project_categories(
+        let mut reindex_versions = bulk_edit_project_categories(
            &categories,
            &project.categories,
            project.inner.id as db_ids::DBProjectId,
@@ -1669,7 +1686,7 @@ pub async fn projects_edit(
        )
        .await?;

-        bulk_edit_project_categories(
+        reindex_versions |= bulk_edit_project_categories(
            &categories,
            &project.additional_categories,
            project.inner.id as db_ids::DBProjectId,
@@ -1728,20 +1745,37 @@ pub async fn projects_edit(
            }
        }

-        changed_projects.push((project.inner.id, project.inner.slug));
+        changed_projects.push((
+            project.inner.id,
+            project.inner.slug,
+            project.versions,
+            reindex_versions,
+        ));
    }

    transaction.commit().await?;

-    for (project_id, slug) in changed_projects {
-        clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
-            project_id,
-            slug,
-            None,
-        )
-        .await?;
+    for (project_id, slug, versions, reindex_versions) in changed_projects {
+        if reindex_versions {
+            db_models::DBProject::clear_cache(project_id, slug, None, &redis)
+                .await?;
+            search_state
+                .queue
+                .push_version_changes(
+                    project_id.into(),
+                    versions.into_iter().map(VersionId::from),
+                )
+                .await;
+        } else {
+            clear_project_cache_and_queue_search(
+                &redis,
+                &search_state,
+                project_id,
+                slug,
+                None,
+            )
+            .await?;
+        }
    }

    Ok(HttpResponse::NoContent().body(""))
@@ -1755,7 +1789,7 @@ pub async fn bulk_edit_project_categories(
    max_num_categories: usize,
    is_additional: bool,
    transaction: &mut PgTransaction<'_>,
-) -> Result<(), ApiError> {
+) -> Result<bool, ApiError> {
    let mut set_categories =
        if let Some(categories) = bulk_changes.categories.clone() {
            categories
@@ -1782,7 +1816,8 @@ pub async fn bulk_edit_project_categories(
        }
    }

-    if &set_categories != project_categories {
+    let changed = &set_categories != project_categories;
+    if changed {
        sqlx::query!(
            "
            DELETE FROM mods_categories
@@ -1815,7 +1850,7 @@ pub async fn bulk_edit_project_categories(
        DBModCategory::insert_many(mod_categories, &mut *transaction).await?;
    }

-    Ok(())
+    Ok(changed)
 }

 #[derive(Serialize, Deserialize)]
@@ -2791,27 +2826,18 @@ pub async fn project_delete_internal(
        .await
        .wrap_internal_err("failed to commit transaction")?;

-    search_state
-        .backend
-        .remove_documents(
-            &project
-                .versions
-                .into_iter()
-                .map(|x| x.into())
-                .collect::<Vec<_>>(),
-        )
-        .await
-        .wrap_internal_err("failed to remove project version documents")?;
-
    if result.is_some() {
-        clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
+        db_models::DBProject::clear_cache(
            project.inner.id,
            project.inner.slug,
            None,
+            &redis,
        )
        .await?;
+        search_state
+            .queue
+            .push_project_removal(project.inner.id.into())
+            .await;
        Ok(())
    } else {
        Err(ApiError::NotFound)
@@ -184,19 +184,20 @@ pub async fn version_create(
        if let Err(e) = rollback_result {
            return Err(e.into());
        }
-    } else if let Ok((_, project_id)) = &result {
+    } else if let Ok((_, project_id, version_id)) = &result {
        transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
-            *project_id,
-            None,
-            Some(true),
-        )
-        .await?;
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
+            .await?;
+        search_state
+            .queue
+            .push_version_changes(
+                (*project_id).into(),
+                [VersionId::from(*version_id)],
+            )
+            .await;
    }

-    result.map(|(response, _)| response)
+    result.map(|(response, _, _)| response)
 }

 #[allow(clippy::too_many_arguments)]
@@ -210,7 +211,8 @@ async fn version_create_inner(
    pool: &PgPool,
    session_queue: &AuthQueue,
    http: &reqwest::Client,
-) -> Result<(HttpResponse, models::DBProjectId), CreateError> {
+) -> Result<(HttpResponse, models::DBProjectId, models::DBVersionId), CreateError>
+{
    let mut initial_version_data = None;
    let mut version_builder = None;
    let mut selected_loaders = None;
@@ -568,7 +570,11 @@ async fn version_create_inner(
        }
    }

-    Ok((HttpResponse::Ok().json(response), project_id))
+    Ok((
+        HttpResponse::Ok().json(response),
+        project_id,
+        models::DBVersionId::from(version_id),
+    ))
 }

 /// Add files to an existing version.
@@ -635,17 +641,18 @@ pub async fn upload_file_to_version(
    let mut transaction = client.begin().await?;
    let mut uploaded_files = Vec::new();

-    let version_id = models::DBVersionId::from(url_data.into_inner().0);
+    let version_id = url_data.into_inner().0;
+    let db_version_id = models::DBVersionId::from(version_id);

    let result = upload_file_to_version_inner(
        req,
        &mut payload,
-        client,
+        client.clone(),
        &mut transaction,
        redis.clone(),
        &**file_host,
        &mut uploaded_files,
-        version_id,
+        db_version_id,
        &session_queue,
        &http,
    )
@@ -665,14 +672,12 @@ pub async fn upload_file_to_version(
        }
    } else if let Ok((_, project_id)) = &result {
        transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
-            *project_id,
-            None,
-            Some(true),
-        )
-        .await?;
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
+            .await?;
+        search_state
+            .queue
+            .push_version_changes((*project_id).into(), [version_id])
+            .await;
    }

    result.map(|(response, _)| response)
@@ -26,8 +26,7 @@ use crate::models::teams::ProjectPermissions;
 use crate::queue::file_scan::get_files_missing_attribution;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
-use crate::search::{SearchBackend, SearchState};
-use crate::util::error::Context;
+use crate::search::SearchState;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
@@ -853,14 +852,20 @@ pub async fn version_edit_helper(
            transaction.commit().await?;
            database::models::DBVersion::clear_cache(&version_item, &redis)
                .await?;
-            super::projects::clear_project_cache_and_queue_search(
-                &redis,
-                &search_state,
+            database::models::DBProject::clear_cache(
                version_item.inner.project_id,
                None,
                Some(true),
+                &redis,
            )
            .await?;
+            search_state
+                .queue
+                .push_version_changes(
+                    version_item.inner.project_id.into(),
+                    [VersionId::from(version_item.inner.id)],
+                )
+                .await;
            Ok(HttpResponse::NoContent().body(""))
        } else {
            Err(ApiError::CustomAuthentication(
@@ -1129,19 +1134,9 @@ pub async fn version_delete_route(
    pool: web::Data<PgPool>,
    redis: web::Data<RedisPool>,
    session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
    search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
-    version_delete(
-        req,
-        info,
-        pool,
-        redis,
-        session_queue,
-        search_backend,
-        search_state,
-    )
-    .await
+    version_delete(req, info, pool, redis, session_queue, search_state).await
 }

 pub async fn version_delete(
@@ -1150,7 +1145,6 @@ pub async fn version_delete(
    pool: web::Data<PgPool>,
    redis: web::Data<RedisPool>,
    session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
    search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
    let user = get_user_from_headers(
@@ -1246,18 +1240,20 @@ pub async fn version_delete(

    transaction.commit().await?;

-    super::projects::clear_project_cache_and_queue_search(
-        &redis,
-        &search_state,
+    database::models::DBProject::clear_cache(
        version.inner.project_id,
        None,
        Some(true),
+        &redis,
    )
    .await?;
-    search_backend
-        .remove_documents(&[version.inner.id.into()])
-        .await
-        .wrap_internal_err("failed to remove documents")?;
+    search_state
+        .queue
+        .push_version_changes(
+            version.inner.project_id.into(),
+            [VersionId::from(version.inner.id)],
+        )
+        .await;
    if result.is_some() {
        Ok(HttpResponse::NoContent().body(""))
    } else {
@@ -50,14 +50,7 @@ pub enum SearchIndex {
    MinecraftJavaServerPlayersOnline,
 }

-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SearchIndexName {
-    Projects,
-    ProjectsFiltered,
-}
-
 pub struct SearchSort {
-    pub index_name: SearchIndexName,
    pub index: SearchIndex,
 }

@@ -65,16 +58,12 @@ pub fn parse_search_index(
    index: &str,
    new_filters: Option<&str>,
 ) -> Result<SearchSort, ApiError> {
-    let projects_name = SearchIndexName::Projects;
-    let projects_filtered_name = SearchIndexName::ProjectsFiltered;
-
    // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
    let is_server = new_filters
        .is_some_and(|f| f.contains("project_types = minecraft_java_server"));

    Ok(match index {
        "relevance" => SearchSort {
-            index_name: projects_name,
            index: if is_server {
                SearchIndex::MinecraftJavaServerVerifiedPlays2w
            } else {
@@ -82,27 +71,21 @@ pub fn parse_search_index(
            },
        },
        "downloads" => SearchSort {
-            index_name: projects_filtered_name,
            index: SearchIndex::Downloads,
        },
        "follows" => SearchSort {
-            index_name: projects_name,
            index: SearchIndex::Follows,
        },
        "updated" | "date_modified" => SearchSort {
-            index_name: projects_name,
            index: SearchIndex::Updated,
        },
        "newest" | "date_created" => SearchSort {
-            index_name: projects_name,
            index: SearchIndex::Newest,
        },
        "minecraft_java_server.verified_plays_2w" => SearchSort {
-            index_name: projects_name,
            index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
        },
        "minecraft_java_server.ping.data.players_online" => SearchSort {
-            index_name: projects_name,
            index: SearchIndex::MinecraftJavaServerPlayersOnline,
        },
        i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),
@@ -2,7 +2,7 @@ mod common;
 pub mod typesense;

 pub use common::{
-    ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
-    combined_search_filters, parse_search_index, parse_search_request,
+    ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters,
+    parse_search_index, parse_search_request,
 };
 pub use typesense::{Typesense, TypesenseConfig};
@@ -0,0 +1,518 @@
+use std::fmt::{self, Display, Formatter};
+
+use eyre::{Result, eyre};
+
+use crate::search::SearchField;
+use crate::search::filter::{
+    FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
+    FilterPredicate,
+};
+
+const MAX_DNF_CLAUSES: usize = 64;
+const MAX_FILTER_DEPTH: usize = 64;
+const MAX_FILTER_NODES: usize = 1024;
+const MAX_SERIALIZED_FILTER_BYTES: usize = 64 * 1024;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum FilterScope {
+    Project,
+    Version,
+    Mixed,
+}
+
+enum TypesenseFilter<'a> {
+    And(Vec<Self>),
+    Or(Vec<Self>),
+    Predicate {
+        predicate: &'a FilterPredicate,
+        version: bool,
+    },
+    Join {
+        collection: &'a str,
+        filter: Box<Self>,
+    },
+}
+
+pub(super) fn serialize_filter(
+    filter: &FilterExpr,
+    versions_collection: &str,
+) -> Result<String> {
+    let (nodes, depth) = filter_complexity(filter);
+    if nodes > MAX_FILTER_NODES {
+        return Err(eyre!("search filter has too many expressions"));
+    }
+    if depth > MAX_FILTER_DEPTH {
+        return Err(eyre!("search filter is nested too deeply"));
+    }
+
+    let filter = plan(filter, versions_collection)?;
+    let serialized = filter.to_string();
+    if serialized.len() > MAX_SERIALIZED_FILTER_BYTES {
+        return Err(eyre!("search filter is too large"));
+    }
+    Ok(serialized)
+}
+
+fn plan<'a>(
+    filter: &'a FilterExpr,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    match filter_scope(filter) {
+        FilterScope::Project => lower(filter, false),
+        FilterScope::Version => Ok(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(lower(filter, true)?),
+        }),
+        FilterScope::Mixed => plan_mixed(filter, versions_collection),
+    }
+}
+
+fn plan_mixed<'a>(
+    filter: &'a FilterExpr,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    match filter {
+        FilterExpr::Or(expressions) => expressions
+            .iter()
+            .map(|expression| plan(expression, versions_collection))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::Or),
+        FilterExpr::And(expressions)
+            if expressions.iter().all(|expression| {
+                filter_scope(expression) != FilterScope::Mixed
+            }) =>
+        {
+            plan_partitioned_and(expressions, versions_collection)
+        }
+        _ => {
+            let clauses = to_dnf(filter)?;
+            clauses
+                .into_iter()
+                .map(|clause| plan_clause(clause, versions_collection))
+                .collect::<Result<Vec<_>>>()
+                .map(TypesenseFilter::Or)
+        }
+    }
+}
+
+fn plan_partitioned_and<'a>(
+    expressions: &'a [FilterExpr],
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    let mut project = Vec::new();
+    let mut version = Vec::new();
+    for expression in expressions {
+        match filter_scope(expression) {
+            FilterScope::Project => project.push(lower(expression, false)?),
+            FilterScope::Version => version.push(lower(expression, true)?),
+            FilterScope::Mixed => {
+                return Err(eyre!("could not partition mixed search filter"));
+            }
+        }
+    }
+    if let Some(filter) = and_filter(version) {
+        project.push(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(filter),
+        });
+    }
+    and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
+}
+
+fn plan_clause<'a>(
+    predicates: Vec<&'a FilterPredicate>,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    let mut project = Vec::new();
+    let mut version = Vec::new();
+    for predicate in predicates {
+        let planned = lower_predicate(predicate)?;
+        if is_version_filter_field(predicate.field.as_str()) {
+            version.push(planned);
+        } else {
+            project.push(planned);
+        }
+    }
+    if let Some(filter) = and_filter(version) {
+        project.push(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(filter),
+        });
+    }
+    and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
+}
+
+fn lower(filter: &FilterExpr, version: bool) -> Result<TypesenseFilter<'_>> {
+    match filter {
+        FilterExpr::And(expressions) => expressions
+            .iter()
+            .map(|expression| lower(expression, version))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::And),
+        FilterExpr::Or(expressions) => expressions
+            .iter()
+            .map(|expression| lower(expression, version))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::Or),
+        FilterExpr::Predicate(predicate) => {
+            validate_predicate(predicate)?;
+            Ok(TypesenseFilter::Predicate { predicate, version })
+        }
+        FilterExpr::Not(_) => {
+            Err(eyre!("search filter contains an unnormalized negation"))
+        }
+    }
+}
+
+fn lower_predicate(predicate: &FilterPredicate) -> Result<TypesenseFilter<'_>> {
+    validate_predicate(predicate)?;
+    Ok(TypesenseFilter::Predicate {
+        predicate,
+        version: is_version_filter_field(predicate.field.as_str()),
+    })
+}
+
+fn validate_predicate(predicate: &FilterPredicate) -> Result<()> {
+    if matches!(predicate.condition, FilterCondition::Exists { .. }) {
+        return Err(eyre!(
+            "filter field `{}` does not support `EXISTS`",
+            predicate.field.as_str()
+        ));
+    }
+    Ok(())
+}
+
+fn filter_scope(filter: &FilterExpr) -> FilterScope {
+    match filter {
+        FilterExpr::Predicate(predicate) => {
+            if is_version_filter_field(predicate.field.as_str()) {
+                FilterScope::Version
+            } else {
+                FilterScope::Project
+            }
+        }
+        FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
+            let mut scopes = expressions.iter().map(filter_scope);
+            let Some(first) = scopes.next() else {
+                return FilterScope::Project;
+            };
+            if scopes.all(|scope| scope == first) {
+                first
+            } else {
+                FilterScope::Mixed
+            }
+        }
+        FilterExpr::Not(expression) => filter_scope(expression),
+    }
+}
+
+fn is_version_filter_field(field: &str) -> bool {
+    <SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
+        search_field.is_version_field()
+            && search_field.typesense_spec().path == field
+    })
+}
+
+fn to_dnf(filter: &FilterExpr) -> Result<Vec<Vec<&FilterPredicate>>> {
+    match filter {
+        FilterExpr::Predicate(predicate) => Ok(vec![vec![predicate]]),
+        FilterExpr::Or(expressions) => {
+            let mut clauses = Vec::new();
+            for expression in expressions.iter() {
+                clauses.extend(to_dnf(expression)?);
+                if clauses.len() > MAX_DNF_CLAUSES {
+                    return Err(eyre!(
+                        "search filter has too many boolean clauses"
+                    ));
+                }
+            }
+            Ok(clauses)
+        }
+        FilterExpr::And(expressions) => {
+            let mut clauses = vec![Vec::new()];
+            for expression in expressions.iter() {
+                let right = to_dnf(expression)?;
+                if clauses.len().saturating_mul(right.len()) > MAX_DNF_CLAUSES {
+                    return Err(eyre!(
+                        "search filter has too many boolean clauses"
+                    ));
+                }
+                clauses = clauses
+                    .into_iter()
+                    .flat_map(|left| {
+                        right.iter().map(move |right| {
+                            let mut clause = left.clone();
+                            clause.extend(right);
+                            clause
+                        })
+                    })
+                    .collect();
+            }
+            Ok(clauses)
+        }
+        FilterExpr::Not(_) => {
+            Err(eyre!("search filter contains an unnormalized negation"))
+        }
+    }
+}
+
+fn and_filter(
+    mut expressions: Vec<TypesenseFilter<'_>>,
+) -> Option<TypesenseFilter<'_>> {
+    match expressions.len() {
+        0 => None,
+        1 => expressions.pop(),
+        _ => Some(TypesenseFilter::And(expressions)),
+    }
+}
+
+fn filter_complexity(filter: &FilterExpr) -> (usize, usize) {
+    match filter {
+        FilterExpr::Predicate(_) => (1, 1),
+        FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
+            expressions.iter().map(filter_complexity).fold(
+                (1, 1),
+                |(nodes, depth), (child_nodes, child_depth)| {
+                    (nodes + child_nodes, depth.max(child_depth + 1))
+                },
+            )
+        }
+        FilterExpr::Not(expression) => {
+            let (nodes, depth) = filter_complexity(expression);
+            (nodes + 1, depth + 1)
+        }
+    }
+}
+
+impl Display for TypesenseFilter<'_> {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
+        self.fmt_with_precedence(formatter, 0)
+    }
+}
+
+impl TypesenseFilter<'_> {
+    fn precedence(&self) -> u8 {
+        match self {
+            Self::Or(_) => 1,
+            Self::And(_) => 2,
+            Self::Predicate { .. } | Self::Join { .. } => 3,
+        }
+    }
+
+    fn fmt_with_precedence(
+        &self,
+        formatter: &mut Formatter<'_>,
+        parent_precedence: u8,
+    ) -> fmt::Result {
+        let precedence = self.precedence();
+        let parenthesize = precedence < parent_precedence;
+        if parenthesize {
+            formatter.write_str("(")?;
+        }
+
+        match self {
+            Self::And(expressions) => {
+                format_expressions(formatter, expressions, " && ", precedence)?;
+            }
+            Self::Or(expressions) => {
+                format_expressions(formatter, expressions, " || ", precedence)?;
+            }
+            Self::Predicate { predicate, version } => {
+                format_predicate(formatter, predicate, *version)?;
+            }
+            Self::Join { collection, filter } => {
+                write!(formatter, "${collection}(")?;
+                filter.fmt_with_precedence(formatter, 0)?;
+                formatter.write_str(")")?;
+            }
+        }
+
+        if parenthesize {
+            formatter.write_str(")")?;
+        }
+        Ok(())
+    }
+}
+
+fn format_expressions(
+    formatter: &mut Formatter<'_>,
+    expressions: &[TypesenseFilter<'_>],
+    separator: &str,
+    precedence: u8,
+) -> fmt::Result {
+    for (index, expression) in expressions.iter().enumerate() {
+        if index != 0 {
+            formatter.write_str(separator)?;
+        }
+        expression.fmt_with_precedence(formatter, precedence)?;
+    }
+    Ok(())
+}
+
+fn format_predicate(
+    formatter: &mut Formatter<'_>,
+    predicate: &FilterPredicate,
+    version: bool,
+) -> fmt::Result {
+    formatter.write_str(predicate.field.as_str())?;
+    match &predicate.condition {
+        FilterCondition::Compare { comparison, value } => {
+            let operator = match comparison {
+                FilterComparison::Equal if version => ":",
+                FilterComparison::Equal => ":=",
+                FilterComparison::NotEqual => ":!=",
+                FilterComparison::GreaterThan => ":>",
+                FilterComparison::GreaterThanOrEqual => ":>=",
+                FilterComparison::LessThan => ":<",
+                FilterComparison::LessThanOrEqual => ":<=",
+            };
+            formatter.write_str(operator)?;
+            format_literal(formatter, value)
+        }
+        FilterCondition::In { values, negated } => {
+            formatter.write_str(if *negated { ":!=" } else { ":" })?;
+            formatter.write_str("[")?;
+            for (index, value) in values.iter().enumerate() {
+                if index != 0 {
+                    formatter.write_str(",")?;
+                }
+                format_literal(formatter, value)?;
+            }
+            formatter.write_str("]")
+        }
+        FilterCondition::Exists { .. } => unreachable!(
+            "unsupported predicates are rejected before serialization"
+        ),
+    }
+}
+
+fn format_literal(
+    formatter: &mut Formatter<'_>,
+    literal: &FilterLiteral,
+) -> fmt::Result {
+    match literal {
+        FilterLiteral::String(value) => {
+            formatter.write_str("`")?;
+            for character in value.chars() {
+                if character == '`' {
+                    formatter.write_str("\\")?;
+                }
+                write!(formatter, "{character}")?;
+            }
+            formatter.write_str("`")
+        }
+        FilterLiteral::Number(value) => formatter.write_str(value),
+        FilterLiteral::Bool(value) => Display::fmt(value, formatter),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::serialize_filter;
+    use crate::search::filter::{normalize, parse_expression};
+
+    fn serialize(input: &str) -> String {
+        let filter = normalize(parse_expression(input).unwrap());
+        serialize_filter(&filter, "versions").unwrap()
+    }
+
+    #[test]
+    fn project_filters_do_not_join_versions() {
+        assert_eq!(serialize("license = MIT"), "license:=`MIT`");
+    }
+
+    #[test]
+    fn correlated_version_filters_share_one_join() {
+        assert_eq!(
+            serialize("categories = fabric AND game_versions = 1.21"),
+            "$versions(categories:`fabric` && game_versions:1.21)"
+        );
+    }
+
+    #[test]
+    fn mixed_boolean_filters_preserve_version_correlation() {
+        let filter = serialize(
+            "(license = MIT OR categories = fabric) AND game_versions = 1.21",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 2);
+        assert!(filter.contains("categories:`fabric` && game_versions:1.21"));
+        assert!(filter.contains("license:=`MIT`"));
+    }
+
+    #[test]
+    fn string_values_are_escaped() {
+        assert_eq!(
+            serialize(r#"license = "value, with (syntax) and `tick`""#),
+            r#"license:=`value, with (syntax) and \`tick\``"#,
+        );
+    }
+
+    #[test]
+    fn serializes_legacy_unary_not_filters() {
+        assert_eq!(
+            serialize(
+                r#"NOT"project_id"="8xOSkvVU" AND NOT"project_id"="DRol93FL""#
+            ),
+            "project_id:!=`8xOSkvVU` && project_id:!=`DRol93FL`"
+        );
+    }
+
+    #[test]
+    fn cartesian_version_filter_uses_one_join() {
+        let filter = serialize(
+            "(project_types = modpack AND game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.21.1 AND categories = forge AND categories = technology)",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("categories:[`fabric`,`forge`]"));
+        assert!(filter.contains("game_versions:[`1.20.1`,`1.21.1`]"));
+        assert!(filter.contains("categories:`technology`"));
+    }
+
+    #[test]
+    fn factored_mixed_filter_uses_one_version_join() {
+        let filter = serialize(
+            "(license = MIT AND game_versions = 1.20.1 AND categories = fabric) OR \
+             (license = MIT AND game_versions = 1.21.1 AND categories = forge)",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("license:=`MIT`"));
+        assert!(filter.contains(
+            "categories:`fabric` && game_versions:`1.20.1` || categories:`forge` && game_versions:`1.21.1`"
+        ));
+    }
+
+    #[test]
+    fn production_cartesian_filter_uses_one_join() {
+        let versions = [
+            "26.2", "26.1.2", "26.1.1", "26.1", "1.21.11", "1.21.10", "1.21.8",
+            "1.21.7", "1.21.5", "1.21.4", "1.21.3", "1.21.1", "1.21", "1.20.6",
+            "1.20.4", "1.20.2", "1.20.1", "1.20", "1.19.4", "1.19.3", "1.19.2",
+            "1.18.2", "1.17.1", "1.12.2", "1.8.9",
+        ];
+        let input = versions
+            .iter()
+            .flat_map(|version| {
+                ["fabric", "forge"].map(|loader| {
+                    format!(
+                        "(project_types = modpack AND game_versions = {version} AND categories = {loader} AND categories = technology)"
+                    )
+                })
+            })
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let filter = serialize(&input);
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("categories:[`fabric`,`forge`]"));
+        assert!(filter.contains("game_versions:["));
+        assert!(filter.contains("`1.8.9`"));
+        assert!(filter.contains("26.2"));
+        assert!(filter.contains("categories:`technology`"));
+    }
+}
@@ -0,0 +1,96 @@
+use smallvec::SmallVec;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterExpr {
+    And(Box<SmallVec<[Self; 4]>>),
+    Or(Box<SmallVec<[Self; 4]>>),
+    Not(Box<Self>),
+    Predicate(FilterPredicate),
+}
+
+impl FilterExpr {
+    pub fn and(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
+        let mut expressions =
+            expressions.into_iter().collect::<SmallVec<[_; 4]>>();
+        match expressions.len() {
+            0 => None,
+            1 => expressions.pop(),
+            _ => Some(Self::And(Box::new(expressions))),
+        }
+    }
+
+    pub fn or(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
+        let mut expressions =
+            expressions.into_iter().collect::<SmallVec<[_; 4]>>();
+        match expressions.len() {
+            0 => None,
+            1 => expressions.pop(),
+            _ => Some(Self::Or(Box::new(expressions))),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct FilterPredicate {
+    pub field: FilterField,
+    pub condition: FilterCondition,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct FilterField(String);
+
+impl FilterField {
+    pub fn new(field: impl Into<String>) -> Self {
+        Self(field.into())
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterCondition {
+    Compare {
+        comparison: FilterComparison,
+        value: FilterLiteral,
+    },
+    In {
+        values: Vec<FilterLiteral>,
+        negated: bool,
+    },
+    Exists {
+        negated: bool,
+    },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterComparison {
+    Equal,
+    NotEqual,
+    GreaterThan,
+    GreaterThanOrEqual,
+    LessThan,
+    LessThanOrEqual,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterLiteral {
+    String(String),
+    Number(String),
+    Bool(bool),
+}
+
+impl FilterLiteral {
+    pub(super) fn from_bare(value: String) -> Self {
+        if value.eq_ignore_ascii_case("true") {
+            Self::Bool(true)
+        } else if value.eq_ignore_ascii_case("false") {
+            Self::Bool(false)
+        } else if value.parse::<f64>().is_ok() {
+            Self::Number(value)
+        } else {
+            Self::String(value)
+        }
+    }
+}
@@ -0,0 +1,93 @@
+use serde_json::Value;
+use thiserror::Error;
+
+use super::{FilterExpr, FilterParseError, parse_expression};
+
+#[derive(Debug, Error)]
+pub enum LegacyV2FacetsError {
+    #[error("invalid facets JSON")]
+    Json(#[from] serde_json::Error),
+    #[error("facet condition must be a string")]
+    InvalidCondition,
+    #[error(transparent)]
+    Filter(#[from] FilterParseError),
+}
+
+pub fn from_legacy_v2_facets_json(
+    input: &str,
+) -> Result<Option<FilterExpr>, LegacyV2FacetsError> {
+    let facets = serde_json::from_str::<Vec<Vec<Value>>>(input)?;
+    let mut groups = Vec::new();
+
+    for or_group in facets {
+        let mut alternatives = Vec::new();
+        for facet in or_group {
+            let expression = match facet {
+                Value::String(condition) => Some(parse_condition(&condition)?),
+                Value::Array(conditions) => {
+                    let mut predicates = Vec::new();
+                    for condition in conditions {
+                        let condition = condition
+                            .as_str()
+                            .ok_or(LegacyV2FacetsError::InvalidCondition)?;
+                        predicates.push(parse_condition(condition)?);
+                    }
+                    FilterExpr::and(predicates)
+                }
+                _ => return Err(LegacyV2FacetsError::InvalidCondition),
+            };
+            if let Some(expression) = expression {
+                alternatives.push(expression);
+            }
+        }
+        if let Some(expression) = FilterExpr::or(alternatives) {
+            groups.push(expression);
+        }
+    }
+
+    Ok(FilterExpr::and(groups))
+}
+
+fn parse_condition(condition: &str) -> Result<FilterExpr, LegacyV2FacetsError> {
+    if ["!=", ">=", "<=", ">", "<", "="]
+        .iter()
+        .any(|operator| condition.contains(operator))
+    {
+        parse_expression(condition).map_err(Into::into)
+    } else if let Some((field, value)) = condition.split_once(':') {
+        parse_expression(&format!("{} = {}", field.trim(), value.trim()))
+            .map_err(Into::into)
+    } else {
+        parse_expression(condition).map_err(Into::into)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::from_legacy_v2_facets_json;
+    use crate::search::filter::FilterExpr;
+
+    #[test]
+    fn converts_v2_boolean_structure() {
+        let expression = from_legacy_v2_facets_json(
+            r#"[["categories:fabric", "categories:forge"], [["game_versions:1.21", "project_types:mod"]]]"#,
+        )
+        .unwrap()
+        .unwrap();
+
+        let FilterExpr::And(groups) = expression else {
+            panic!("expected outer facets to be joined with AND");
+        };
+        assert!(matches!(groups[0], FilterExpr::Or(_)));
+        assert!(matches!(groups[1], FilterExpr::And(_)));
+    }
+
+    #[test]
+    fn preserves_colons_inside_comparison_values() {
+        from_legacy_v2_facets_json(
+            r#"[["license='https://example.com/license'"]]"#,
+        )
+        .unwrap()
+        .unwrap();
+    }
+}
@@ -0,0 +1,12 @@
+mod ast;
+mod legacy_v2;
+mod normalize;
+mod parse;
+
+pub use ast::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+pub use legacy_v2::from_legacy_v2_facets_json;
+pub use normalize::normalize;
+pub use parse::{FilterParseError, parse_expression};
@@ -0,0 +1,413 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use smallvec::SmallVec;
+
+use super::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+
+pub fn normalize(expression: FilterExpr) -> FilterExpr {
+    match expression {
+        FilterExpr::And(expressions) => normalize_and(*expressions),
+        FilterExpr::Or(expressions) => normalize_or(*expressions),
+        FilterExpr::Not(expression) => normalize_not(*expression),
+        FilterExpr::Predicate(predicate) => normalize_predicate(predicate),
+    }
+}
+
+fn normalize_not(expression: FilterExpr) -> FilterExpr {
+    match expression {
+        FilterExpr::And(expressions) => normalize_or(
+            (*expressions)
+                .into_iter()
+                .map(|expression| FilterExpr::Not(Box::new(expression)))
+                .collect(),
+        ),
+        FilterExpr::Or(expressions) => normalize_and(
+            (*expressions)
+                .into_iter()
+                .map(|expression| FilterExpr::Not(Box::new(expression)))
+                .collect(),
+        ),
+        FilterExpr::Not(expression) => normalize(*expression),
+        FilterExpr::Predicate(mut predicate) => {
+            predicate.condition = match predicate.condition {
+                FilterCondition::Compare { comparison, value } => {
+                    FilterCondition::Compare {
+                        comparison: match comparison {
+                            FilterComparison::Equal => {
+                                FilterComparison::NotEqual
+                            }
+                            FilterComparison::NotEqual => {
+                                FilterComparison::Equal
+                            }
+                            FilterComparison::GreaterThan => {
+                                FilterComparison::LessThanOrEqual
+                            }
+                            FilterComparison::GreaterThanOrEqual => {
+                                FilterComparison::LessThan
+                            }
+                            FilterComparison::LessThan => {
+                                FilterComparison::GreaterThanOrEqual
+                            }
+                            FilterComparison::LessThanOrEqual => {
+                                FilterComparison::GreaterThan
+                            }
+                        },
+                        value,
+                    }
+                }
+                FilterCondition::In { values, negated } => {
+                    FilterCondition::In {
+                        values,
+                        negated: !negated,
+                    }
+                }
+                FilterCondition::Exists { negated } => {
+                    FilterCondition::Exists { negated: !negated }
+                }
+            };
+            normalize_predicate(predicate)
+        }
+    }
+}
+
+fn normalize_predicate(mut predicate: FilterPredicate) -> FilterExpr {
+    if let FilterCondition::In { values, .. } = &mut predicate.condition {
+        values.sort();
+        values.dedup();
+    }
+
+    if predicate.field.as_str() == "minecraft_java_server.ping.data"
+        && let FilterCondition::Exists { negated } = predicate.condition
+    {
+        return FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("minecraft_java_server.is_online"),
+            condition: FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: FilterLiteral::Bool(!negated),
+            },
+        });
+    }
+
+    FilterExpr::Predicate(predicate)
+}
+
+fn normalize_and(expressions: SmallVec<[FilterExpr; 4]>) -> FilterExpr {
+    let mut normalized = expressions.into_iter().map(normalize).fold(
+        SmallVec::<[_; 4]>::new(),
+        |mut flattened, expression| {
+            match expression {
+                FilterExpr::And(children) => flattened.extend(*children),
+                expression => flattened.push(expression),
+            }
+            flattened
+        },
+    );
+    normalized.sort();
+    normalized.dedup();
+
+    FilterExpr::and(normalized).expect("an AND expression is non-empty")
+}
+
+fn normalize_or(expressions: SmallVec<[FilterExpr; 4]>) -> FilterExpr {
+    let mut normalized = expressions.into_iter().map(normalize).fold(
+        SmallVec::<[_; 4]>::new(),
+        |mut flattened, expression| {
+            match expression {
+                FilterExpr::Or(children) => flattened.extend(*children),
+                expression => flattened.push(expression),
+            }
+            flattened
+        },
+    );
+    normalized.sort();
+    normalized.dedup();
+
+    if let Some(expression) = factor_common_predicates(&normalized) {
+        return normalize(expression);
+    }
+
+    if let Some(expression) = compact_cartesian_product(&normalized) {
+        return expression;
+    }
+
+    FilterExpr::or(normalized).expect("an OR expression is non-empty")
+}
+
+fn factor_common_predicates(expressions: &[FilterExpr]) -> Option<FilterExpr> {
+    let clauses = expressions
+        .iter()
+        .map(predicate_clause)
+        .collect::<Option<Vec<_>>>()?;
+    if clauses.len() < 2 {
+        return None;
+    }
+
+    let common = clauses
+        .iter()
+        .skip(1)
+        .fold(clauses[0].clone(), |common, clause| {
+            common.intersection(clause).cloned().collect()
+        });
+    if common.is_empty() {
+        return None;
+    }
+
+    let remaining = clauses
+        .iter()
+        .map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
+        .collect::<Vec<_>>();
+    if remaining.iter().any(Vec::is_empty) {
+        return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
+    }
+
+    let alternatives = remaining.into_iter().map(|clause| {
+        FilterExpr::and(clause.into_iter().map(FilterExpr::Predicate))
+            .expect("a factored OR clause is non-empty")
+    });
+    let alternatives =
+        FilterExpr::or(alternatives).expect("a factored OR has alternatives");
+
+    FilterExpr::and(
+        common
+            .into_iter()
+            .map(FilterExpr::Predicate)
+            .chain([alternatives]),
+    )
+}
+
+fn compact_cartesian_product(expressions: &[FilterExpr]) -> Option<FilterExpr> {
+    let clauses = expressions
+        .iter()
+        .map(predicate_clause)
+        .collect::<Option<Vec<_>>>()?;
+    if clauses.len() < 2 {
+        return None;
+    }
+
+    let common = clauses
+        .iter()
+        .skip(1)
+        .fold(clauses[0].clone(), |common, clause| {
+            common.intersection(clause).cloned().collect()
+        });
+    let remaining = clauses
+        .iter()
+        .map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
+        .collect::<Vec<_>>();
+
+    if remaining.iter().any(Vec::is_empty) {
+        return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
+    }
+
+    let mut values_by_field =
+        BTreeMap::<FilterField, BTreeSet<FilterLiteral>>::new();
+    let expected_fields = remaining[0]
+        .iter()
+        .map(equality_parts)
+        .collect::<Option<Vec<_>>>()?
+        .into_iter()
+        .map(|(field, _)| field.clone())
+        .collect::<BTreeSet<_>>();
+
+    let mut unique_clauses = BTreeSet::new();
+    for clause in &remaining {
+        let parts = clause
+            .iter()
+            .map(equality_parts)
+            .collect::<Option<Vec<_>>>()?;
+        let fields = parts
+            .iter()
+            .map(|(field, _)| (*field).clone())
+            .collect::<BTreeSet<_>>();
+        if fields != expected_fields || fields.len() != parts.len() {
+            return None;
+        }
+        for (field, value) in parts {
+            values_by_field
+                .entry(field.clone())
+                .or_default()
+                .insert(value.clone());
+        }
+        unique_clauses.insert(clause.clone());
+    }
+
+    let combinations = values_by_field
+        .values()
+        .try_fold(1usize, |count, values| count.checked_mul(values.len()))?;
+    if combinations != unique_clauses.len() {
+        return None;
+    }
+
+    let compacted = values_by_field.into_iter().map(|(field, values)| {
+        let values = values.into_iter().collect::<Vec<_>>();
+        let condition = if values.len() == 1 {
+            FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: values.into_iter().next().expect("one value exists"),
+            }
+        } else {
+            FilterCondition::In {
+                values,
+                negated: false,
+            }
+        };
+        FilterPredicate { field, condition }
+    });
+
+    FilterExpr::and(
+        common
+            .into_iter()
+            .chain(compacted)
+            .map(FilterExpr::Predicate),
+    )
+}
+
+fn predicate_clause(
+    expression: &FilterExpr,
+) -> Option<BTreeSet<FilterPredicate>> {
+    match expression {
+        FilterExpr::Predicate(predicate) => {
+            Some(BTreeSet::from([predicate.clone()]))
+        }
+        FilterExpr::And(expressions) => expressions
+            .iter()
+            .map(|expression| match expression {
+                FilterExpr::Predicate(predicate) => Some(predicate.clone()),
+                _ => None,
+            })
+            .collect(),
+        FilterExpr::Or(_) | FilterExpr::Not(_) => None,
+    }
+}
+
+fn equality_parts(
+    predicate: &FilterPredicate,
+) -> Option<(&FilterField, &FilterLiteral)> {
+    match &predicate.condition {
+        FilterCondition::Compare {
+            comparison: FilterComparison::Equal,
+            value,
+        } => Some((&predicate.field, value)),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::normalize;
+    use crate::search::filter::{
+        FilterCondition, FilterExpr, FilterField, FilterLiteral,
+        FilterPredicate, parse_expression,
+    };
+
+    #[test]
+    fn compacts_cartesian_product() {
+        let expression = parse_expression(
+            "(game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
+             (game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
+             (game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
+             (game_versions = 1.21.1 AND categories = forge AND categories = technology)",
+        )
+        .unwrap();
+        let FilterExpr::And(predicates) = normalize(expression) else {
+            panic!("expected a compacted conjunction");
+        };
+
+        assert_eq!(predicates.len(), 3);
+        assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("categories"),
+            condition: FilterCondition::In {
+                values: vec![
+                    FilterLiteral::String("fabric".into()),
+                    FilterLiteral::String("forge".into()),
+                ],
+                negated: false,
+            },
+        })));
+        assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("game_versions"),
+            condition: FilterCondition::In {
+                values: vec![
+                    FilterLiteral::String("1.20.1".into()),
+                    FilterLiteral::String("1.21.1".into()),
+                ],
+                negated: false,
+            },
+        })));
+    }
+
+    #[test]
+    fn factors_common_predicates_before_compacting() {
+        let expression = parse_expression(
+            "(license = MIT AND game_versions = 1.20.1 AND categories = fabric) OR \
+             (license = MIT AND game_versions = 1.21.1 AND categories = forge)",
+        )
+        .unwrap();
+        let FilterExpr::And(expressions) = normalize(expression) else {
+            panic!("expected a factored conjunction");
+        };
+
+        assert!(expressions.iter().any(|expression| matches!(
+            expression,
+            FilterExpr::Predicate(predicate)
+                if predicate.field.as_str() == "license"
+        )));
+        assert!(
+            expressions
+                .iter()
+                .any(|expression| matches!(expression, FilterExpr::Or(_)))
+        );
+    }
+
+    #[test]
+    fn compacts_production_sized_cartesian_product() {
+        let versions = [
+            "26.2", "26.1.2", "26.1.1", "26.1", "1.21.11", "1.21.10", "1.21.8",
+            "1.21.7", "1.21.5", "1.21.4", "1.21.3", "1.21.1", "1.21", "1.20.6",
+            "1.20.4", "1.20.2", "1.20.1", "1.20", "1.19.4", "1.19.3", "1.19.2",
+            "1.18.2", "1.17.1", "1.12.2", "1.8.9",
+        ];
+        let input = versions
+            .iter()
+            .flat_map(|version| {
+                ["fabric", "forge"].map(|loader| {
+                    format!(
+                        "(project_types = modpack AND game_versions = {version} AND categories = {loader} AND categories = technology)"
+                    )
+                })
+            })
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let normalized = normalize(parse_expression(&input).unwrap());
+        let expected = normalize(
+            parse_expression(&format!(
+                "project_types = modpack AND game_versions IN [{}] AND categories IN [fabric, forge] AND categories = technology",
+                versions.join(", ")
+            ))
+            .unwrap(),
+        );
+
+        assert_eq!(normalized, expected);
+    }
+
+    #[test]
+    fn normalizes_unary_not() {
+        let normalized = normalize(
+            parse_expression(
+                r#"NOT"project_id"="8xOSkvVU" AND NOT (downloads > 100 OR open_source = true)"#,
+            )
+            .unwrap(),
+        );
+        let expected = normalize(
+            parse_expression(
+                r#"project_id != "8xOSkvVU" AND downloads <= 100 AND open_source != true"#,
+            )
+            .unwrap(),
+        );
+
+        assert_eq!(normalized, expected);
+    }
+}
@@ -0,0 +1,266 @@
+use std::ops::Range;
+
+use chumsky::{Parser, prelude::*};
+use thiserror::Error;
+
+use super::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+
+#[derive(Debug, Error)]
+#[error("invalid filter at byte {position}: {message}")]
+pub struct FilterParseError {
+    position: usize,
+    message: String,
+}
+
+fn keyword(
+    keyword: &'static str,
+) -> BoxedParser<'static, char, (), Simple<char>> {
+    let keyword =
+        keyword
+            .chars()
+            .fold(empty().ignored().boxed(), |parser, character| {
+                parser
+                    .then_ignore(one_of([
+                        character.to_ascii_lowercase(),
+                        character.to_ascii_uppercase(),
+                    ]))
+                    .boxed()
+            });
+    let boundary = filter(|character: &char| {
+        !character.is_ascii_alphanumeric() && !"_.".contains(*character)
+    })
+    .rewind()
+    .ignored()
+    .or(end());
+
+    keyword.then_ignore(boundary).boxed()
+}
+
+fn quoted_literal(
+    quote: char,
+) -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
+    let escaped = just('\\').ignore_then(any());
+    let character = escaped.or(filter(move |character| {
+        *character != quote && *character != '\\'
+    }));
+
+    character
+        .repeated()
+        .collect::<String>()
+        .delimited_by(just(quote), just(quote))
+        .map(FilterLiteral::String)
+        .boxed()
+}
+
+fn literal_parser() -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
+    let quoted = choice((
+        quoted_literal('\''),
+        quoted_literal('"'),
+        quoted_literal('`'),
+    ));
+    let bare = filter(|character: &char| {
+        !character.is_whitespace() && !",[]()".contains(*character)
+    })
+    .repeated()
+    .at_least(1)
+    .collect::<String>()
+    .map(FilterLiteral::from_bare);
+
+    quoted.or(bare).padded().boxed()
+}
+
+fn field_name_parser() -> BoxedParser<'static, char, String, Simple<char>> {
+    filter(|character: &char| {
+        character.is_ascii_alphabetic() || "_.".contains(*character)
+    })
+    .then(
+        filter(|character: &char| {
+            character.is_ascii_alphanumeric() || "_.".contains(*character)
+        })
+        .repeated(),
+    )
+    .map(|(first, rest)| std::iter::once(first).chain(rest).collect::<String>())
+    .boxed()
+}
+
+fn parser() -> impl Parser<char, FilterExpr, Error = Simple<char>> {
+    let field_name = field_name_parser();
+    let field = choice((
+        field_name.clone(),
+        field_name.clone().delimited_by(just('"'), just('"')),
+        field_name.clone().delimited_by(just('\''), just('\'')),
+        field_name.delimited_by(just('`'), just('`')),
+    ))
+    .map(FilterField::new)
+    .padded();
+
+    let literal = literal_parser();
+    let list = literal
+        .clone()
+        .separated_by(just(',').padded())
+        .at_least(1)
+        .allow_trailing()
+        .delimited_by(just('[').padded(), just(']').padded());
+
+    let comparison = choice((
+        just("!=").to(FilterComparison::NotEqual),
+        just(">=").to(FilterComparison::GreaterThanOrEqual),
+        just("<=").to(FilterComparison::LessThanOrEqual),
+        just('>').to(FilterComparison::GreaterThan),
+        just('<').to(FilterComparison::LessThan),
+        just('=').to(FilterComparison::Equal),
+    ))
+    .padded()
+    .then(literal.clone())
+    .map(|(comparison, value)| FilterCondition::Compare { comparison, value });
+
+    let not_in = keyword("NOT")
+        .padded()
+        .ignore_then(keyword("IN"))
+        .padded()
+        .ignore_then(list.clone())
+        .map(|values| FilterCondition::In {
+            values,
+            negated: true,
+        });
+    let in_list = keyword("IN").padded().ignore_then(list).map(|values| {
+        FilterCondition::In {
+            values,
+            negated: false,
+        }
+    });
+    let not_exists = keyword("NOT")
+        .padded()
+        .ignore_then(keyword("EXISTS"))
+        .map(|()| FilterCondition::Exists { negated: true });
+    let exists =
+        keyword("EXISTS").map(|()| FilterCondition::Exists { negated: false });
+
+    let predicate = field
+        .then(choice((not_in, in_list, not_exists, exists, comparison)))
+        .map(|(field, condition)| {
+            FilterExpr::Predicate(FilterPredicate { field, condition })
+        });
+
+    recursive(|expression| {
+        let atom = predicate
+            .clone()
+            .or(expression.delimited_by(just('(').padded(), just(')').padded()))
+            .padded();
+        let unary = keyword("NOT").padded().repeated().then(atom).map(
+            |(operators, expression)| {
+                operators.into_iter().fold(expression, |expression, ()| {
+                    FilterExpr::Not(Box::new(expression))
+                })
+            },
+        );
+        let and = unary
+            .clone()
+            .then(keyword("AND").padded().ignore_then(unary).repeated())
+            .map(|(first, rest)| {
+                FilterExpr::and(std::iter::once(first).chain(rest))
+                    .expect("an expression always contains one operand")
+            });
+
+        and.clone()
+            .then(keyword("OR").padded().ignore_then(and).repeated())
+            .map(|(first, rest)| {
+                FilterExpr::or(std::iter::once(first).chain(rest))
+                    .expect("an expression always contains one operand")
+            })
+    })
+    .padded()
+    .then_ignore(end())
+}
+
+pub fn parse_expression(input: &str) -> Result<FilterExpr, FilterParseError> {
+    parser().parse(input).map_err(|errors| {
+        let error = errors
+            .into_iter()
+            .next()
+            .unwrap_or_else(|| Simple::custom(0..0, "invalid filter"));
+        let Range { start, .. } = error.span();
+        FilterParseError {
+            position: start,
+            message: error.to_string(),
+        }
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_expression;
+    use crate::search::filter::{
+        FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
+    };
+
+    #[test]
+    fn parses_boolean_precedence() {
+        let expression = parse_expression(
+            "license = MIT OR downloads >= 100 AND open_source = true",
+        )
+        .unwrap();
+
+        let FilterExpr::Or(expressions) = expression else {
+            panic!("expected an OR expression");
+        };
+        assert_eq!(expressions.len(), 2);
+        assert!(matches!(expressions[1], FilterExpr::And(_)));
+    }
+
+    #[test]
+    fn parses_quoted_and_list_values() {
+        let expression = parse_expression(
+            r#"name = "value with spaces" AND categories IN [fabric, "forge"]"#,
+        )
+        .unwrap();
+
+        let FilterExpr::And(expressions) = expression else {
+            panic!("expected an AND expression");
+        };
+        let FilterExpr::Predicate(predicate) = &expressions[0] else {
+            panic!("expected a predicate");
+        };
+        assert!(matches!(
+            &predicate.condition,
+            FilterCondition::Compare {
+                value: FilterLiteral::String(value),
+                ..
+            } if value == "value with spaces"
+        ));
+    }
+
+    #[test]
+    fn parses_unary_not_with_quoted_field() {
+        let expression =
+            parse_expression(r#"NOT"project_id"="8xOSkvVU""#).unwrap();
+
+        let FilterExpr::Not(expression) = expression else {
+            panic!("expected a NOT expression");
+        };
+        let FilterExpr::Predicate(predicate) = *expression else {
+            panic!("expected a predicate");
+        };
+        assert_eq!(predicate.field.as_str(), "project_id");
+        assert!(matches!(
+            predicate.condition,
+            FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: FilterLiteral::String(value),
+            } if value == "8xOSkvVU"
+        ));
+    }
+
+    #[test]
+    fn does_not_parse_not_prefix_in_field_name_as_operator() {
+        let expression = parse_expression("notification = true").unwrap();
+
+        let FilterExpr::Predicate(predicate) = expression else {
+            panic!("expected a predicate");
+        };
+        assert_eq!(predicate.field.as_str(), "notification");
+    }
+}
@@ -1,43 +1,66 @@
 pub mod consume;

-use std::{mem, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    mem,
+    sync::Arc,
+    time::Duration,
+};

 use rdkafka::{producer::FutureRecord, util::Timeout};
 use serde::Serialize;
 use tokio::sync::Mutex;

 use crate::{
-    models::ids::ProjectId,
+    models::ids::{ProjectId, VersionId},
    util::kafka::{KAFKA_OPERATION_INTERVAL, KafkaClientState, KafkaEvent},
 };

 pub const SEARCH_PROJECT_INDEX_QUEUE_TOPIC: &str =
    "public.labrinth.search-project-index-queue.v1";
+const QUEUE_FLUSH_INTERVAL: Duration = Duration::from_secs(10);

 #[derive(Clone)]
 pub struct IncrementalSearchQueue {
-    operations: Arc<Mutex<Vec<SearchIndexOperation>>>,
+    operations: Arc<Mutex<PendingSearchIndexOperations>>,
    kafka_client: actix_web::web::Data<KafkaClientState>,
 }

 impl IncrementalSearchQueue {
    pub fn new(kafka_client: actix_web::web::Data<KafkaClientState>) -> Self {
        Self {
-            operations: Arc::new(Mutex::new(Vec::new())),
+            operations: Arc::new(Mutex::new(
+                PendingSearchIndexOperations::default(),
+            )),
            kafka_client,
        }
    }

-    pub async fn push(&self, project_id: ProjectId) {
+    pub async fn push_project_change(&self, project_id: ProjectId) {
+        self.operations.lock().await.push_project_change(project_id);
+    }
+
+    pub async fn push_version_changes(
+        &self,
+        project_id: ProjectId,
+        version_ids: impl IntoIterator<Item = VersionId>,
+    ) {
        self.operations
            .lock()
            .await
-            .push(SearchIndexOperation { project_id });
+            .push_version_change(project_id, version_ids);
+    }
+
+    pub async fn push_project_removal(&self, project_id: ProjectId) {
+        self.operations
+            .lock()
+            .await
+            .push_project_removal(project_id);
    }

    pub async fn run(self) {
        loop {
-            tokio::time::sleep(KAFKA_OPERATION_INTERVAL).await;
+            tokio::time::sleep(QUEUE_FLUSH_INTERVAL).await;

            if let Err(err) = self.drain().await {
                tracing::error!(
@@ -57,13 +80,11 @@ impl IncrementalSearchQueue {
            return Ok(());
        }

-        let mut operations = operations.into_iter();
+        let mut operations = operations.into_events().into_iter();
        while let Some(operation) = operations.next() {
            let event = KafkaEvent::new(
                SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
-                SearchProjectIndexQueueEventData {
-                    project_id: operation.project_id,
-                },
+                operation.clone(),
            );
            let event_id = event.event_metadata.event_id;
            let key = event_id.to_string();
@@ -79,8 +100,10 @@ impl IncrementalSearchQueue {
                .await
            {
                let mut queued_operations = self.operations.lock().await;
-                queued_operations.push(operation);
-                queued_operations.extend(operations);
+                queued_operations.push_event(operation);
+                for operation in operations {
+                    queued_operations.push_event(operation);
+                }

                return Err(err.into());
            }
@@ -90,12 +113,100 @@ impl IncrementalSearchQueue {
    }
 }

-#[derive(Debug, Clone)]
-pub struct SearchIndexOperation {
-    pub project_id: ProjectId,
+#[derive(Default)]
+struct PendingSearchIndexOperations {
+    changed_project_ids: HashSet<ProjectId>,
+    changed_project_versions: HashMap<ProjectId, HashSet<VersionId>>,
+    removed_project_ids: HashSet<ProjectId>,
 }

-#[derive(Debug, Serialize)]
-pub struct SearchProjectIndexQueueEventData {
-    pub project_id: ProjectId,
+impl PendingSearchIndexOperations {
+    fn is_empty(&self) -> bool {
+        self.changed_project_ids.is_empty()
+            && self.changed_project_versions.is_empty()
+            && self.removed_project_ids.is_empty()
+    }
+
+    fn push_project_change(&mut self, project_id: ProjectId) {
+        if !self.removed_project_ids.contains(&project_id) {
+            self.changed_project_ids.insert(project_id);
+        }
+    }
+
+    fn push_version_change(
+        &mut self,
+        project_id: ProjectId,
+        version_ids: impl IntoIterator<Item = VersionId>,
+    ) {
+        if self.removed_project_ids.contains(&project_id) {
+            return;
+        }
+
+        let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
+        if !version_ids.is_empty() {
+            self.changed_project_versions
+                .entry(project_id)
+                .or_default()
+                .extend(version_ids);
+        }
+    }
+
+    fn push_project_removal(&mut self, project_id: ProjectId) {
+        self.changed_project_ids.remove(&project_id);
+        self.changed_project_versions.remove(&project_id);
+        self.removed_project_ids.insert(project_id);
+    }
+
+    fn push_event(&mut self, event: SearchProjectIndexQueueEventData) {
+        match event {
+            SearchProjectIndexQueueEventData::Change { project_id } => {
+                self.push_project_change(project_id)
+            }
+            SearchProjectIndexQueueEventData::VersionChange {
+                project_id,
+                version_ids,
+            } => self.push_version_change(project_id, version_ids),
+            SearchProjectIndexQueueEventData::Removal { project_id } => {
+                self.push_project_removal(project_id)
+            }
+        }
+    }
+
+    fn into_events(self) -> Vec<SearchProjectIndexQueueEventData> {
+        let mut events = Vec::with_capacity(
+            self.changed_project_ids.len()
+                + self.changed_project_versions.len()
+                + self.removed_project_ids.len(),
+        );
+
+        events.extend(self.removed_project_ids.into_iter().map(|project_id| {
+            SearchProjectIndexQueueEventData::Removal { project_id }
+        }));
+        events.extend(self.changed_project_ids.into_iter().map(|project_id| {
+            SearchProjectIndexQueueEventData::Change { project_id }
+        }));
+        events.extend(self.changed_project_versions.into_iter().map(
+            |(project_id, version_ids)| {
+                SearchProjectIndexQueueEventData::VersionChange {
+                    project_id,
+                    version_ids: version_ids.into_iter().collect(),
+                }
+            },
+        ));
+        events
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum SearchProjectIndexQueueEventData {
+    #[serde(rename = "project_change")]
+    Change { project_id: ProjectId },
+    #[serde(rename = "project_version_change")]
+    VersionChange {
+        project_id: ProjectId,
+        version_ids: Vec<VersionId>,
+    },
+    #[serde(rename = "project_removal")]
+    Removal { project_id: ProjectId },
 }
@@ -1,21 +1,27 @@
 use actix_web::web;
 use eyre::WrapErr;
-use futures::FutureExt;
 use rdkafka::{
    Message,
    consumer::{CommitMode, Consumer, StreamConsumer},
    message::BorrowedMessage,
 };
 use serde::Deserialize;
-use std::collections::HashSet;
+use std::{
+    collections::HashSet,
+    time::{Duration, Instant},
+};
+use tracing::{Instrument, info, info_span};
 use xredis::RedisPool;

 use crate::{
    database::PgPool,
-    models::ids::ProjectId,
+    env::ENV,
+    models::ids::{ProjectId, VersionId},
    search::{
-        SearchBackend, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
-        indexing::index_project_documents,
+        SearchBackend, SearchDocumentBatch, SearchIndexUpdate,
+        UploadSearchProject,
+        incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
+        indexing::{build_project_documents, build_version_change_documents},
    },
    util::kafka::{
        INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL,
@@ -23,8 +29,6 @@ use crate::{
    },
 };

-const BATCH_SIZE: usize = 100;
-
 pub async fn run(
    ro_pool: PgPool,
    redis_pool: RedisPool,
@@ -61,25 +65,60 @@ async fn consume(
    search_backend: &dyn SearchBackend,
    consumer: &StreamConsumer,
 ) -> eyre::Result<()> {
+    // keep buffer capacity (pre-)allocated
+    let mut messages = Vec::with_capacity(1024);
    loop {
-        let mut messages = Vec::with_capacity(BATCH_SIZE);
-        messages.push(
-            consumer
-                .recv()
-                .await
-                .wrap_err("failed to receive Kafka message")?,
+        messages.clear();
+
+        // wait for a first message to come in...
+        let first_message = consumer
+            .recv()
+            .await
+            .wrap_err("failed to receive Kafka message")?;
+        messages.push(first_message);
+
+        let delay = Duration::from_secs(
+            ENV.SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS,
+        );
+        info!(
+            "Received initial Kafka message; waiting {delay:.2?} for more to batch",
        );

-        while messages.len() < BATCH_SIZE {
-            let Some(message) = consumer.recv().now_or_never() else {
-                break;
-            };
-
-            messages.push(message.wrap_err("failed to receive Kafka message")?);
+        // ..then wait a while for more messages to batch up
+        // so that we can process a big batch to reindex.
+        // we stop until either we've reached the max batch size,
+        // or we've waited enough time - whichever is first.
+        //
+        // do a little trick with an `AsyncFnMut` closure
+        // so that we can explicitly specify the return type
+        let mut collect_more_messages = async || -> eyre::Result<()> {
+            while messages.len() < ENV.SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE {
+                let message = consumer
+                    .recv()
+                    .await
+                    .wrap_err("failed to receive Kafka message")?;
+                messages.push(message);
+            }
+            eyre::Ok(())
+        };
+        match tokio::time::timeout(delay, collect_more_messages()).await {
+            Ok(Ok(())) | Err(_) => {}
+            Ok(Err(err)) => {
+                return Err(
+                    err.wrap_err("failed to receive more Kafka messages")
+                );
+            }
        }

-        consume_batch(ro_pool, redis_pool, search_backend, consumer, messages)
-            .await?;
+        info!("Consuming batch of {} messages", messages.len());
+        consume_batch(
+            ro_pool,
+            redis_pool,
+            search_backend,
+            consumer,
+            messages.drain(..),
+        )
+        .await?;
    }
 }

@@ -88,10 +127,14 @@ async fn consume_batch(
    redis_pool: &RedisPool,
    search_backend: &dyn SearchBackend,
    consumer: &StreamConsumer,
-    messages: Vec<BorrowedMessage<'_>>,
+    messages: impl IntoIterator<Item = BorrowedMessage<'_>>,
 ) -> eyre::Result<()> {
-    let mut project_ids = Vec::new();
-    let mut seen_project_ids = HashSet::new();
+    let start = Instant::now();
+
+    let mut project_ids_to_change = HashSet::new();
+    let mut project_ids_with_version_changes = HashSet::new();
+    let mut project_ids_to_remove = HashSet::new();
+    let mut version_ids_to_change = HashSet::new();
    let mut messages_to_commit = Vec::new();

    for message in messages {
@@ -126,25 +169,149 @@ async fn consume_batch(
            }
        };

-        if seen_project_ids.insert(event.project_id) {
-            project_ids.push(event.project_id);
+        let event = match event {
+            SearchProjectIndexQueueEvent::Current(event) => event,
+            SearchProjectIndexQueueEvent::Legacy { project_id } => {
+                SearchProjectIndexQueueEventData::Change { project_id }
+            }
+        };
+
+        match event {
+            SearchProjectIndexQueueEventData::Change { project_id } => {
+                project_ids_to_change.insert(project_id);
+            }
+            SearchProjectIndexQueueEventData::VersionChange {
+                project_id,
+                version_ids,
+            } => {
+                if !version_ids.is_empty() {
+                    project_ids_with_version_changes.insert(project_id);
+                    version_ids_to_change.extend(version_ids);
+                }
+            }
+            SearchProjectIndexQueueEventData::Removal { project_id } => {
+                project_ids_to_remove.insert(project_id);
+            }
        }
        messages_to_commit.push(message);
    }

-    if project_ids.is_empty() {
-        return Ok(());
+    project_ids_to_change
+        .retain(|project_id| !project_ids_to_remove.contains(project_id));
+    project_ids_with_version_changes
+        .retain(|project_id| !project_ids_to_remove.contains(project_id));
+    project_ids_to_change.retain(|project_id| {
+        !project_ids_with_version_changes.contains(project_id)
+    });
+    let project_ids_to_change =
+        project_ids_to_change.into_iter().collect::<Vec<_>>();
+    let project_ids_with_version_changes = project_ids_with_version_changes
+        .into_iter()
+        .collect::<Vec<_>>();
+    let mut project_ids_to_remove =
+        project_ids_to_remove.into_iter().collect::<Vec<_>>();
+    let version_ids_to_change =
+        version_ids_to_change.into_iter().collect::<Vec<_>>();
+
+    info!(
+        kafka.message_count = messages_to_commit.len(),
+        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove",
+        start.elapsed(),
+        project_ids_to_change.len(),
+        project_ids_with_version_changes.len(),
+        version_ids_to_change.len(),
+        project_ids_to_remove.len(),
+    );
+    let start = Instant::now();
+    let mut documents = SearchDocumentBatch::default();
+
+    if !project_ids_with_version_changes.is_empty() {
+        let operation_start = Instant::now();
+        let changed_documents = build_version_change_documents(
+            ro_pool,
+            redis_pool,
+            &project_ids_with_version_changes,
+            &version_ids_to_change,
+        )
+        .instrument(info_span!(
+            "index",
+            batch_size = project_ids_with_version_changes.len(),
+            version_count = version_ids_to_change.len()
+        ))
+        .await
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects and {} versions",
+                project_ids_with_version_changes.len(),
+                version_ids_to_change.len()
+            )
+        })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_with_version_changes,
+            &changed_documents.projects,
+        ));
+        documents.projects.extend(changed_documents.projects);
+        documents.versions.extend(changed_documents.versions);
+        info!(
+            project_count = project_ids_with_version_changes.len(),
+            version_count = version_ids_to_change.len(),
+            "Built changed project versions in {:.2?}",
+            operation_start.elapsed()
+        );
    }

-    tracing::info!(
-        kafka.message_count = messages_to_commit.len(),
-        project_count = project_ids.len(),
-        "Consumed incremental search index event batch"
-    );
-
-    reindex_projects(ro_pool, redis_pool, search_backend, &project_ids)
+    if !project_ids_to_change.is_empty() {
+        let operation_start = Instant::now();
+        info!(
+            project_count = project_ids_to_change.len(),
+            "Building changed projects"
+        );
+        let changed_project_documents = build_project_documents(
+            ro_pool,
+            redis_pool,
+            &project_ids_to_change,
+        )
+        .instrument(info_span!(
+            "index",
+            batch_size = project_ids_to_change.len()
+        ))
        .await
-        .wrap_err("failed to reindex project batch")?;
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects",
+                project_ids_to_change.len()
+            )
+        })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_to_change,
+            &changed_project_documents,
+        ));
+        documents.projects.extend(changed_project_documents);
+        info!(
+            project_count = project_ids_to_change.len(),
+            "Built changed projects in {:.2?}",
+            operation_start.elapsed()
+        );
+    }
+
+    let operation_start = Instant::now();
+    search_backend
+        .apply_update(SearchIndexUpdate {
+            projects: &documents.projects,
+            versions: &documents.versions,
+            removed_projects: &project_ids_to_remove,
+            removed_versions: &version_ids_to_change,
+        })
+        .await
+        .wrap_err("failed to apply search index update")?;
+    info!(
+        project_count = documents.projects.len(),
+        version_count = documents.versions.len(),
+        removed_project_count = project_ids_to_remove.len(),
+        removed_version_count = version_ids_to_change.len(),
+        "Applied search index update in {:.2?}",
+        operation_start.elapsed()
+    );

    for message in messages_to_commit {
        consumer
@@ -152,45 +319,94 @@ async fn consume_batch(
            .wrap_err("failed to commit Kafka message")?;
    }

+    info!(
+        "Changed {} projects and removed {} projects in {:.2?}",
+        project_ids_to_change.len(),
+        project_ids_to_remove.len(),
+        start.elapsed()
+    );
+
    Ok(())
 }

-pub async fn reindex_project(
+pub async fn reindex_project_document(
    ro_pool: &PgPool,
    redis_pool: &RedisPool,
    search_backend: &dyn SearchBackend,
    project_id: ProjectId,
 ) -> eyre::Result<()> {
-    reindex_projects(ro_pool, redis_pool, search_backend, &[project_id]).await
+    reindex_project_documents(
+        ro_pool,
+        redis_pool,
+        search_backend,
+        &[project_id],
+    )
+    .await
 }

-pub async fn reindex_projects(
+pub async fn reindex_project_documents(
    ro_pool: &PgPool,
    redis_pool: &RedisPool,
    search_backend: &dyn SearchBackend,
    project_ids: &[ProjectId],
 ) -> eyre::Result<()> {
-    search_backend.remove_project_documents(project_ids).await?;
-
-    let mut documents = Vec::new();
-    for project_id in project_ids {
-        documents.extend(
-            index_project_documents(ro_pool, redis_pool, *project_id)
-                .await
-                .wrap_err_with(|| {
-                    format!(
-                        "failed to build project {project_id} search documents"
-                    )
-                })?,
-        );
-    }
-
-    search_backend.index_documents(&documents).await?;
+    info!("Creating project documents");
+    let projects = build_project_documents(ro_pool, redis_pool, project_ids)
+        .instrument(info_span!("index", batch_size = project_ids.len()))
+        .await
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects",
+                project_ids.len()
+            )
+        })?;
+    let removed_projects = missing_project_document_ids(project_ids, &projects);
+    search_backend
+        .apply_update(SearchIndexUpdate {
+            projects: &projects,
+            removed_projects: &removed_projects,
+            ..SearchIndexUpdate::default()
+        })
+        .await?;

    Ok(())
 }

+fn missing_project_document_ids(
+    project_ids: &[ProjectId],
+    documents: &[UploadSearchProject],
+) -> Vec<ProjectId> {
+    let built_project_ids = documents
+        .iter()
+        .map(|project| project.project_id.as_str())
+        .collect::<HashSet<_>>();
+
+    project_ids
+        .iter()
+        .copied()
+        .filter(|project_id| {
+            !built_project_ids.contains(project_id.to_string().as_str())
+        })
+        .collect()
+}
+
 #[derive(Debug, Deserialize)]
-struct SearchProjectIndexQueueEvent {
-    project_id: ProjectId,
+#[serde(untagged)]
+enum SearchProjectIndexQueueEvent {
+    Current(SearchProjectIndexQueueEventData),
+    Legacy { project_id: ProjectId },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum SearchProjectIndexQueueEventData {
+    #[serde(rename = "project_change")]
+    Change { project_id: ProjectId },
+    #[serde(rename = "project_version_change")]
+    VersionChange {
+        project_id: ProjectId,
+        version_ids: Vec<VersionId>,
+    },
+    #[serde(rename = "project_removal")]
+    Removal { project_id: ProjectId },
 }
@@ -5,7 +5,7 @@ use futures::TryStreamExt;
 use heck::ToKebabCase;
 use itertools::Itertools;
 use regex::Regex;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 use tracing::{info, warn};

@@ -19,11 +19,14 @@ use crate::database::models::{
    LoaderFieldEnumValueId, LoaderFieldId,
 };
 use crate::models::exp;
-use crate::models::ids::ProjectId;
+use crate::models::ids::{ProjectId, VersionId};
 use crate::models::projects::{DependencyType, from_duplicate_version_fields};
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
-use crate::search::{SearchProjectDependency, UploadSearchProject};
+use crate::search::{
+    SearchDocumentBatch, SearchProjectDependency, UploadSearchProject,
+    UploadSearchVersion,
+};
 use crate::util::error::Context;
 use xredis::RedisPool;

@@ -68,7 +71,7 @@ pub async fn index_local(
    redis: &RedisPool,
    cursor: i64,
    limit: i64,
-) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
+) -> eyre::Result<(SearchDocumentBatch, i64)> {
    info!("Indexing local projects!");

    let searchable_statuses = searchable_statuses();
@@ -111,20 +114,52 @@ pub async fn index_local(

    let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
    let Some(largest) = project_ids.iter().max() else {
-        return Ok((vec![], i64::MAX));
+        return Ok((SearchDocumentBatch::default(), i64::MAX));
    };

-    let uploads = build_search_documents(pool, redis, db_projects).await?;
-    Ok((uploads, *largest))
+    let documents =
+        build_search_documents(pool, redis, db_projects, None).await?;
+    Ok((documents, *largest))
 }

-pub async fn index_project_documents(
+pub async fn build_project_documents(
    pool: &PgPool,
    redis: &RedisPool,
-    project_id: ProjectId,
+    project_ids: &[ProjectId],
 ) -> eyre::Result<Vec<UploadSearchProject>> {
+    let version_ids = HashSet::new();
+    Ok(
+        build_search_document_batch(pool, redis, project_ids, &version_ids)
+            .await?
+            .projects,
+    )
+}
+
+pub async fn build_version_change_documents(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+    version_ids: &[VersionId],
+) -> eyre::Result<SearchDocumentBatch> {
+    let version_ids = version_ids
+        .iter()
+        .copied()
+        .map(DBVersionId::from)
+        .collect::<HashSet<_>>();
+    build_search_document_batch(pool, redis, project_ids, &version_ids).await
+}
+
+async fn build_search_document_batch(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+    version_ids: &HashSet<DBVersionId>,
+) -> eyre::Result<SearchDocumentBatch> {
    let searchable_statuses = searchable_statuses();
-    let project_ids = vec![DBProjectId::from(project_id).0];
+    let project_ids = project_ids
+        .iter()
+        .map(|project_id| DBProjectId::from(*project_id).0)
+        .collect::<Vec<_>>();

    let db_projects = sqlx::query!(
        r#"
@@ -158,14 +193,15 @@ pub async fn index_project_documents(
    .await
    .wrap_err("failed to fetch project")?;

-    build_search_documents(pool, redis, db_projects).await
+    build_search_documents(pool, redis, db_projects, Some(version_ids)).await
 }

 async fn build_search_documents(
    pool: &PgPool,
    redis: &RedisPool,
    db_projects: Vec<PartialProject>,
-) -> eyre::Result<Vec<UploadSearchProject>> {
+    version_ids: Option<&HashSet<DBVersionId>>,
+) -> eyre::Result<SearchDocumentBatch> {
    let searchable_statuses = searchable_statuses();
    let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
    let project_components = db_projects
@@ -281,7 +317,7 @@ async fn build_search_documents(
    .await?;

    info!("Indexing local versions!");
-    let mut versions = index_versions(pool, project_ids.clone()).await?;
+    let mut versions = load_project_versions(pool, project_ids.clone()).await?;

    info!("Indexing local org owners!");

@@ -333,7 +369,7 @@ async fn build_search_documents(
    .await?;

    info!("Getting all loader fields!");
-    let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
+    let loader_field_definitions: Vec<QueryLoaderField> = sqlx::query!(
        "
        SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional
        FROM loader_fields lf
@@ -351,7 +387,8 @@ async fn build_search_documents(
    })
    .try_collect()
    .await?;
-    let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect();
+    let loader_field_definitions: Vec<&QueryLoaderField> =
+        loader_field_definitions.iter().collect();

    info!("Getting all loader field enum values!");

@@ -376,7 +413,8 @@ async fn build_search_documents(
        .await?;

    info!("Indexing loaders, project types!");
-    let mut uploads = Vec::new();
+    let mut project_uploads = Vec::new();
+    let mut version_uploads = Vec::new();

    let total_len = db_projects.len();
    let mut count = 0;
@@ -453,6 +491,9 @@ async fn build_search_documents(
            } else {
                (vec![], vec![])
            };
+        let mut project_categories = categories;
+        project_categories.sort();
+        project_categories.dedup();
        let dependencies = dependencies
            .get(&project.id)
            .map(|x| x.clone())
@@ -475,176 +516,231 @@ async fn build_search_documents(
            .collect::<Vec<_>>();

        if let Some(versions) = versions.remove(&project.id) {
-            // Aggregated project loader fields
+            let Some(latest_version) = versions.iter().max_by(|a, b| {
+                a.date_published
+                    .cmp(&b.date_published)
+                    .then_with(|| a.id.0.cmp(&b.id.0))
+            }) else {
+                continue;
+            };
+
            let project_version_fields = versions
                .iter()
                .flat_map(|x| x.version_fields.clone())
                .collect::<Vec<_>>();
            let aggregated_version_fields = VersionField::from_query_json(
                project_version_fields,
-                &loader_fields,
+                &loader_field_definitions,
                &loader_field_enum_values,
                true,
            );
-            let project_loader_fields =
+            let unvectorized_loader_fields = aggregated_version_fields
+                .iter()
+                .map(|field| {
+                    (field.field_name.clone(), field.value.serialize_internal())
+                })
+                .collect();
+            let mut loader_fields =
                from_duplicate_version_fields(aggregated_version_fields);
+            let project_loader_fields = loader_fields.clone();

-            // aggregated project loaders
-            let project_loaders = versions
+            let mut project_loaders = versions
                .iter()
                .flat_map(|x| x.loaders.clone())
                .collect::<Vec<_>>();
+            project_loaders.sort();
+            project_loaders.dedup();

-            // all valid project types across every version of the project, so that
-            // filters can exclude projects that have *any* version of a given
-            // project type (unlike the version-specific `project_types` field).
-            let mut all_project_types = versions
+            let mut project_types = versions
                .iter()
                .flat_map(|x| x.project_types.clone())
                .collect::<Vec<_>>();
-            all_project_types.sort();
-            all_project_types.dedup();
+            project_types.sort();
+            project_types.dedup();
            exp::compat::correct_project_types(
                &project.components,
-                &mut all_project_types,
+                &mut project_types,
            );

-            for version in versions {
+            let project_id = ProjectId::from(project.id).to_string();
+            version_uploads.extend(versions.iter().filter_map(|version| {
+                if version_ids.is_some_and(|version_ids| {
+                    !version_ids.contains(&version.id)
+                }) {
+                    return None;
+                }
+
                let version_fields = VersionField::from_query_json(
-                    version.version_fields,
-                    &loader_fields,
+                    version.version_fields.clone(),
+                    &loader_field_definitions,
                    &loader_field_enum_values,
                    false,
                );
                let unvectorized_loader_fields = version_fields
                    .iter()
-                    .map(|vf| {
-                        (vf.field_name.clone(), vf.value.serialize_internal())
+                    .map(|field| {
+                        (
+                            field.field_name.clone(),
+                            field.value.serialize_internal(),
+                        )
                    })
                    .collect();
-                let mut loader_fields =
-                    from_duplicate_version_fields(version_fields);
-                let mut project_types = version.project_types;
-
+                let mut fields = from_duplicate_version_fields(version_fields);
+                let mut version_project_types = version.project_types.clone();
                exp::compat::correct_project_types(
                    &project.components,
-                    &mut project_types,
+                    &mut version_project_types,
                );

-                let mut version_loaders = version.loaders;
-
-                // Uses version loaders, not project loaders.
-                let mut categories = categories.clone();
-                categories.append(&mut version_loaders.clone());
-
-                let display_categories = display_categories.clone();
-                categories.append(&mut version_loaders);
-
                // SPECIAL BEHAVIOUR
                // Todo: revisit.
                // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category.
                // These were previously considered the loader, and in v2, the loader is a category for searching.
                // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories.
-                // The loaders are kept in loader_fields as well, so that no information is lost on retrieval.
-                let mrpack_loaders = loader_fields
+                // The loaders are kept in the project document's aggregated loader fields as well, so that no information is lost on retrieval.
+                let mut version_categories = project_categories.clone();
+                version_categories.extend(version.loaders.iter().cloned());
+                let mrpack_loaders = fields
                    .get("mrpack_loaders")
-                    .cloned()
-                    .map(|x| {
-                        x.into_iter()
-                            .filter_map(|x| x.as_str().map(String::from))
-                            .collect::<Vec<_>>()
-                    })
-                    .unwrap_or_default();
-                categories.extend(mrpack_loaders);
-                if loader_fields.contains_key("mrpack_loaders") {
-                    categories.retain(|x| *x != "mrpack");
+                    .into_iter()
+                    .flatten()
+                    .filter_map(|value| value.as_str().map(String::from))
+                    .collect::<Vec<_>>();
+                version_categories.extend(mrpack_loaders);
+                if fields.contains_key("mrpack_loaders") {
+                    version_categories.retain(|category| category != "mrpack");
                }
+                version_categories.sort();
+                version_categories.dedup();

-                // SPECIAL BEHAVIOUR:
-                // For consistency with v2 searching, we manually input the
-                // client_side and server_side fields from the loader fields into
-                // separate loader fields.
-                // 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
                let (_, v2_og_project_type) =
-                    LegacyProject::get_project_type(&project_types);
+                    LegacyProject::get_project_type(&version_project_types);
                let (client_side, server_side) =
                    v2_reroute::convert_v3_side_types_to_v2_side_types(
                        &unvectorized_loader_fields,
                        Some(&v2_og_project_type),
                    );
-
                if let Ok(client_side) = serde_json::to_value(client_side) {
-                    loader_fields
-                        .insert("client_side".to_string(), vec![client_side]);
+                    fields.insert("client_side".to_string(), vec![client_side]);
                }
                if let Ok(server_side) = serde_json::to_value(server_side) {
-                    loader_fields
-                        .insert("server_side".to_string(), vec![server_side]);
+                    fields.insert("server_side".to_string(), vec![server_side]);
                }
-
-                let components = project
-                    .components
-                    .clone()
-                    .into_query(
-                        ProjectId::from(project.id),
-                        &project_query_context,
+                fields.retain(|field, _| {
+                    matches!(
+                        field.as_str(),
+                        "environment"
+                            | "game_versions"
+                            | "client_side"
+                            | "server_side"
                    )
-                    .wrap_err("failed to populate query components")?;
+                });

-                let usp = UploadSearchProject {
-                    version_id: crate::models::ids::VersionId::from(version.id)
-                        .to_string(),
-                    project_id: crate::models::ids::ProjectId::from(project.id)
-                        .to_string(),
-                    name: project.name.clone(),
-                    indexed_name: normalize_for_search(&project.name),
-                    summary: project.summary.clone(),
-                    categories: categories.clone(),
-                    display_categories: display_categories.clone(),
-                    follows: project.follows,
-                    downloads: project.downloads,
-                    log_downloads: (project.downloads.max(1) as f64).ln(),
-                    icon_url: project.icon_url.clone(),
-                    author: username.clone(),
-                    author_id: ariadne::ids::UserId::from(user_id).to_string(),
-                    organization: org_name.clone(),
-                    organization_id: org_id.map(|e| {
-                        crate::models::ids::OrganizationId::from(e).to_string()
-                    }),
-                    indexed_author: normalize_for_search(&username),
-                    date_created: project.approved,
-                    created_timestamp: project.approved.timestamp(),
-                    date_modified: project.updated,
-                    modified_timestamp: project.updated.timestamp(),
+                Some(UploadSearchVersion {
+                    version_id: VersionId::from(version.id).to_string(),
+                    project_id: project_id.clone(),
+                    categories: version_categories,
+                    project_types: version_project_types,
                    version_published_timestamp: version
                        .date_published
                        .timestamp(),
-                    license: license.clone(),
-                    slug: project.slug.clone(),
-                    // TODO
-                    project_types,
-                    all_project_types: all_project_types.clone(),
-                    gallery: gallery.clone(),
-                    featured_gallery: featured_gallery.clone(),
-                    open_source,
-                    color: project.color.map(|x| x as u32),
-                    dependency_project_ids: dependency_project_ids.clone(),
-                    compatible_dependency_project_ids:
-                        compatible_dependency_project_ids.clone(),
-                    dependencies: dependencies.clone(),
-                    loader_fields,
-                    project_loader_fields: project_loader_fields.clone(),
-                    // 'loaders' is aggregate of all versions' loaders
-                    loaders: project_loaders.clone(),
-                    components,
-                };
+                    loader_fields: fields,
+                })
+            }));

-                uploads.push(usp);
+            let mut categories = project_categories.clone();
+            categories.extend(project_loaders.iter().cloned());
+
+            let mrpack_loaders = loader_fields
+                .get("mrpack_loaders")
+                .into_iter()
+                .flatten()
+                .filter_map(|value| value.as_str().map(String::from))
+                .collect::<Vec<_>>();
+            categories.extend(mrpack_loaders);
+            if loader_fields.contains_key("mrpack_loaders") {
+                categories.retain(|category| category != "mrpack");
            }
+            categories.sort();
+            categories.dedup();
+
+            let (_, v2_og_project_type) =
+                LegacyProject::get_project_type(&project_types);
+            let (client_side, server_side) =
+                v2_reroute::convert_v3_side_types_to_v2_side_types(
+                    &unvectorized_loader_fields,
+                    Some(&v2_og_project_type),
+                );
+
+            if let Ok(client_side) = serde_json::to_value(client_side) {
+                loader_fields
+                    .insert("client_side".to_string(), vec![client_side]);
+            }
+            if let Ok(server_side) = serde_json::to_value(server_side) {
+                loader_fields
+                    .insert("server_side".to_string(), vec![server_side]);
+            }
+
+            let components = project
+                .components
+                .clone()
+                .into_query(ProjectId::from(project.id), &project_query_context)
+                .wrap_err("failed to populate query components")?;
+            let indexed_name = normalize_for_search(&project.name);
+
+            project_uploads.push(UploadSearchProject {
+                version_id: crate::models::ids::VersionId::from(
+                    latest_version.id,
+                )
+                .to_string(),
+                project_id,
+                name: project.name,
+                indexed_name,
+                summary: project.summary,
+                categories,
+                project_categories,
+                display_categories,
+                follows: project.follows,
+                downloads: project.downloads,
+                log_downloads: (project.downloads.max(1) as f64).ln(),
+                icon_url: project.icon_url,
+                author: username.clone(),
+                author_id: ariadne::ids::UserId::from(user_id).to_string(),
+                organization: org_name,
+                organization_id: org_id.map(|id| {
+                    crate::models::ids::OrganizationId::from(id).to_string()
+                }),
+                indexed_author: normalize_for_search(&username),
+                date_created: project.approved,
+                created_timestamp: project.approved.timestamp(),
+                date_modified: project.updated,
+                modified_timestamp: project.updated.timestamp(),
+                version_published_timestamp: latest_version
+                    .date_published
+                    .timestamp(),
+                license,
+                slug: project.slug,
+                project_types: project_types.clone(),
+                all_project_types: project_types,
+                gallery,
+                featured_gallery,
+                open_source,
+                color: project.color.map(|x| x as u32),
+                dependency_project_ids,
+                compatible_dependency_project_ids,
+                dependencies,
+                project_loader_fields,
+                loader_fields,
+                loaders: project_loaders,
+                components,
+            });
        }
    }

-    Ok(uploads)
+    Ok(SearchDocumentBatch {
+        projects: project_uploads,
+        versions: version_uploads,
+    })
 }

 struct PartialVersion {
@@ -655,7 +751,7 @@ struct PartialVersion {
    date_published: DateTime<Utc>,
 }

-async fn index_versions(
+async fn load_project_versions(
    pool: &PgPool,
    project_ids: Vec<i64>,
 ) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>> {
@@ -16,6 +16,7 @@ use utoipa::ToSchema;
 use xredis::RedisPool;

 pub mod backend;
+pub mod filter;
 pub mod incremental;
 pub mod indexing;

@@ -105,24 +106,17 @@ pub trait SearchBackend: Send + Sync {
        info: &SearchRequest,
    ) -> Result<SearchResults, ApiError>;

-    async fn index_projects(
+    async fn rebuild_index(
        &self,
        ro_pool: PgPool,
        redis: RedisPool,
    ) -> eyre::Result<()>;

-    async fn index_documents(
+    async fn apply_update(
        &self,
-        documents: &[UploadSearchProject],
+        update: SearchIndexUpdate<'_>,
    ) -> eyre::Result<()>;

-    async fn remove_project_documents(
-        &self,
-        ids: &[ProjectId],
-    ) -> eyre::Result<()>;
-
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
-
    async fn tasks(&self) -> eyre::Result<Value>;

    async fn tasks_cancel(
@@ -235,8 +229,12 @@ impl FromStr for SearchBackendKind {
    }
 }

+/// Nullable fields in Typesense-bound documents should use
+/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
+/// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
+    /// ID of the most recently published version.
    pub version_id: String,
    pub project_id: String,
    //
@@ -246,20 +244,25 @@ pub struct UploadSearchProject {
    pub slug: Option<String>,
    pub author: String,
    pub author_id: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub organization: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub organization_id: Option<String>,
    pub indexed_author: String,
    pub name: String,
    pub indexed_name: String,
    pub summary: String,
    pub categories: Vec<String>,
+    pub project_categories: Vec<String>,
    pub display_categories: Vec<String>,
    pub follows: i32,
    pub downloads: i32,
    pub log_downloads: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub icon_url: Option<String>,
    pub license: String,
    pub gallery: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub featured_gallery: Option<String>,
    /// RFC 3339 formatted creation date of the project
    pub date_created: DateTime<Utc>,
@@ -269,9 +272,10 @@ pub struct UploadSearchProject {
    pub date_modified: DateTime<Utc>,
    /// Unix timestamp of the last major modification
    pub modified_timestamp: i64,
-    /// Unix timestamp of the publication date of the version
+    /// Unix timestamp of the most recently published version.
    pub version_published_timestamp: i64,
    pub open_source: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub color: Option<u32>,
    #[serde(default)]
    pub dependency_project_ids: Vec<String>,
@@ -290,12 +294,45 @@ pub struct UploadSearchProject {
    pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }

+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct UploadSearchVersion {
+    pub version_id: String,
+    pub project_id: String,
+    pub categories: Vec<String>,
+    pub project_types: Vec<String>,
+    pub version_published_timestamp: i64,
+    #[serde(flatten)]
+    pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
+}
+
+#[derive(Debug, Default)]
+pub struct SearchDocumentBatch {
+    pub projects: Vec<UploadSearchProject>,
+    pub versions: Vec<UploadSearchVersion>,
+}
+
+/// A logical search index mutation. Removals are applied before replacements,
+/// so a document may be present in both a removed and replacement field.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct SearchIndexUpdate<'a> {
+    pub projects: &'a [UploadSearchProject],
+    pub versions: &'a [UploadSearchVersion],
+    /// Projects and all of their version documents to remove.
+    pub removed_projects: &'a [ProjectId],
+    pub removed_versions: &'a [VersionId],
+}
+
+/// Nullable fields in Typesense-bound documents should use
+/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
+/// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct SearchProjectDependency {
    pub project_id: String,
    pub dependency_type: DependencyType,
    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub slug: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
    pub icon_url: Option<String>,
 }

@@ -309,6 +346,7 @@ pub struct SearchResults {

 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct ResultSearchProject {
+    /// ID of the most recently published version.
    pub version_id: String,
    pub project_id: String,
    pub project_types: Vec<String>,
@@ -213,18 +213,7 @@ async fn index_swaps() {
                test_env.api.remove_project("alpha", USER_USER_PAT).await;
            assert_status!(&resp, StatusCode::NO_CONTENT);

-            // We should wait for deletions to be indexed
-            let projects = test_env
-                .api
-                .search_deserialized(
-                    None,
-                    Some(json!([["categories:fabric"]])),
-                    USER_USER_PAT,
-                )
-                .await;
-            assert_eq!(projects.total_hits, 0);
-
-            // When we reindex, it should be still gone
+            // When we reindex, the deleted project should be gone
            let resp = test_env.api.reset_search_index().await;
            assert_status!(&resp, StatusCode::NO_CONTENT);

@@ -121,7 +121,7 @@ dunce = { workspace = true }
 [target.'cfg(target_os = "macos")'.dependencies]
 cidre = { workspace = true, features = ["blocks", "nw"] }

-[target.'cfg(windows)'.dependencies]
+[target."cfg(windows)".dependencies]
 windows = { workspace = true, features = ["Networking_Connectivity"] }
 windows-core = { workspace = true }
 winreg = { workspace = true }
@@ -10,6 +10,16 @@ export type VersionEntry = {
 }

 const VERSIONS: VersionEntry[] = [
+	{
+		date: `2026-07-26T01:32:43+00:00`,
+		product: 'web',
+		body: `## Changed
+- Allow crowdin badges to bypass the image proxy.
+- Bio and Username fields in profile settings now show the character limit.
+
+## Fixed
+- Fixed project moderation page banners and info messages not showing for non-staff users.`,
+	},
 	{
 		date: `2026-07-24T18:04:04+00:00`,
 		product: 'app',
@@ -1,6 +0,0 @@
-## Private Use
-
-Under normal circumstances, your project would be rejected due to the issues listed below.  
-However, since your project is not intended for public use, these requirements will be waived and your project will be unlisted. This means it will remain accessible through a direct link without appearing in public search results, allowing you to share it privately.  
-If you're okay with this, or submitted your project to be unlisted already, than no further action is necessary.  
-If you would like to publish your project publicly, please address all moderation concerns before resubmitting this project.
@@ -0,0 +1 @@
+We've recently launched %SHARED_INSTANCES_FLINK% which allow you to share your modpacks directly to other users. This feature may be better suited for your use case than creating a project page!
@@ -0,0 +1,7 @@
+## Private Use
+
+Under normal circumstances, your project would be rejected due to the issues listed below. \
+However, since it looks like this project is primarily intended to be shared privately or amongst a specific community, these requirements will be waived so your project can remain Unlisted.
+
+If you're okay with this, or submitted your project to be unlisted, then no further action is necessary. \
+If you would like to publish your project publicly, please address all moderation concerns before resubmitting this project.
@@ -1,6 +1,7 @@
 ## Private Community

-Under normal circumstances, your server would be rejected due to the issues listed below.  
-However, since your server is intended for a private community, these requirements will be waived and your server project will be unlisted. This means it will remain accessible through a direct link without appearing in public search results, allowing you to share it privately.  
-If you're okay with this, or submitted your server to be unlisted already, than no further action is necessary.  
+Under normal circumstances, your server would be rejected due to the issues listed below. \
+However, since your server is intended for a private community, these requirements will be waived so your project can remain Unlisted.
+
+If you're okay with this, or submitted your server to be unlisted, then no further action is necessary. \
 If you would like public modrinth users to see and connect to your server, please address all moderation concerns before resubmitting.
@@ -9,6 +9,7 @@ import {
 	getBooleanChildState,
 	group,
 	isNodeActive,
+	md,
 	resolveChildren,
 	stage,
 	toggle,
@@ -66,18 +67,35 @@ export default function (
 						}),
 					),

-				//TODO: chyz combine these
 				toggle('private-use', 'Private use')
-					.shown(computed(() => !project.value.minecraft_server))
 					.suggestedStatus('flagged')
-					.message()
-					.priority(Priorities.alerts),
+					.priority(Priorities.alerts)
+					.rawMessage(async (state) => {
+						let msg

-				toggle('private-use-server', 'Private community')
-					.shown(computed(() => !!project.value.minecraft_server))
-					.suggestedStatus('flagged')
-					.message()
-					.priority(Priorities.alerts),
+						if (
+							project.value.minecraft_java_server &&
+							project.value.minecraft_java_server.content?.kind === 'modpack'
+						) {
+							msg =
+								(await md('checklist/messages/status-alerts/private-use/server')(state)) +
+								'\n' +
+								(await md('checklist/messages/status-alerts/private-use/note/shared-instance')(
+									state,
+								))
+						} else {
+							msg = await md('checklist/messages/status-alerts/private-use/project')(state)
+							if (project.value.project_types.includes('modpack')) {
+								msg +=
+									'\n' +
+									(await md('checklist/messages/status-alerts/private-use/note/shared-instance')(
+										state,
+									))
+							}
+						}
+
+						return msg
+					}),

 				toggle('server-use', 'Server use')
 					.shown(
@@ -335,6 +335,8 @@ export function flattenStaticVariables(): Record<string, string> {
 	vars[`NEW_ENVIRONMENTS_LINK`] = `https://modrinth.com/news/article/new-environments`
 	vars[`LEARN_MORE_ABOUT_SERVERS_FLINK`] =
 		`[learn more about server projects from our news feed](https://modrinth.com/news/article/introducing-server-projects/)`
+	vars[`SHARED_INSTANCES_FLINK`] =
+		`[Shared Instances](https://modrinth.com/news/article/shared-instances/)`

 	return vars
 }
@@ -1 +1 @@
-CLAUDE.md
+CLAUDE.md
@@ -115,6 +115,7 @@ export const configuredXss = new FilterXSS({
 					'cdn.serilum.com',
 					'workflow.serilum.com',
 					'modfolio.creeperkatze.dev',
+					'badges.crowdin.net',
 				]

 				const allowedHostnameSuffixes = ['.github.io']
Author	SHA1	Message	Date
Prospector	9b7d690a5c	update 24-48hrs -> within a week to give more realistic estimate	2026-07-26 11:57:49 -07:00
aecsocketandGitHub	158de019a6	fix: search indexing performance and batching (#6521 ) * Add consume batching delay * maybe fix * max batch size * more logging * parallelize remove tasks * delete/upsert project/version messages * prepare * more logging * log number of docs * try more targeted, homogenous version change ops * ensure only necessary fields are serialized into typesense * disable index background task * wip: script changes * don't facet by project id * fix * fix clippy * batch by document and dedup loaders * fix test * wip: projects/versions collections * clean up SearchBackend interface * cleanup pass * cleanup pass 2 * standardise fn names * cleanup pass * fix compile * factor out filter rewriting * query perf * put categories into the project doc * wip: search filter AST * doc comment * more AST normalization * (temp) convert search request errors into internal errors * implement unary NOT * tombi fmt * fix tests * revert request error * try increase stack size	2026-07-26 14:55:00 +00:00
Prospector	54cc1898fa	changelog	2026-07-25 18:32:56 -07:00
Arthur GitHub Prospector	b96a1cfa58	fix: allow crowdin badges to skip image proxy (#6861 ) Add crowdin to allowed hostnames Co-authored-by: Prospector <6166773+Prospector@users.noreply.github.com>	2026-07-26 01:14:21 +00:00
P4ncake GitHub Prospector	59a794ebb1	Minor UX UI tweaks (#6596 ) * feat(ux): Add letter counters to the bio and username the username letter counter only appears after the 30th letter to not encourage people to have long usernames for no reason * feat(ux): Added "Expand" button to bio on mobile * Revert "feat(ux): Added "Expand" button to bio on mobile" This reverts commit `f7e036002a`. * prepr --------- Co-authored-by: Prospector <6166773+Prospector@users.noreply.github.com>	2026-07-26 00:47:14 +00:00
jade GitHub coolbot100s Prospector	b4380aee87	feat(moderation): modpack scanning improvements (#6843 ) * feat(moderation): make modpack scanning improvements * oopsie * feat(moderation): batch size for modpack scan * feat: i18n for modpack scan modal * chore: remove debug line * run intl:extract and prepr --------- Co-authored-by: coolbot100s <76798835+coolbot100s@users.noreply.github.com> Co-authored-by: Prospector <6166773+Prospector@users.noreply.github.com>	2026-07-26 00:14:09 +00:00
François-Xavier TalbotandGitHub	2cf71a53f2	chore(ci): increase rust stack size (#6874 )	2026-07-26 00:11:08 +00:00
coolbotandGitHub	44be49416b	feat: update private use moderation msgs and stage (#6872 )	2026-07-26 00:10:48 +00:00
ProspectorandGitHub	7f07e66e4f	fix: users not seeing moderation page banners and info 🤦 (#6882 )	2026-07-25 17:10:24 -07:00
				`@@ -0,0 +1 @@`
				`We've recently launched %SHARED_INSTANCES_FLINK% which allow you to share your modpacks directly to other users. This feature may be better suited for your use case than creating a project page!`