fix: search indexing performance and batching (#6521)

* Add consume batching delay

* maybe fix

* max batch size

* more logging

* parallelize remove tasks

* delete/upsert project/version messages

* prepare

* more logging

* log number of docs

* try more targeted, homogenous version change ops

* ensure only necessary fields are serialized into typesense

* disable index background task

* wip: script changes

* don't facet by project id

* fix

* fix clippy

* batch by document and dedup loaders

* fix test

* wip: projects/versions collections

* clean up SearchBackend interface

* cleanup pass

* cleanup pass 2

* standardise fn names

* cleanup pass

* fix compile

* factor out filter rewriting

* query perf

* put categories into the project doc

* wip: search filter AST

* doc comment

* more AST normalization

* (temp) convert search request errors into internal errors

* implement unary NOT

* tombi fmt

* fix tests

* revert request error

* try increase stack size

.cargo/config.toml

@@ -1,5 +1,5 @@
 # Windows has stack overflows when calling from Tauri, so we increase the default stack size used by the compiler
-[target.'cfg(windows)']
+[target."cfg(windows)"]
 rustflags = ["-C", "link-args=/STACK:16777220"]
 
 [target.x86_64-pc-windows-msvc]

.github/workflows/turbo-ci.yml

@@ -81,7 +81,7 @@ jobs:
       REDIS_CONNECTION_TYPE: multiplexed
       REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005
       # Avoid stack overflows in tests
-      RUST_MIN_STACK: 33554432
+      RUST_MIN_STACK: 67108864
 
     steps:
       - name: Check out code

Cargo.lock

@@ -5465,6 +5465,7 @@ dependencies = [
  "bytes",
  "censor",
  "chrono",
+ "chumsky",
  "clap 4.5.48",
  "clickhouse",
  "color-eyre",
@@ -5518,6 +5519,7 @@ dependencies = [
  "serde_with",
  "sha1 0.10.6",
  "sha2",
+ "smallvec",
  "spdx",
  "sqlx",
  "sqlx-tracing",

Cargo.toml

@@ -59,6 +59,7 @@ bytes = "1.10.1"
 censor = "0.3.0"
 chardetng = "0.1.17"
 chrono = "0.4.42"
+chumsky = "0.9.3"
 cidre = { version = "0.15.0", default-features = false, features = [
   "macos_15_0"
 ] }
@@ -192,6 +193,7 @@ sha1 = "0.10.6"
 sha1_smol = { version = "1.0.1", features = ["std"] }
 sha2 = "0.10.9"
 shlex = "1.3.0"
+smallvec = "1.15.1"
 spdx = "0.12.0"
 sqlx = { version = "0.8.6", default-features = false }
 sqlx-tracing = { path = "packages/sqlx-tracing" }

apps/app/Cargo.toml

@@ -55,11 +55,11 @@ core-foundation.workspace = true
 core-graphics.workspace = true
 objc2-app-kit = { workspace = true, features = ["NSWindow"] }
 
-[target.'cfg(windows)'.dependencies]
+[target."cfg(windows)".dependencies]
 webview2-com.workspace = true
 windows-core.workspace = true
 
-[target.'cfg(windows)'.dependencies.windows]
+[target."cfg(windows)".dependencies.windows]
 workspace = true
 features = [
   "Win32_Foundation",

apps/labrinth/.env.docker-compose

@@ -26,9 +26,12 @@ ELASTICSEARCH_INDEX_PREFIX=labrinth
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=elastic
 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000
 
 REDIS_MODE=standalone
 REDIS_CONNECTION_TYPE=pooled

apps/labrinth/.env.local

@@ -44,9 +44,12 @@ ELASTICSEARCH_USERNAME=
 ELASTICSEARCH_PASSWORD=
 
 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000
 
 REDIS_MODE=standalone
 REDIS_CONNECTION_TYPE=pooled

apps/labrinth/Cargo.toml

@@ -35,6 +35,7 @@ bitflags = { workspace = true }
 bytes = { workspace = true }
 censor = { workspace = true }
 chrono = { workspace = true, features = ["serde"] }
+chumsky = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 clickhouse = { workspace = true, features = ["time", "uuid"] }
 color-eyre = { workspace = true }
@@ -114,6 +115,7 @@ serde_json = { workspace = true }
 serde_with = { workspace = true }
 sha1 = { workspace = true }
 sha2 = { workspace = true }
+smallvec = { workspace = true }
 spdx = { workspace = true, features = ["text"] }
 sqlx = { workspace = true, features = [
   "chrono",

apps/labrinth/src/background_task.rs

@@ -18,7 +18,7 @@ use crate::util::anrok;
 use actix_web::web;
 use clap::ValueEnum;
 use eyre::WrapErr;
-use tracing::info;
+use tracing::{info, instrument};
 use xredis::RedisPool;
 
 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
@@ -50,6 +50,7 @@ pub enum BackgroundTask {
 
 impl BackgroundTask {
     #[allow(clippy::too_many_arguments)]
+    #[instrument(skip_all, fields(background_task = ?self))]
     pub async fn run(
         self,
         pool: PgPool,
@@ -176,7 +177,7 @@ pub async fn index_search(
     search_backend: web::Data<dyn SearchBackend>,
 ) -> eyre::Result<()> {
     info!("Indexing local database");
-    search_backend.index_projects(ro_pool, redis_pool).await
+    search_backend.rebuild_index(ro_pool, redis_pool).await
 }
 
 pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {

apps/labrinth/src/env.rs

@@ -43,7 +43,7 @@ macro_rules! vars {
                     )]
                     let $field: Option<$ty> = {
                         let mut default = None::<$ty>;
-                        $( default = Some({ $default }.into()); )?
+                        $( default = Some(<$ty>::from({ $default })); )?
 
                         match parse_value::<$ty>(stringify!($field), default) {
                             Ok(value) => Some(value),
@@ -208,9 +208,14 @@ vars! {
     // search
     SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
     SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS: u64 = 5u64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE: usize = 1000usize;
     TYPESENSE_URL: String = "http://localhost:8108";
     TYPESENSE_API_KEY: String = "modrinth";
     TYPESENSE_INDEX_PREFIX: String = "labrinth";
+    TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize;
+    TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize;
+    TYPESENSE_USE_CACHE: bool = true;
 
     // storage
     STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;

apps/labrinth/src/lib.rs

@@ -131,30 +131,6 @@ pub fn app_setup(
     ));
 
     if enable_background_tasks {
-        // The interval in seconds at which the local database is indexed
-        // for searching.  Defaults to 1 hour if unset.
-        let local_index_interval =
-            Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
-        let pool_ref = pool.clone();
-        let redis_pool_ref = redis_pool.clone();
-        let search_backend_ref = search_backend.clone();
-        scheduler.run(local_index_interval, move || {
-            let pool_ref = pool_ref.clone();
-            let redis_pool_ref = redis_pool_ref.clone();
-            let search_backend = search_backend_ref.clone();
-            async move {
-                if let Err(err) = background_task::index_search(
-                    pool_ref,
-                    redis_pool_ref,
-                    search_backend,
-                )
-                .await
-                {
-                    warn!("Failed to index search: {err:?}");
-                }
-            }
-        });
-
         // Changes statuses of scheduled projects/versions
         let pool_ref = pool.clone();
         // TODO: Clear cache when these are run

apps/labrinth/src/queue/server_ping.rs

@@ -178,8 +178,9 @@ impl ServerPingQueue {
                         None,
                         &self.redis,
                     );
-                    let queue_search =
+                    let queue_search = self
-                        self.incremental_search_queue.push(*project_id);
+                        .incremental_search_queue
+                        .push_project_change(*project_id);
 
                     let (clear_cache_result, _) =
                         join(clear_cache, queue_search).await;

apps/labrinth/src/routes/internal/admin.rs

@@ -7,7 +7,7 @@ use crate::queue::analytics::AnalyticsQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::search::SearchBackend;
-use crate::search::incremental::consume::reindex_project;
+use crate::search::incremental::consume::reindex_project_document;
 use crate::util::date::get_current_tenths_of_ms;
 use crate::util::error::Context;
 use crate::util::guards::admin_key_guard;
@@ -330,7 +330,7 @@ pub async fn force_reindex(
 ) -> Result<HttpResponse, ApiError> {
     let redis = redis.get_ref();
     search_backend
-        .index_projects(pool.as_ref().clone(), redis.clone())
+        .rebuild_index(pool.as_ref().clone(), redis.clone())
         .await
         .wrap_internal_err("failed to index projects")?;
     Ok(HttpResponse::NoContent().finish())
@@ -355,7 +355,7 @@ pub async fn force_reindex_project(
     search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
     let (project_id,) = path.into_inner();
-    reindex_project(
+    reindex_project_document(
         pool.as_ref(),
         redis.as_ref(),
         search_backend.as_ref(),

apps/labrinth/src/routes/v2/versions.rs

@@ -10,7 +10,7 @@ use crate::models::projects::{
 use crate::models::v2::projects::LegacyVersion;
 use crate::queue::session::AuthQueue;
 use crate::routes::{v2_reroute, v3};
-use crate::search::{SearchBackend, SearchState};
+use crate::search::SearchState;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
@@ -488,7 +488,6 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
     search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so we don't need to convert the response
@@ -498,7 +497,6 @@ pub async fn version_delete(
         pool,
         redis,
         session_queue,
-        search_backend,
         search_state,
     )
     .await

apps/labrinth/src/routes/v3/projects.rs

@@ -85,7 +85,11 @@ pub async fn clear_project_cache_and_queue_search(
         redis,
     )
     .await?;
-    search_state.queue.push(project_id.into()).await;
+
+    search_state
+        .queue
+        .push_project_change(project_id.into())
+        .await;
 
     Ok(())
 }
@@ -1053,10 +1057,10 @@ pub async fn project_edit_internal(
         edit: Option<Option<E>>,
         mut component: &mut Option<E::Component>,
         perms: ProjectPermissions,
-    ) -> Result<(), ApiError> {
+    ) -> Result<bool, ApiError> {
         let Some(edit) = edit else {
             // component is not specified in the input JSON - leave alone
-            return Ok(());
+            return Ok(false);
         };
 
         if !perms.contains(ProjectPermissions::EDIT_DETAILS) {
@@ -1095,10 +1099,13 @@ pub async fn project_edit_internal(
             }
         }
 
-        Ok(())
+        Ok(true)
     }
 
-    update(
+    let mut reindex_versions = new_project.categories.is_some()
+        || new_project.additional_categories.is_some();
+
+    reindex_versions |= update(
         &mut transaction,
         id,
         new_project.minecraft_server,
@@ -1106,7 +1113,7 @@ pub async fn project_edit_internal(
         perms,
     )
     .await?;
-    update(
+    reindex_versions |= update(
         &mut transaction,
         id,
         new_project.minecraft_java_server,
@@ -1114,7 +1121,7 @@ pub async fn project_edit_internal(
         perms,
     )
     .await?;
-    update(
+    reindex_versions |= update(
         &mut transaction,
         id,
         new_project.minecraft_bedrock_server,
@@ -1167,14 +1174,31 @@ pub async fn project_edit_internal(
 
     transaction.commit().await?;
 
-    clear_project_cache_and_queue_search(
+    if reindex_versions {
-        &redis,
+        db_models::DBProject::clear_cache(
-        &search_state,
+            project_item.inner.id,
-        project_item.inner.id,
+            project_item.inner.slug,
-        project_item.inner.slug,
+            None,
-        None,
+            &redis,
-    )
+        )
-    .await?;
+        .await?;
+        search_state
+            .queue
+            .push_version_changes(
+                project_item.inner.id.into(),
+                project_item.versions.iter().copied().map(VersionId::from),
+            )
+            .await;
+    } else {
+        clear_project_cache_and_queue_search(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+        )
+        .await?;
+    }
 
     // Remove no longer searchable projects from search index
     if let (true, Some(false)) = (
@@ -1182,16 +1206,9 @@ pub async fn project_edit_internal(
         new_project.status.map(|status| status.is_searchable()),
     ) {
         search_state
-            .backend
+            .queue
-            .remove_documents(
+            .push_project_removal(project_item.inner.id.into())
-                &project_item
+            .await;
-                    .versions
-                    .into_iter()
-                    .map(|x| x.into())
-                    .collect::<Vec<_>>(),
-            )
-            .await
-            .wrap_internal_err("failed to remove documents")?;
     }
 
     Ok(HttpResponse::NoContent().body(""))
@@ -1654,7 +1671,7 @@ pub async fn projects_edit(
             };
         }
 
-        bulk_edit_project_categories(
+        let mut reindex_versions = bulk_edit_project_categories(
             &categories,
             &project.categories,
             project.inner.id as db_ids::DBProjectId,
@@ -1669,7 +1686,7 @@ pub async fn projects_edit(
         )
         .await?;
 
-        bulk_edit_project_categories(
+        reindex_versions |= bulk_edit_project_categories(
             &categories,
             &project.additional_categories,
             project.inner.id as db_ids::DBProjectId,
@@ -1728,20 +1745,37 @@ pub async fn projects_edit(
             }
         }
 
-        changed_projects.push((project.inner.id, project.inner.slug));
+        changed_projects.push((
+            project.inner.id,
+            project.inner.slug,
+            project.versions,
+            reindex_versions,
+        ));
     }
 
     transaction.commit().await?;
 
-    for (project_id, slug) in changed_projects {
+    for (project_id, slug, versions, reindex_versions) in changed_projects {
-        clear_project_cache_and_queue_search(
+        if reindex_versions {
-            &redis,
+            db_models::DBProject::clear_cache(project_id, slug, None, &redis)
-            &search_state,
+                .await?;
-            project_id,
+            search_state
-            slug,
+                .queue
-            None,
+                .push_version_changes(
-        )
+                    project_id.into(),
-        .await?;
+                    versions.into_iter().map(VersionId::from),
+                )
+                .await;
+        } else {
+            clear_project_cache_and_queue_search(
+                &redis,
+                &search_state,
+                project_id,
+                slug,
+                None,
+            )
+            .await?;
+        }
     }
 
     Ok(HttpResponse::NoContent().body(""))
@@ -1755,7 +1789,7 @@ pub async fn bulk_edit_project_categories(
     max_num_categories: usize,
     is_additional: bool,
     transaction: &mut PgTransaction<'_>,
-) -> Result<(), ApiError> {
+) -> Result<bool, ApiError> {
     let mut set_categories =
         if let Some(categories) = bulk_changes.categories.clone() {
             categories
@@ -1782,7 +1816,8 @@ pub async fn bulk_edit_project_categories(
         }
     }
 
-    if &set_categories != project_categories {
+    let changed = &set_categories != project_categories;
+    if changed {
         sqlx::query!(
             "
             DELETE FROM mods_categories
@@ -1815,7 +1850,7 @@ pub async fn bulk_edit_project_categories(
         DBModCategory::insert_many(mod_categories, &mut *transaction).await?;
     }
 
-    Ok(())
+    Ok(changed)
 }
 
 #[derive(Serialize, Deserialize)]
@@ -2791,27 +2826,18 @@ pub async fn project_delete_internal(
         .await
         .wrap_internal_err("failed to commit transaction")?;
 
-    search_state
-        .backend
-        .remove_documents(
-            &project
-                .versions
-                .into_iter()
-                .map(|x| x.into())
-                .collect::<Vec<_>>(),
-        )
-        .await
-        .wrap_internal_err("failed to remove project version documents")?;
-
     if result.is_some() {
-        clear_project_cache_and_queue_search(
+        db_models::DBProject::clear_cache(
-            &redis,
-            &search_state,
             project.inner.id,
             project.inner.slug,
             None,
+            &redis,
         )
         .await?;
+        search_state
+            .queue
+            .push_project_removal(project.inner.id.into())
+            .await;
         Ok(())
     } else {
         Err(ApiError::NotFound)

apps/labrinth/src/routes/v3/version_creation.rs

@@ -184,19 +184,20 @@ pub async fn version_create(
         if let Err(e) = rollback_result {
             return Err(e.into());
         }
-    } else if let Ok((_, project_id)) = &result {
+    } else if let Ok((_, project_id, version_id)) = &result {
         transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
-            &redis,
+            .await?;
-            &search_state,
+        search_state
-            *project_id,
+            .queue
-            None,
+            .push_version_changes(
-            Some(true),
+                (*project_id).into(),
-        )
+                [VersionId::from(*version_id)],
-        .await?;
+            )
+            .await;
     }
 
-    result.map(|(response, _)| response)
+    result.map(|(response, _, _)| response)
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -210,7 +211,8 @@ async fn version_create_inner(
     pool: &PgPool,
     session_queue: &AuthQueue,
     http: &reqwest::Client,
-) -> Result<(HttpResponse, models::DBProjectId), CreateError> {
+) -> Result<(HttpResponse, models::DBProjectId, models::DBVersionId), CreateError>
+{
     let mut initial_version_data = None;
     let mut version_builder = None;
     let mut selected_loaders = None;
@@ -568,7 +570,11 @@ async fn version_create_inner(
         }
     }
 
-    Ok((HttpResponse::Ok().json(response), project_id))
+    Ok((
+        HttpResponse::Ok().json(response),
+        project_id,
+        models::DBVersionId::from(version_id),
+    ))
 }
 
 /// Add files to an existing version.
@@ -635,17 +641,18 @@ pub async fn upload_file_to_version(
     let mut transaction = client.begin().await?;
     let mut uploaded_files = Vec::new();
 
-    let version_id = models::DBVersionId::from(url_data.into_inner().0);
+    let version_id = url_data.into_inner().0;
+    let db_version_id = models::DBVersionId::from(version_id);
 
     let result = upload_file_to_version_inner(
         req,
         &mut payload,
-        client,
+        client.clone(),
         &mut transaction,
         redis.clone(),
         &**file_host,
         &mut uploaded_files,
-        version_id,
+        db_version_id,
         &session_queue,
         &http,
     )
@@ -665,14 +672,12 @@ pub async fn upload_file_to_version(
         }
     } else if let Ok((_, project_id)) = &result {
         transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
-            &redis,
+            .await?;
-            &search_state,
+        search_state
-            *project_id,
+            .queue
-            None,
+            .push_version_changes((*project_id).into(), [version_id])
-            Some(true),
+            .await;
-        )
-        .await?;
     }
 
     result.map(|(response, _)| response)

apps/labrinth/src/routes/v3/versions.rs

@@ -26,8 +26,7 @@ use crate::models::teams::ProjectPermissions;
 use crate::queue::file_scan::get_files_missing_attribution;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
-use crate::search::{SearchBackend, SearchState};
+use crate::search::SearchState;
-use crate::util::error::Context;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
@@ -853,14 +852,20 @@ pub async fn version_edit_helper(
             transaction.commit().await?;
             database::models::DBVersion::clear_cache(&version_item, &redis)
                 .await?;
-            super::projects::clear_project_cache_and_queue_search(
+            database::models::DBProject::clear_cache(
-                &redis,
-                &search_state,
                 version_item.inner.project_id,
                 None,
                 Some(true),
+                &redis,
             )
             .await?;
+            search_state
+                .queue
+                .push_version_changes(
+                    version_item.inner.project_id.into(),
+                    [VersionId::from(version_item.inner.id)],
+                )
+                .await;
             Ok(HttpResponse::NoContent().body(""))
         } else {
             Err(ApiError::CustomAuthentication(
@@ -1129,19 +1134,9 @@ pub async fn version_delete_route(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
     search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
-    version_delete(
+    version_delete(req, info, pool, redis, session_queue, search_state).await
-        req,
-        info,
-        pool,
-        redis,
-        session_queue,
-        search_backend,
-        search_state,
-    )
-    .await
 }
 
 pub async fn version_delete(
@@ -1150,7 +1145,6 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
     search_state: web::Data<SearchState>,
 ) -> Result<HttpResponse, ApiError> {
     let user = get_user_from_headers(
@@ -1246,18 +1240,20 @@ pub async fn version_delete(
 
     transaction.commit().await?;
 
-    super::projects::clear_project_cache_and_queue_search(
+    database::models::DBProject::clear_cache(
-        &redis,
-        &search_state,
         version.inner.project_id,
         None,
         Some(true),
+        &redis,
     )
     .await?;
-    search_backend
+    search_state
-        .remove_documents(&[version.inner.id.into()])
+        .queue
-        .await
+        .push_version_changes(
-        .wrap_internal_err("failed to remove documents")?;
+            version.inner.project_id.into(),
+            [VersionId::from(version.inner.id)],
+        )
+        .await;
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))
     } else {

apps/labrinth/src/search/backend/common.rs

@@ -50,14 +50,7 @@ pub enum SearchIndex {
     MinecraftJavaServerPlayersOnline,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SearchIndexName {
-    Projects,
-    ProjectsFiltered,
-}
-
 pub struct SearchSort {
-    pub index_name: SearchIndexName,
     pub index: SearchIndex,
 }
 
@@ -65,16 +58,12 @@ pub fn parse_search_index(
     index: &str,
     new_filters: Option<&str>,
 ) -> Result<SearchSort, ApiError> {
-    let projects_name = SearchIndexName::Projects;
-    let projects_filtered_name = SearchIndexName::ProjectsFiltered;
-
     // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
     let is_server = new_filters
         .is_some_and(|f| f.contains("project_types = minecraft_java_server"));
 
     Ok(match index {
         "relevance" => SearchSort {
-            index_name: projects_name,
             index: if is_server {
                 SearchIndex::MinecraftJavaServerVerifiedPlays2w
             } else {
@@ -82,27 +71,21 @@ pub fn parse_search_index(
             },
         },
         "downloads" => SearchSort {
-            index_name: projects_filtered_name,
             index: SearchIndex::Downloads,
         },
         "follows" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Follows,
         },
         "updated" | "date_modified" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Updated,
         },
         "newest" | "date_created" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Newest,
         },
         "minecraft_java_server.verified_plays_2w" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
         },
         "minecraft_java_server.ping.data.players_online" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerPlayersOnline,
         },
         i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),

apps/labrinth/src/search/backend/mod.rs

@@ -2,7 +2,7 @@ mod common;
 pub mod typesense;
 
 pub use common::{
-    ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
+    ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters,
-    combined_search_filters, parse_search_index, parse_search_request,
+    parse_search_index, parse_search_request,
 };
 pub use typesense::{Typesense, TypesenseConfig};

apps/labrinth/src/search/backend/typesense/filter.rs

@@ -0,0 +1,518 @@
+use std::fmt::{self, Display, Formatter};
+
+use eyre::{Result, eyre};
+
+use crate::search::SearchField;
+use crate::search::filter::{
+    FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
+    FilterPredicate,
+};
+
+const MAX_DNF_CLAUSES: usize = 64;
+const MAX_FILTER_DEPTH: usize = 64;
+const MAX_FILTER_NODES: usize = 1024;
+const MAX_SERIALIZED_FILTER_BYTES: usize = 64 * 1024;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum FilterScope {
+    Project,
+    Version,
+    Mixed,
+}
+
+enum TypesenseFilter<'a> {
+    And(Vec<Self>),
+    Or(Vec<Self>),
+    Predicate {
+        predicate: &'a FilterPredicate,
+        version: bool,
+    },
+    Join {
+        collection: &'a str,
+        filter: Box<Self>,
+    },
+}
+
+pub(super) fn serialize_filter(
+    filter: &FilterExpr,
+    versions_collection: &str,
+) -> Result<String> {
+    let (nodes, depth) = filter_complexity(filter);
+    if nodes > MAX_FILTER_NODES {
+        return Err(eyre!("search filter has too many expressions"));
+    }
+    if depth > MAX_FILTER_DEPTH {
+        return Err(eyre!("search filter is nested too deeply"));
+    }
+
+    let filter = plan(filter, versions_collection)?;
+    let serialized = filter.to_string();
+    if serialized.len() > MAX_SERIALIZED_FILTER_BYTES {
+        return Err(eyre!("search filter is too large"));
+    }
+    Ok(serialized)
+}
+
+fn plan<'a>(
+    filter: &'a FilterExpr,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    match filter_scope(filter) {
+        FilterScope::Project => lower(filter, false),
+        FilterScope::Version => Ok(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(lower(filter, true)?),
+        }),
+        FilterScope::Mixed => plan_mixed(filter, versions_collection),
+    }
+}
+
+fn plan_mixed<'a>(
+    filter: &'a FilterExpr,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    match filter {
+        FilterExpr::Or(expressions) => expressions
+            .iter()
+            .map(|expression| plan(expression, versions_collection))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::Or),
+        FilterExpr::And(expressions)
+            if expressions.iter().all(|expression| {
+                filter_scope(expression) != FilterScope::Mixed
+            }) =>
+        {
+            plan_partitioned_and(expressions, versions_collection)
+        }
+        _ => {
+            let clauses = to_dnf(filter)?;
+            clauses
+                .into_iter()
+                .map(|clause| plan_clause(clause, versions_collection))
+                .collect::<Result<Vec<_>>>()
+                .map(TypesenseFilter::Or)
+        }
+    }
+}
+
+fn plan_partitioned_and<'a>(
+    expressions: &'a [FilterExpr],
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    let mut project = Vec::new();
+    let mut version = Vec::new();
+    for expression in expressions {
+        match filter_scope(expression) {
+            FilterScope::Project => project.push(lower(expression, false)?),
+            FilterScope::Version => version.push(lower(expression, true)?),
+            FilterScope::Mixed => {
+                return Err(eyre!("could not partition mixed search filter"));
+            }
+        }
+    }
+    if let Some(filter) = and_filter(version) {
+        project.push(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(filter),
+        });
+    }
+    and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
+}
+
+fn plan_clause<'a>(
+    predicates: Vec<&'a FilterPredicate>,
+    versions_collection: &'a str,
+) -> Result<TypesenseFilter<'a>> {
+    let mut project = Vec::new();
+    let mut version = Vec::new();
+    for predicate in predicates {
+        let planned = lower_predicate(predicate)?;
+        if is_version_filter_field(predicate.field.as_str()) {
+            version.push(planned);
+        } else {
+            project.push(planned);
+        }
+    }
+    if let Some(filter) = and_filter(version) {
+        project.push(TypesenseFilter::Join {
+            collection: versions_collection,
+            filter: Box::new(filter),
+        });
+    }
+    and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
+}
+
+fn lower(filter: &FilterExpr, version: bool) -> Result<TypesenseFilter<'_>> {
+    match filter {
+        FilterExpr::And(expressions) => expressions
+            .iter()
+            .map(|expression| lower(expression, version))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::And),
+        FilterExpr::Or(expressions) => expressions
+            .iter()
+            .map(|expression| lower(expression, version))
+            .collect::<Result<Vec<_>>>()
+            .map(TypesenseFilter::Or),
+        FilterExpr::Predicate(predicate) => {
+            validate_predicate(predicate)?;
+            Ok(TypesenseFilter::Predicate { predicate, version })
+        }
+        FilterExpr::Not(_) => {
+            Err(eyre!("search filter contains an unnormalized negation"))
+        }
+    }
+}
+
+fn lower_predicate(predicate: &FilterPredicate) -> Result<TypesenseFilter<'_>> {
+    validate_predicate(predicate)?;
+    Ok(TypesenseFilter::Predicate {
+        predicate,
+        version: is_version_filter_field(predicate.field.as_str()),
+    })
+}
+
+fn validate_predicate(predicate: &FilterPredicate) -> Result<()> {
+    if matches!(predicate.condition, FilterCondition::Exists { .. }) {
+        return Err(eyre!(
+            "filter field `{}` does not support `EXISTS`",
+            predicate.field.as_str()
+        ));
+    }
+    Ok(())
+}
+
+fn filter_scope(filter: &FilterExpr) -> FilterScope {
+    match filter {
+        FilterExpr::Predicate(predicate) => {
+            if is_version_filter_field(predicate.field.as_str()) {
+                FilterScope::Version
+            } else {
+                FilterScope::Project
+            }
+        }
+        FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
+            let mut scopes = expressions.iter().map(filter_scope);
+            let Some(first) = scopes.next() else {
+                return FilterScope::Project;
+            };
+            if scopes.all(|scope| scope == first) {
+                first
+            } else {
+                FilterScope::Mixed
+            }
+        }
+        FilterExpr::Not(expression) => filter_scope(expression),
+    }
+}
+
+fn is_version_filter_field(field: &str) -> bool {
+    <SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
+        search_field.is_version_field()
+            && search_field.typesense_spec().path == field
+    })
+}
+
+fn to_dnf(filter: &FilterExpr) -> Result<Vec<Vec<&FilterPredicate>>> {
+    match filter {
+        FilterExpr::Predicate(predicate) => Ok(vec![vec![predicate]]),
+        FilterExpr::Or(expressions) => {
+            let mut clauses = Vec::new();
+            for expression in expressions.iter() {
+                clauses.extend(to_dnf(expression)?);
+                if clauses.len() > MAX_DNF_CLAUSES {
+                    return Err(eyre!(
+                        "search filter has too many boolean clauses"
+                    ));
+                }
+            }
+            Ok(clauses)
+        }
+        FilterExpr::And(expressions) => {
+            let mut clauses = vec![Vec::new()];
+            for expression in expressions.iter() {
+                let right = to_dnf(expression)?;
+                if clauses.len().saturating_mul(right.len()) > MAX_DNF_CLAUSES {
+                    return Err(eyre!(
+                        "search filter has too many boolean clauses"
+                    ));
+                }
+                clauses = clauses
+                    .into_iter()
+                    .flat_map(|left| {
+                        right.iter().map(move |right| {
+                            let mut clause = left.clone();
+                            clause.extend(right);
+                            clause
+                        })
+                    })
+                    .collect();
+            }
+            Ok(clauses)
+        }
+        FilterExpr::Not(_) => {
+            Err(eyre!("search filter contains an unnormalized negation"))
+        }
+    }
+}
+
+fn and_filter(
+    mut expressions: Vec<TypesenseFilter<'_>>,
+) -> Option<TypesenseFilter<'_>> {
+    match expressions.len() {
+        0 => None,
+        1 => expressions.pop(),
+        _ => Some(TypesenseFilter::And(expressions)),
+    }
+}
+
+fn filter_complexity(filter: &FilterExpr) -> (usize, usize) {
+    match filter {
+        FilterExpr::Predicate(_) => (1, 1),
+        FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
+            expressions.iter().map(filter_complexity).fold(
+                (1, 1),
+                |(nodes, depth), (child_nodes, child_depth)| {
+                    (nodes + child_nodes, depth.max(child_depth + 1))
+                },
+            )
+        }
+        FilterExpr::Not(expression) => {
+            let (nodes, depth) = filter_complexity(expression);
+            (nodes + 1, depth + 1)
+        }
+    }
+}
+
+impl Display for TypesenseFilter<'_> {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
+        self.fmt_with_precedence(formatter, 0)
+    }
+}
+
+impl TypesenseFilter<'_> {
+    fn precedence(&self) -> u8 {
+        match self {
+            Self::Or(_) => 1,
+            Self::And(_) => 2,
+            Self::Predicate { .. } | Self::Join { .. } => 3,
+        }
+    }
+
+    fn fmt_with_precedence(
+        &self,
+        formatter: &mut Formatter<'_>,
+        parent_precedence: u8,
+    ) -> fmt::Result {
+        let precedence = self.precedence();
+        let parenthesize = precedence < parent_precedence;
+        if parenthesize {
+            formatter.write_str("(")?;
+        }
+
+        match self {
+            Self::And(expressions) => {
+                format_expressions(formatter, expressions, " && ", precedence)?;
+            }
+            Self::Or(expressions) => {
+                format_expressions(formatter, expressions, " || ", precedence)?;
+            }
+            Self::Predicate { predicate, version } => {
+                format_predicate(formatter, predicate, *version)?;
+            }
+            Self::Join { collection, filter } => {
+                write!(formatter, "${collection}(")?;
+                filter.fmt_with_precedence(formatter, 0)?;
+                formatter.write_str(")")?;
+            }
+        }
+
+        if parenthesize {
+            formatter.write_str(")")?;
+        }
+        Ok(())
+    }
+}
+
+fn format_expressions(
+    formatter: &mut Formatter<'_>,
+    expressions: &[TypesenseFilter<'_>],
+    separator: &str,
+    precedence: u8,
+) -> fmt::Result {
+    for (index, expression) in expressions.iter().enumerate() {
+        if index != 0 {
+            formatter.write_str(separator)?;
+        }
+        expression.fmt_with_precedence(formatter, precedence)?;
+    }
+    Ok(())
+}
+
+fn format_predicate(
+    formatter: &mut Formatter<'_>,
+    predicate: &FilterPredicate,
+    version: bool,
+) -> fmt::Result {
+    formatter.write_str(predicate.field.as_str())?;
+    match &predicate.condition {
+        FilterCondition::Compare { comparison, value } => {
+            let operator = match comparison {
+                FilterComparison::Equal if version => ":",
+                FilterComparison::Equal => ":=",
+                FilterComparison::NotEqual => ":!=",
+                FilterComparison::GreaterThan => ":>",
+                FilterComparison::GreaterThanOrEqual => ":>=",
+                FilterComparison::LessThan => ":<",
+                FilterComparison::LessThanOrEqual => ":<=",
+            };
+            formatter.write_str(operator)?;
+            format_literal(formatter, value)
+        }
+        FilterCondition::In { values, negated } => {
+            formatter.write_str(if *negated { ":!=" } else { ":" })?;
+            formatter.write_str("[")?;
+            for (index, value) in values.iter().enumerate() {
+                if index != 0 {
+                    formatter.write_str(",")?;
+                }
+                format_literal(formatter, value)?;
+            }
+            formatter.write_str("]")
+        }
+        FilterCondition::Exists { .. } => unreachable!(
+            "unsupported predicates are rejected before serialization"
+        ),
+    }
+}
+
+fn format_literal(
+    formatter: &mut Formatter<'_>,
+    literal: &FilterLiteral,
+) -> fmt::Result {
+    match literal {
+        FilterLiteral::String(value) => {
+            formatter.write_str("`")?;
+            for character in value.chars() {
+                if character == '`' {
+                    formatter.write_str("\\")?;
+                }
+                write!(formatter, "{character}")?;
+            }
+            formatter.write_str("`")
+        }
+        FilterLiteral::Number(value) => formatter.write_str(value),
+        FilterLiteral::Bool(value) => Display::fmt(value, formatter),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::serialize_filter;
+    use crate::search::filter::{normalize, parse_expression};
+
+    fn serialize(input: &str) -> String {
+        let filter = normalize(parse_expression(input).unwrap());
+        serialize_filter(&filter, "versions").unwrap()
+    }
+
+    #[test]
+    fn project_filters_do_not_join_versions() {
+        assert_eq!(serialize("license = MIT"), "license:=`MIT`");
+    }
+
+    #[test]
+    fn correlated_version_filters_share_one_join() {
+        assert_eq!(
+            serialize("categories = fabric AND game_versions = 1.21"),
+            "$versions(categories:`fabric` && game_versions:1.21)"
+        );
+    }
+
+    #[test]
+    fn mixed_boolean_filters_preserve_version_correlation() {
+        let filter = serialize(
+            "(license = MIT OR categories = fabric) AND game_versions = 1.21",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 2);
+        assert!(filter.contains("categories:`fabric` && game_versions:1.21"));
+        assert!(filter.contains("license:=`MIT`"));
+    }
+
+    #[test]
+    fn string_values_are_escaped() {
+        assert_eq!(
+            serialize(r#"license = "value, with (syntax) and `tick`""#),
+            r#"license:=`value, with (syntax) and \`tick\``"#,
+        );
+    }
+
+    #[test]
+    fn serializes_legacy_unary_not_filters() {
+        assert_eq!(
+            serialize(
+                r#"NOT"project_id"="8xOSkvVU" AND NOT"project_id"="DRol93FL""#
+            ),
+            "project_id:!=`8xOSkvVU` && project_id:!=`DRol93FL`"
+        );
+    }
+
+    #[test]
+    fn cartesian_version_filter_uses_one_join() {
+        let filter = serialize(
+            "(project_types = modpack AND game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
+             (project_types = modpack AND game_versions = 1.21.1 AND categories = forge AND categories = technology)",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("categories:[`fabric`,`forge`]"));
+        assert!(filter.contains("game_versions:[`1.20.1`,`1.21.1`]"));
+        assert!(filter.contains("categories:`technology`"));
+    }
+
+    #[test]
+    fn factored_mixed_filter_uses_one_version_join() {
+        let filter = serialize(
+            "(license = MIT AND game_versions = 1.20.1 AND categories = fabric) OR \
+             (license = MIT AND game_versions = 1.21.1 AND categories = forge)",
+        );
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("license:=`MIT`"));
+        assert!(filter.contains(
+            "categories:`fabric` && game_versions:`1.20.1` || categories:`forge` && game_versions:`1.21.1`"
+        ));
+    }
+
+    #[test]
+    fn production_cartesian_filter_uses_one_join() {
+        let versions = [
+            "26.2", "26.1.2", "26.1.1", "26.1", "1.21.11", "1.21.10", "1.21.8",
+            "1.21.7", "1.21.5", "1.21.4", "1.21.3", "1.21.1", "1.21", "1.20.6",
+            "1.20.4", "1.20.2", "1.20.1", "1.20", "1.19.4", "1.19.3", "1.19.2",
+            "1.18.2", "1.17.1", "1.12.2", "1.8.9",
+        ];
+        let input = versions
+            .iter()
+            .flat_map(|version| {
+                ["fabric", "forge"].map(|loader| {
+                    format!(
+                        "(project_types = modpack AND game_versions = {version} AND categories = {loader} AND categories = technology)"
+                    )
+                })
+            })
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let filter = serialize(&input);
+
+        assert_eq!(filter.matches("$versions(").count(), 1);
+        assert!(filter.contains("categories:[`fabric`,`forge`]"));
+        assert!(filter.contains("game_versions:["));
+        assert!(filter.contains("`1.8.9`"));
+        assert!(filter.contains("26.2"));
+        assert!(filter.contains("categories:`technology`"));
+    }
+}

apps/labrinth/src/search/filter/ast.rs

@@ -0,0 +1,96 @@
+use smallvec::SmallVec;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterExpr {
+    And(Box<SmallVec<[Self; 4]>>),
+    Or(Box<SmallVec<[Self; 4]>>),
+    Not(Box<Self>),
+    Predicate(FilterPredicate),
+}
+
+impl FilterExpr {
+    pub fn and(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
+        let mut expressions =
+            expressions.into_iter().collect::<SmallVec<[_; 4]>>();
+        match expressions.len() {
+            0 => None,
+            1 => expressions.pop(),
+            _ => Some(Self::And(Box::new(expressions))),
+        }
+    }
+
+    pub fn or(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
+        let mut expressions =
+            expressions.into_iter().collect::<SmallVec<[_; 4]>>();
+        match expressions.len() {
+            0 => None,
+            1 => expressions.pop(),
+            _ => Some(Self::Or(Box::new(expressions))),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct FilterPredicate {
+    pub field: FilterField,
+    pub condition: FilterCondition,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct FilterField(String);
+
+impl FilterField {
+    pub fn new(field: impl Into<String>) -> Self {
+        Self(field.into())
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterCondition {
+    Compare {
+        comparison: FilterComparison,
+        value: FilterLiteral,
+    },
+    In {
+        values: Vec<FilterLiteral>,
+        negated: bool,
+    },
+    Exists {
+        negated: bool,
+    },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterComparison {
+    Equal,
+    NotEqual,
+    GreaterThan,
+    GreaterThanOrEqual,
+    LessThan,
+    LessThanOrEqual,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum FilterLiteral {
+    String(String),
+    Number(String),
+    Bool(bool),
+}
+
+impl FilterLiteral {
+    pub(super) fn from_bare(value: String) -> Self {
+        if value.eq_ignore_ascii_case("true") {
+            Self::Bool(true)
+        } else if value.eq_ignore_ascii_case("false") {
+            Self::Bool(false)
+        } else if value.parse::<f64>().is_ok() {
+            Self::Number(value)
+        } else {
+            Self::String(value)
+        }
+    }
+}

apps/labrinth/src/search/filter/legacy_v2.rs

@@ -0,0 +1,93 @@
+use serde_json::Value;
+use thiserror::Error;
+
+use super::{FilterExpr, FilterParseError, parse_expression};
+
+#[derive(Debug, Error)]
+pub enum LegacyV2FacetsError {
+    #[error("invalid facets JSON")]
+    Json(#[from] serde_json::Error),
+    #[error("facet condition must be a string")]
+    InvalidCondition,
+    #[error(transparent)]
+    Filter(#[from] FilterParseError),
+}
+
+pub fn from_legacy_v2_facets_json(
+    input: &str,
+) -> Result<Option<FilterExpr>, LegacyV2FacetsError> {
+    let facets = serde_json::from_str::<Vec<Vec<Value>>>(input)?;
+    let mut groups = Vec::new();
+
+    for or_group in facets {
+        let mut alternatives = Vec::new();
+        for facet in or_group {
+            let expression = match facet {
+                Value::String(condition) => Some(parse_condition(&condition)?),
+                Value::Array(conditions) => {
+                    let mut predicates = Vec::new();
+                    for condition in conditions {
+                        let condition = condition
+                            .as_str()
+                            .ok_or(LegacyV2FacetsError::InvalidCondition)?;
+                        predicates.push(parse_condition(condition)?);
+                    }
+                    FilterExpr::and(predicates)
+                }
+                _ => return Err(LegacyV2FacetsError::InvalidCondition),
+            };
+            if let Some(expression) = expression {
+                alternatives.push(expression);
+            }
+        }
+        if let Some(expression) = FilterExpr::or(alternatives) {
+            groups.push(expression);
+        }
+    }
+
+    Ok(FilterExpr::and(groups))
+}
+
+fn parse_condition(condition: &str) -> Result<FilterExpr, LegacyV2FacetsError> {
+    if ["!=", ">=", "<=", ">", "<", "="]
+        .iter()
+        .any(|operator| condition.contains(operator))
+    {
+        parse_expression(condition).map_err(Into::into)
+    } else if let Some((field, value)) = condition.split_once(':') {
+        parse_expression(&format!("{} = {}", field.trim(), value.trim()))
+            .map_err(Into::into)
+    } else {
+        parse_expression(condition).map_err(Into::into)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::from_legacy_v2_facets_json;
+    use crate::search::filter::FilterExpr;
+
+    #[test]
+    fn converts_v2_boolean_structure() {
+        let expression = from_legacy_v2_facets_json(
+            r#"[["categories:fabric", "categories:forge"], [["game_versions:1.21", "project_types:mod"]]]"#,
+        )
+        .unwrap()
+        .unwrap();
+
+        let FilterExpr::And(groups) = expression else {
+            panic!("expected outer facets to be joined with AND");
+        };
+        assert!(matches!(groups[0], FilterExpr::Or(_)));
+        assert!(matches!(groups[1], FilterExpr::And(_)));
+    }
+
+    #[test]
+    fn preserves_colons_inside_comparison_values() {
+        from_legacy_v2_facets_json(
+            r#"[["license='https://example.com/license'"]]"#,
+        )
+        .unwrap()
+        .unwrap();
+    }
+}

apps/labrinth/src/search/filter/mod.rs

@@ -0,0 +1,12 @@
+mod ast;
+mod legacy_v2;
+mod normalize;
+mod parse;
+
+pub use ast::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+pub use legacy_v2::from_legacy_v2_facets_json;
+pub use normalize::normalize;
+pub use parse::{FilterParseError, parse_expression};

apps/labrinth/src/search/filter/normalize.rs

@@ -0,0 +1,413 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use smallvec::SmallVec;
+
+use super::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+
+pub fn normalize(expression: FilterExpr) -> FilterExpr {
+    match expression {
+        FilterExpr::And(expressions) => normalize_and(*expressions),
+        FilterExpr::Or(expressions) => normalize_or(*expressions),
+        FilterExpr::Not(expression) => normalize_not(*expression),
+        FilterExpr::Predicate(predicate) => normalize_predicate(predicate),
+    }
+}
+
+fn normalize_not(expression: FilterExpr) -> FilterExpr {
+    match expression {
+        FilterExpr::And(expressions) => normalize_or(
+            (*expressions)
+                .into_iter()
+                .map(|expression| FilterExpr::Not(Box::new(expression)))
+                .collect(),
+        ),
+        FilterExpr::Or(expressions) => normalize_and(
+            (*expressions)
+                .into_iter()
+                .map(|expression| FilterExpr::Not(Box::new(expression)))
+                .collect(),
+        ),
+        FilterExpr::Not(expression) => normalize(*expression),
+        FilterExpr::Predicate(mut predicate) => {
+            predicate.condition = match predicate.condition {
+                FilterCondition::Compare { comparison, value } => {
+                    FilterCondition::Compare {
+                        comparison: match comparison {
+                            FilterComparison::Equal => {
+                                FilterComparison::NotEqual
+                            }
+                            FilterComparison::NotEqual => {
+                                FilterComparison::Equal
+                            }
+                            FilterComparison::GreaterThan => {
+                                FilterComparison::LessThanOrEqual
+                            }
+                            FilterComparison::GreaterThanOrEqual => {
+                                FilterComparison::LessThan
+                            }
+                            FilterComparison::LessThan => {
+                                FilterComparison::GreaterThanOrEqual
+                            }
+                            FilterComparison::LessThanOrEqual => {
+                                FilterComparison::GreaterThan
+                            }
+                        },
+                        value,
+                    }
+                }
+                FilterCondition::In { values, negated } => {
+                    FilterCondition::In {
+                        values,
+                        negated: !negated,
+                    }
+                }
+                FilterCondition::Exists { negated } => {
+                    FilterCondition::Exists { negated: !negated }
+                }
+            };
+            normalize_predicate(predicate)
+        }
+    }
+}
+
+fn normalize_predicate(mut predicate: FilterPredicate) -> FilterExpr {
+    if let FilterCondition::In { values, .. } = &mut predicate.condition {
+        values.sort();
+        values.dedup();
+    }
+
+    if predicate.field.as_str() == "minecraft_java_server.ping.data"
+        && let FilterCondition::Exists { negated } = predicate.condition
+    {
+        return FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("minecraft_java_server.is_online"),
+            condition: FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: FilterLiteral::Bool(!negated),
+            },
+        });
+    }
+
+    FilterExpr::Predicate(predicate)
+}
+
+fn normalize_and(expressions: SmallVec<[FilterExpr; 4]>) -> FilterExpr {
+    let mut normalized = expressions.into_iter().map(normalize).fold(
+        SmallVec::<[_; 4]>::new(),
+        |mut flattened, expression| {
+            match expression {
+                FilterExpr::And(children) => flattened.extend(*children),
+                expression => flattened.push(expression),
+            }
+            flattened
+        },
+    );
+    normalized.sort();
+    normalized.dedup();
+
+    FilterExpr::and(normalized).expect("an AND expression is non-empty")
+}
+
+fn normalize_or(expressions: SmallVec<[FilterExpr; 4]>) -> FilterExpr {
+    let mut normalized = expressions.into_iter().map(normalize).fold(
+        SmallVec::<[_; 4]>::new(),
+        |mut flattened, expression| {
+            match expression {
+                FilterExpr::Or(children) => flattened.extend(*children),
+                expression => flattened.push(expression),
+            }
+            flattened
+        },
+    );
+    normalized.sort();
+    normalized.dedup();
+
+    if let Some(expression) = factor_common_predicates(&normalized) {
+        return normalize(expression);
+    }
+
+    if let Some(expression) = compact_cartesian_product(&normalized) {
+        return expression;
+    }
+
+    FilterExpr::or(normalized).expect("an OR expression is non-empty")
+}
+
+fn factor_common_predicates(expressions: &[FilterExpr]) -> Option<FilterExpr> {
+    let clauses = expressions
+        .iter()
+        .map(predicate_clause)
+        .collect::<Option<Vec<_>>>()?;
+    if clauses.len() < 2 {
+        return None;
+    }
+
+    let common = clauses
+        .iter()
+        .skip(1)
+        .fold(clauses[0].clone(), |common, clause| {
+            common.intersection(clause).cloned().collect()
+        });
+    if common.is_empty() {
+        return None;
+    }
+
+    let remaining = clauses
+        .iter()
+        .map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
+        .collect::<Vec<_>>();
+    if remaining.iter().any(Vec::is_empty) {
+        return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
+    }
+
+    let alternatives = remaining.into_iter().map(|clause| {
+        FilterExpr::and(clause.into_iter().map(FilterExpr::Predicate))
+            .expect("a factored OR clause is non-empty")
+    });
+    let alternatives =
+        FilterExpr::or(alternatives).expect("a factored OR has alternatives");
+
+    FilterExpr::and(
+        common
+            .into_iter()
+            .map(FilterExpr::Predicate)
+            .chain([alternatives]),
+    )
+}
+
+fn compact_cartesian_product(expressions: &[FilterExpr]) -> Option<FilterExpr> {
+    let clauses = expressions
+        .iter()
+        .map(predicate_clause)
+        .collect::<Option<Vec<_>>>()?;
+    if clauses.len() < 2 {
+        return None;
+    }
+
+    let common = clauses
+        .iter()
+        .skip(1)
+        .fold(clauses[0].clone(), |common, clause| {
+            common.intersection(clause).cloned().collect()
+        });
+    let remaining = clauses
+        .iter()
+        .map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
+        .collect::<Vec<_>>();
+
+    if remaining.iter().any(Vec::is_empty) {
+        return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
+    }
+
+    let mut values_by_field =
+        BTreeMap::<FilterField, BTreeSet<FilterLiteral>>::new();
+    let expected_fields = remaining[0]
+        .iter()
+        .map(equality_parts)
+        .collect::<Option<Vec<_>>>()?
+        .into_iter()
+        .map(|(field, _)| field.clone())
+        .collect::<BTreeSet<_>>();
+
+    let mut unique_clauses = BTreeSet::new();
+    for clause in &remaining {
+        let parts = clause
+            .iter()
+            .map(equality_parts)
+            .collect::<Option<Vec<_>>>()?;
+        let fields = parts
+            .iter()
+            .map(|(field, _)| (*field).clone())
+            .collect::<BTreeSet<_>>();
+        if fields != expected_fields || fields.len() != parts.len() {
+            return None;
+        }
+        for (field, value) in parts {
+            values_by_field
+                .entry(field.clone())
+                .or_default()
+                .insert(value.clone());
+        }
+        unique_clauses.insert(clause.clone());
+    }
+
+    let combinations = values_by_field
+        .values()
+        .try_fold(1usize, |count, values| count.checked_mul(values.len()))?;
+    if combinations != unique_clauses.len() {
+        return None;
+    }
+
+    let compacted = values_by_field.into_iter().map(|(field, values)| {
+        let values = values.into_iter().collect::<Vec<_>>();
+        let condition = if values.len() == 1 {
+            FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: values.into_iter().next().expect("one value exists"),
+            }
+        } else {
+            FilterCondition::In {
+                values,
+                negated: false,
+            }
+        };
+        FilterPredicate { field, condition }
+    });
+
+    FilterExpr::and(
+        common
+            .into_iter()
+            .chain(compacted)
+            .map(FilterExpr::Predicate),
+    )
+}
+
+fn predicate_clause(
+    expression: &FilterExpr,
+) -> Option<BTreeSet<FilterPredicate>> {
+    match expression {
+        FilterExpr::Predicate(predicate) => {
+            Some(BTreeSet::from([predicate.clone()]))
+        }
+        FilterExpr::And(expressions) => expressions
+            .iter()
+            .map(|expression| match expression {
+                FilterExpr::Predicate(predicate) => Some(predicate.clone()),
+                _ => None,
+            })
+            .collect(),
+        FilterExpr::Or(_) | FilterExpr::Not(_) => None,
+    }
+}
+
+fn equality_parts(
+    predicate: &FilterPredicate,
+) -> Option<(&FilterField, &FilterLiteral)> {
+    match &predicate.condition {
+        FilterCondition::Compare {
+            comparison: FilterComparison::Equal,
+            value,
+        } => Some((&predicate.field, value)),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::normalize;
+    use crate::search::filter::{
+        FilterCondition, FilterExpr, FilterField, FilterLiteral,
+        FilterPredicate, parse_expression,
+    };
+
+    #[test]
+    fn compacts_cartesian_product() {
+        let expression = parse_expression(
+            "(game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
+             (game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
+             (game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
+             (game_versions = 1.21.1 AND categories = forge AND categories = technology)",
+        )
+        .unwrap();
+        let FilterExpr::And(predicates) = normalize(expression) else {
+            panic!("expected a compacted conjunction");
+        };
+
+        assert_eq!(predicates.len(), 3);
+        assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("categories"),
+            condition: FilterCondition::In {
+                values: vec![
+                    FilterLiteral::String("fabric".into()),
+                    FilterLiteral::String("forge".into()),
+                ],
+                negated: false,
+            },
+        })));
+        assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
+            field: FilterField::new("game_versions"),
+            condition: FilterCondition::In {
+                values: vec![
+                    FilterLiteral::String("1.20.1".into()),
+                    FilterLiteral::String("1.21.1".into()),
+                ],
+                negated: false,
+            },
+        })));
+    }
+
+    #[test]
+    fn factors_common_predicates_before_compacting() {
+        let expression = parse_expression(
+            "(license = MIT AND game_versions = 1.20.1 AND categories = fabric) OR \
+             (license = MIT AND game_versions = 1.21.1 AND categories = forge)",
+        )
+        .unwrap();
+        let FilterExpr::And(expressions) = normalize(expression) else {
+            panic!("expected a factored conjunction");
+        };
+
+        assert!(expressions.iter().any(|expression| matches!(
+            expression,
+            FilterExpr::Predicate(predicate)
+                if predicate.field.as_str() == "license"
+        )));
+        assert!(
+            expressions
+                .iter()
+                .any(|expression| matches!(expression, FilterExpr::Or(_)))
+        );
+    }
+
+    #[test]
+    fn compacts_production_sized_cartesian_product() {
+        let versions = [
+            "26.2", "26.1.2", "26.1.1", "26.1", "1.21.11", "1.21.10", "1.21.8",
+            "1.21.7", "1.21.5", "1.21.4", "1.21.3", "1.21.1", "1.21", "1.20.6",
+            "1.20.4", "1.20.2", "1.20.1", "1.20", "1.19.4", "1.19.3", "1.19.2",
+            "1.18.2", "1.17.1", "1.12.2", "1.8.9",
+        ];
+        let input = versions
+            .iter()
+            .flat_map(|version| {
+                ["fabric", "forge"].map(|loader| {
+                    format!(
+                        "(project_types = modpack AND game_versions = {version} AND categories = {loader} AND categories = technology)"
+                    )
+                })
+            })
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let normalized = normalize(parse_expression(&input).unwrap());
+        let expected = normalize(
+            parse_expression(&format!(
+                "project_types = modpack AND game_versions IN [{}] AND categories IN [fabric, forge] AND categories = technology",
+                versions.join(", ")
+            ))
+            .unwrap(),
+        );
+
+        assert_eq!(normalized, expected);
+    }
+
+    #[test]
+    fn normalizes_unary_not() {
+        let normalized = normalize(
+            parse_expression(
+                r#"NOT"project_id"="8xOSkvVU" AND NOT (downloads > 100 OR open_source = true)"#,
+            )
+            .unwrap(),
+        );
+        let expected = normalize(
+            parse_expression(
+                r#"project_id != "8xOSkvVU" AND downloads <= 100 AND open_source != true"#,
+            )
+            .unwrap(),
+        );
+
+        assert_eq!(normalized, expected);
+    }
+}

apps/labrinth/src/search/filter/parse.rs

@@ -0,0 +1,266 @@
+use std::ops::Range;
+
+use chumsky::{Parser, prelude::*};
+use thiserror::Error;
+
+use super::{
+    FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
+    FilterPredicate,
+};
+
+#[derive(Debug, Error)]
+#[error("invalid filter at byte {position}: {message}")]
+pub struct FilterParseError {
+    position: usize,
+    message: String,
+}
+
+fn keyword(
+    keyword: &'static str,
+) -> BoxedParser<'static, char, (), Simple<char>> {
+    let keyword =
+        keyword
+            .chars()
+            .fold(empty().ignored().boxed(), |parser, character| {
+                parser
+                    .then_ignore(one_of([
+                        character.to_ascii_lowercase(),
+                        character.to_ascii_uppercase(),
+                    ]))
+                    .boxed()
+            });
+    let boundary = filter(|character: &char| {
+        !character.is_ascii_alphanumeric() && !"_.".contains(*character)
+    })
+    .rewind()
+    .ignored()
+    .or(end());
+
+    keyword.then_ignore(boundary).boxed()
+}
+
+fn quoted_literal(
+    quote: char,
+) -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
+    let escaped = just('\\').ignore_then(any());
+    let character = escaped.or(filter(move |character| {
+        *character != quote && *character != '\\'
+    }));
+
+    character
+        .repeated()
+        .collect::<String>()
+        .delimited_by(just(quote), just(quote))
+        .map(FilterLiteral::String)
+        .boxed()
+}
+
+fn literal_parser() -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
+    let quoted = choice((
+        quoted_literal('\''),
+        quoted_literal('"'),
+        quoted_literal('`'),
+    ));
+    let bare = filter(|character: &char| {
+        !character.is_whitespace() && !",[]()".contains(*character)
+    })
+    .repeated()
+    .at_least(1)
+    .collect::<String>()
+    .map(FilterLiteral::from_bare);
+
+    quoted.or(bare).padded().boxed()
+}
+
+fn field_name_parser() -> BoxedParser<'static, char, String, Simple<char>> {
+    filter(|character: &char| {
+        character.is_ascii_alphabetic() || "_.".contains(*character)
+    })
+    .then(
+        filter(|character: &char| {
+            character.is_ascii_alphanumeric() || "_.".contains(*character)
+        })
+        .repeated(),
+    )
+    .map(|(first, rest)| std::iter::once(first).chain(rest).collect::<String>())
+    .boxed()
+}
+
+fn parser() -> impl Parser<char, FilterExpr, Error = Simple<char>> {
+    let field_name = field_name_parser();
+    let field = choice((
+        field_name.clone(),
+        field_name.clone().delimited_by(just('"'), just('"')),
+        field_name.clone().delimited_by(just('\''), just('\'')),
+        field_name.delimited_by(just('`'), just('`')),
+    ))
+    .map(FilterField::new)
+    .padded();
+
+    let literal = literal_parser();
+    let list = literal
+        .clone()
+        .separated_by(just(',').padded())
+        .at_least(1)
+        .allow_trailing()
+        .delimited_by(just('[').padded(), just(']').padded());
+
+    let comparison = choice((
+        just("!=").to(FilterComparison::NotEqual),
+        just(">=").to(FilterComparison::GreaterThanOrEqual),
+        just("<=").to(FilterComparison::LessThanOrEqual),
+        just('>').to(FilterComparison::GreaterThan),
+        just('<').to(FilterComparison::LessThan),
+        just('=').to(FilterComparison::Equal),
+    ))
+    .padded()
+    .then(literal.clone())
+    .map(|(comparison, value)| FilterCondition::Compare { comparison, value });
+
+    let not_in = keyword("NOT")
+        .padded()
+        .ignore_then(keyword("IN"))
+        .padded()
+        .ignore_then(list.clone())
+        .map(|values| FilterCondition::In {
+            values,
+            negated: true,
+        });
+    let in_list = keyword("IN").padded().ignore_then(list).map(|values| {
+        FilterCondition::In {
+            values,
+            negated: false,
+        }
+    });
+    let not_exists = keyword("NOT")
+        .padded()
+        .ignore_then(keyword("EXISTS"))
+        .map(|()| FilterCondition::Exists { negated: true });
+    let exists =
+        keyword("EXISTS").map(|()| FilterCondition::Exists { negated: false });
+
+    let predicate = field
+        .then(choice((not_in, in_list, not_exists, exists, comparison)))
+        .map(|(field, condition)| {
+            FilterExpr::Predicate(FilterPredicate { field, condition })
+        });
+
+    recursive(|expression| {
+        let atom = predicate
+            .clone()
+            .or(expression.delimited_by(just('(').padded(), just(')').padded()))
+            .padded();
+        let unary = keyword("NOT").padded().repeated().then(atom).map(
+            |(operators, expression)| {
+                operators.into_iter().fold(expression, |expression, ()| {
+                    FilterExpr::Not(Box::new(expression))
+                })
+            },
+        );
+        let and = unary
+            .clone()
+            .then(keyword("AND").padded().ignore_then(unary).repeated())
+            .map(|(first, rest)| {
+                FilterExpr::and(std::iter::once(first).chain(rest))
+                    .expect("an expression always contains one operand")
+            });
+
+        and.clone()
+            .then(keyword("OR").padded().ignore_then(and).repeated())
+            .map(|(first, rest)| {
+                FilterExpr::or(std::iter::once(first).chain(rest))
+                    .expect("an expression always contains one operand")
+            })
+    })
+    .padded()
+    .then_ignore(end())
+}
+
+pub fn parse_expression(input: &str) -> Result<FilterExpr, FilterParseError> {
+    parser().parse(input).map_err(|errors| {
+        let error = errors
+            .into_iter()
+            .next()
+            .unwrap_or_else(|| Simple::custom(0..0, "invalid filter"));
+        let Range { start, .. } = error.span();
+        FilterParseError {
+            position: start,
+            message: error.to_string(),
+        }
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_expression;
+    use crate::search::filter::{
+        FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
+    };
+
+    #[test]
+    fn parses_boolean_precedence() {
+        let expression = parse_expression(
+            "license = MIT OR downloads >= 100 AND open_source = true",
+        )
+        .unwrap();
+
+        let FilterExpr::Or(expressions) = expression else {
+            panic!("expected an OR expression");
+        };
+        assert_eq!(expressions.len(), 2);
+        assert!(matches!(expressions[1], FilterExpr::And(_)));
+    }
+
+    #[test]
+    fn parses_quoted_and_list_values() {
+        let expression = parse_expression(
+            r#"name = "value with spaces" AND categories IN [fabric, "forge"]"#,
+        )
+        .unwrap();
+
+        let FilterExpr::And(expressions) = expression else {
+            panic!("expected an AND expression");
+        };
+        let FilterExpr::Predicate(predicate) = &expressions[0] else {
+            panic!("expected a predicate");
+        };
+        assert!(matches!(
+            &predicate.condition,
+            FilterCondition::Compare {
+                value: FilterLiteral::String(value),
+                ..
+            } if value == "value with spaces"
+        ));
+    }
+
+    #[test]
+    fn parses_unary_not_with_quoted_field() {
+        let expression =
+            parse_expression(r#"NOT"project_id"="8xOSkvVU""#).unwrap();
+
+        let FilterExpr::Not(expression) = expression else {
+            panic!("expected a NOT expression");
+        };
+        let FilterExpr::Predicate(predicate) = *expression else {
+            panic!("expected a predicate");
+        };
+        assert_eq!(predicate.field.as_str(), "project_id");
+        assert!(matches!(
+            predicate.condition,
+            FilterCondition::Compare {
+                comparison: FilterComparison::Equal,
+                value: FilterLiteral::String(value),
+            } if value == "8xOSkvVU"
+        ));
+    }
+
+    #[test]
+    fn does_not_parse_not_prefix_in_field_name_as_operator() {
+        let expression = parse_expression("notification = true").unwrap();
+
+        let FilterExpr::Predicate(predicate) = expression else {
+            panic!("expected a predicate");
+        };
+        assert_eq!(predicate.field.as_str(), "notification");
+    }
+}

apps/labrinth/src/search/incremental.rs

@@ -1,43 +1,66 @@
 pub mod consume;
 
-use std::{mem, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    mem,
+    sync::Arc,
+    time::Duration,
+};
 
 use rdkafka::{producer::FutureRecord, util::Timeout};
 use serde::Serialize;
 use tokio::sync::Mutex;
 
 use crate::{
-    models::ids::ProjectId,
+    models::ids::{ProjectId, VersionId},
     util::kafka::{KAFKA_OPERATION_INTERVAL, KafkaClientState, KafkaEvent},
 };
 
 pub const SEARCH_PROJECT_INDEX_QUEUE_TOPIC: &str =
     "public.labrinth.search-project-index-queue.v1";
+const QUEUE_FLUSH_INTERVAL: Duration = Duration::from_secs(10);
 
 #[derive(Clone)]
 pub struct IncrementalSearchQueue {
-    operations: Arc<Mutex<Vec<SearchIndexOperation>>>,
+    operations: Arc<Mutex<PendingSearchIndexOperations>>,
     kafka_client: actix_web::web::Data<KafkaClientState>,
 }
 
 impl IncrementalSearchQueue {
     pub fn new(kafka_client: actix_web::web::Data<KafkaClientState>) -> Self {
         Self {
-            operations: Arc::new(Mutex::new(Vec::new())),
+            operations: Arc::new(Mutex::new(
+                PendingSearchIndexOperations::default(),
+            )),
             kafka_client,
         }
     }
 
-    pub async fn push(&self, project_id: ProjectId) {
+    pub async fn push_project_change(&self, project_id: ProjectId) {
+        self.operations.lock().await.push_project_change(project_id);
+    }
+
+    pub async fn push_version_changes(
+        &self,
+        project_id: ProjectId,
+        version_ids: impl IntoIterator<Item = VersionId>,
+    ) {
         self.operations
             .lock()
             .await
-            .push(SearchIndexOperation { project_id });
+            .push_version_change(project_id, version_ids);
+    }
+
+    pub async fn push_project_removal(&self, project_id: ProjectId) {
+        self.operations
+            .lock()
+            .await
+            .push_project_removal(project_id);
     }
 
     pub async fn run(self) {
         loop {
-            tokio::time::sleep(KAFKA_OPERATION_INTERVAL).await;
+            tokio::time::sleep(QUEUE_FLUSH_INTERVAL).await;
 
             if let Err(err) = self.drain().await {
                 tracing::error!(
@@ -57,13 +80,11 @@ impl IncrementalSearchQueue {
             return Ok(());
         }
 
-        let mut operations = operations.into_iter();
+        let mut operations = operations.into_events().into_iter();
         while let Some(operation) = operations.next() {
             let event = KafkaEvent::new(
                 SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
-                SearchProjectIndexQueueEventData {
+                operation.clone(),
-                    project_id: operation.project_id,
-                },
             );
             let event_id = event.event_metadata.event_id;
             let key = event_id.to_string();
@@ -79,8 +100,10 @@ impl IncrementalSearchQueue {
                 .await
             {
                 let mut queued_operations = self.operations.lock().await;
-                queued_operations.push(operation);
+                queued_operations.push_event(operation);
-                queued_operations.extend(operations);
+                for operation in operations {
+                    queued_operations.push_event(operation);
+                }
 
                 return Err(err.into());
             }
@@ -90,12 +113,100 @@ impl IncrementalSearchQueue {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Default)]
-pub struct SearchIndexOperation {
+struct PendingSearchIndexOperations {
-    pub project_id: ProjectId,
+    changed_project_ids: HashSet<ProjectId>,
+    changed_project_versions: HashMap<ProjectId, HashSet<VersionId>>,
+    removed_project_ids: HashSet<ProjectId>,
 }
 
-#[derive(Debug, Serialize)]
+impl PendingSearchIndexOperations {
-pub struct SearchProjectIndexQueueEventData {
+    fn is_empty(&self) -> bool {
-    pub project_id: ProjectId,
+        self.changed_project_ids.is_empty()
+            && self.changed_project_versions.is_empty()
+            && self.removed_project_ids.is_empty()
+    }
+
+    fn push_project_change(&mut self, project_id: ProjectId) {
+        if !self.removed_project_ids.contains(&project_id) {
+            self.changed_project_ids.insert(project_id);
+        }
+    }
+
+    fn push_version_change(
+        &mut self,
+        project_id: ProjectId,
+        version_ids: impl IntoIterator<Item = VersionId>,
+    ) {
+        if self.removed_project_ids.contains(&project_id) {
+            return;
+        }
+
+        let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
+        if !version_ids.is_empty() {
+            self.changed_project_versions
+                .entry(project_id)
+                .or_default()
+                .extend(version_ids);
+        }
+    }
+
+    fn push_project_removal(&mut self, project_id: ProjectId) {
+        self.changed_project_ids.remove(&project_id);
+        self.changed_project_versions.remove(&project_id);
+        self.removed_project_ids.insert(project_id);
+    }
+
+    fn push_event(&mut self, event: SearchProjectIndexQueueEventData) {
+        match event {
+            SearchProjectIndexQueueEventData::Change { project_id } => {
+                self.push_project_change(project_id)
+            }
+            SearchProjectIndexQueueEventData::VersionChange {
+                project_id,
+                version_ids,
+            } => self.push_version_change(project_id, version_ids),
+            SearchProjectIndexQueueEventData::Removal { project_id } => {
+                self.push_project_removal(project_id)
+            }
+        }
+    }
+
+    fn into_events(self) -> Vec<SearchProjectIndexQueueEventData> {
+        let mut events = Vec::with_capacity(
+            self.changed_project_ids.len()
+                + self.changed_project_versions.len()
+                + self.removed_project_ids.len(),
+        );
+
+        events.extend(self.removed_project_ids.into_iter().map(|project_id| {
+            SearchProjectIndexQueueEventData::Removal { project_id }
+        }));
+        events.extend(self.changed_project_ids.into_iter().map(|project_id| {
+            SearchProjectIndexQueueEventData::Change { project_id }
+        }));
+        events.extend(self.changed_project_versions.into_iter().map(
+            |(project_id, version_ids)| {
+                SearchProjectIndexQueueEventData::VersionChange {
+                    project_id,
+                    version_ids: version_ids.into_iter().collect(),
+                }
+            },
+        ));
+        events
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum SearchProjectIndexQueueEventData {
+    #[serde(rename = "project_change")]
+    Change { project_id: ProjectId },
+    #[serde(rename = "project_version_change")]
+    VersionChange {
+        project_id: ProjectId,
+        version_ids: Vec<VersionId>,
+    },
+    #[serde(rename = "project_removal")]
+    Removal { project_id: ProjectId },
 }

apps/labrinth/src/search/incremental/consume.rs

@@ -1,21 +1,27 @@
 use actix_web::web;
 use eyre::WrapErr;
-use futures::FutureExt;
 use rdkafka::{
     Message,
     consumer::{CommitMode, Consumer, StreamConsumer},
     message::BorrowedMessage,
 };
 use serde::Deserialize;
-use std::collections::HashSet;
+use std::{
+    collections::HashSet,
+    time::{Duration, Instant},
+};
+use tracing::{Instrument, info, info_span};
 use xredis::RedisPool;
 
 use crate::{
     database::PgPool,
-    models::ids::ProjectId,
+    env::ENV,
+    models::ids::{ProjectId, VersionId},
     search::{
-        SearchBackend, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
+        SearchBackend, SearchDocumentBatch, SearchIndexUpdate,
-        indexing::index_project_documents,
+        UploadSearchProject,
+        incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
+        indexing::{build_project_documents, build_version_change_documents},
     },
     util::kafka::{
         INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL,
@@ -23,8 +29,6 @@ use crate::{
     },
 };
 
-const BATCH_SIZE: usize = 100;
-
 pub async fn run(
     ro_pool: PgPool,
     redis_pool: RedisPool,
@@ -61,25 +65,60 @@ async fn consume(
     search_backend: &dyn SearchBackend,
     consumer: &StreamConsumer,
 ) -> eyre::Result<()> {
+    // keep buffer capacity (pre-)allocated
+    let mut messages = Vec::with_capacity(1024);
     loop {
-        let mut messages = Vec::with_capacity(BATCH_SIZE);
+        messages.clear();
-        messages.push(
+
-            consumer
+        // wait for a first message to come in...
-                .recv()
+        let first_message = consumer
-                .await
+            .recv()
-                .wrap_err("failed to receive Kafka message")?,
+            .await
+            .wrap_err("failed to receive Kafka message")?;
+        messages.push(first_message);
+
+        let delay = Duration::from_secs(
+            ENV.SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS,
+        );
+        info!(
+            "Received initial Kafka message; waiting {delay:.2?} for more to batch",
         );
 
-        while messages.len() < BATCH_SIZE {
+        // ..then wait a while for more messages to batch up
-            let Some(message) = consumer.recv().now_or_never() else {
+        // so that we can process a big batch to reindex.
-                break;
+        // we stop until either we've reached the max batch size,
-            };
+        // or we've waited enough time - whichever is first.
-
+        //
-            messages.push(message.wrap_err("failed to receive Kafka message")?);
+        // do a little trick with an `AsyncFnMut` closure
+        // so that we can explicitly specify the return type
+        let mut collect_more_messages = async || -> eyre::Result<()> {
+            while messages.len() < ENV.SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE {
+                let message = consumer
+                    .recv()
+                    .await
+                    .wrap_err("failed to receive Kafka message")?;
+                messages.push(message);
+            }
+            eyre::Ok(())
+        };
+        match tokio::time::timeout(delay, collect_more_messages()).await {
+            Ok(Ok(())) | Err(_) => {}
+            Ok(Err(err)) => {
+                return Err(
+                    err.wrap_err("failed to receive more Kafka messages")
+                );
+            }
         }
 
-        consume_batch(ro_pool, redis_pool, search_backend, consumer, messages)
+        info!("Consuming batch of {} messages", messages.len());
-            .await?;
+        consume_batch(
+            ro_pool,
+            redis_pool,
+            search_backend,
+            consumer,
+            messages.drain(..),
+        )
+        .await?;
     }
 }
 
@@ -88,10 +127,14 @@ async fn consume_batch(
     redis_pool: &RedisPool,
     search_backend: &dyn SearchBackend,
     consumer: &StreamConsumer,
-    messages: Vec<BorrowedMessage<'_>>,
+    messages: impl IntoIterator<Item = BorrowedMessage<'_>>,
 ) -> eyre::Result<()> {
-    let mut project_ids = Vec::new();
+    let start = Instant::now();
-    let mut seen_project_ids = HashSet::new();
+
+    let mut project_ids_to_change = HashSet::new();
+    let mut project_ids_with_version_changes = HashSet::new();
+    let mut project_ids_to_remove = HashSet::new();
+    let mut version_ids_to_change = HashSet::new();
     let mut messages_to_commit = Vec::new();
 
     for message in messages {
@@ -126,25 +169,149 @@ async fn consume_batch(
             }
         };
 
-        if seen_project_ids.insert(event.project_id) {
+        let event = match event {
-            project_ids.push(event.project_id);
+            SearchProjectIndexQueueEvent::Current(event) => event,
+            SearchProjectIndexQueueEvent::Legacy { project_id } => {
+                SearchProjectIndexQueueEventData::Change { project_id }
+            }
+        };
+
+        match event {
+            SearchProjectIndexQueueEventData::Change { project_id } => {
+                project_ids_to_change.insert(project_id);
+            }
+            SearchProjectIndexQueueEventData::VersionChange {
+                project_id,
+                version_ids,
+            } => {
+                if !version_ids.is_empty() {
+                    project_ids_with_version_changes.insert(project_id);
+                    version_ids_to_change.extend(version_ids);
+                }
+            }
+            SearchProjectIndexQueueEventData::Removal { project_id } => {
+                project_ids_to_remove.insert(project_id);
+            }
         }
         messages_to_commit.push(message);
     }
 
-    if project_ids.is_empty() {
+    project_ids_to_change
-        return Ok(());
+        .retain(|project_id| !project_ids_to_remove.contains(project_id));
+    project_ids_with_version_changes
+        .retain(|project_id| !project_ids_to_remove.contains(project_id));
+    project_ids_to_change.retain(|project_id| {
+        !project_ids_with_version_changes.contains(project_id)
+    });
+    let project_ids_to_change =
+        project_ids_to_change.into_iter().collect::<Vec<_>>();
+    let project_ids_with_version_changes = project_ids_with_version_changes
+        .into_iter()
+        .collect::<Vec<_>>();
+    let mut project_ids_to_remove =
+        project_ids_to_remove.into_iter().collect::<Vec<_>>();
+    let version_ids_to_change =
+        version_ids_to_change.into_iter().collect::<Vec<_>>();
+
+    info!(
+        kafka.message_count = messages_to_commit.len(),
+        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove",
+        start.elapsed(),
+        project_ids_to_change.len(),
+        project_ids_with_version_changes.len(),
+        version_ids_to_change.len(),
+        project_ids_to_remove.len(),
+    );
+    let start = Instant::now();
+    let mut documents = SearchDocumentBatch::default();
+
+    if !project_ids_with_version_changes.is_empty() {
+        let operation_start = Instant::now();
+        let changed_documents = build_version_change_documents(
+            ro_pool,
+            redis_pool,
+            &project_ids_with_version_changes,
+            &version_ids_to_change,
+        )
+        .instrument(info_span!(
+            "index",
+            batch_size = project_ids_with_version_changes.len(),
+            version_count = version_ids_to_change.len()
+        ))
+        .await
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects and {} versions",
+                project_ids_with_version_changes.len(),
+                version_ids_to_change.len()
+            )
+        })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_with_version_changes,
+            &changed_documents.projects,
+        ));
+        documents.projects.extend(changed_documents.projects);
+        documents.versions.extend(changed_documents.versions);
+        info!(
+            project_count = project_ids_with_version_changes.len(),
+            version_count = version_ids_to_change.len(),
+            "Built changed project versions in {:.2?}",
+            operation_start.elapsed()
+        );
     }
 
-    tracing::info!(
+    if !project_ids_to_change.is_empty() {
-        kafka.message_count = messages_to_commit.len(),
+        let operation_start = Instant::now();
-        project_count = project_ids.len(),
+        info!(
-        "Consumed incremental search index event batch"
+            project_count = project_ids_to_change.len(),
-    );
+            "Building changed projects"
-
+        );
-    reindex_projects(ro_pool, redis_pool, search_backend, &project_ids)
+        let changed_project_documents = build_project_documents(
+            ro_pool,
+            redis_pool,
+            &project_ids_to_change,
+        )
+        .instrument(info_span!(
+            "index",
+            batch_size = project_ids_to_change.len()
+        ))
         .await
-        .wrap_err("failed to reindex project batch")?;
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects",
+                project_ids_to_change.len()
+            )
+        })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_to_change,
+            &changed_project_documents,
+        ));
+        documents.projects.extend(changed_project_documents);
+        info!(
+            project_count = project_ids_to_change.len(),
+            "Built changed projects in {:.2?}",
+            operation_start.elapsed()
+        );
+    }
+
+    let operation_start = Instant::now();
+    search_backend
+        .apply_update(SearchIndexUpdate {
+            projects: &documents.projects,
+            versions: &documents.versions,
+            removed_projects: &project_ids_to_remove,
+            removed_versions: &version_ids_to_change,
+        })
+        .await
+        .wrap_err("failed to apply search index update")?;
+    info!(
+        project_count = documents.projects.len(),
+        version_count = documents.versions.len(),
+        removed_project_count = project_ids_to_remove.len(),
+        removed_version_count = version_ids_to_change.len(),
+        "Applied search index update in {:.2?}",
+        operation_start.elapsed()
+    );
 
     for message in messages_to_commit {
         consumer
@@ -152,45 +319,94 @@ async fn consume_batch(
             .wrap_err("failed to commit Kafka message")?;
     }
 
+    info!(
+        "Changed {} projects and removed {} projects in {:.2?}",
+        project_ids_to_change.len(),
+        project_ids_to_remove.len(),
+        start.elapsed()
+    );
+
     Ok(())
 }
 
-pub async fn reindex_project(
+pub async fn reindex_project_document(
     ro_pool: &PgPool,
     redis_pool: &RedisPool,
     search_backend: &dyn SearchBackend,
     project_id: ProjectId,
 ) -> eyre::Result<()> {
-    reindex_projects(ro_pool, redis_pool, search_backend, &[project_id]).await
+    reindex_project_documents(
+        ro_pool,
+        redis_pool,
+        search_backend,
+        &[project_id],
+    )
+    .await
 }
 
-pub async fn reindex_projects(
+pub async fn reindex_project_documents(
     ro_pool: &PgPool,
     redis_pool: &RedisPool,
     search_backend: &dyn SearchBackend,
     project_ids: &[ProjectId],
 ) -> eyre::Result<()> {
-    search_backend.remove_project_documents(project_ids).await?;
+    info!("Creating project documents");
-
+    let projects = build_project_documents(ro_pool, redis_pool, project_ids)
-    let mut documents = Vec::new();
+        .instrument(info_span!("index", batch_size = project_ids.len()))
-    for project_id in project_ids {
+        .await
-        documents.extend(
+        .wrap_err_with(|| {
-            index_project_documents(ro_pool, redis_pool, *project_id)
+            format!(
-                .await
+                "failed to build search documents for {} projects",
-                .wrap_err_with(|| {
+                project_ids.len()
-                    format!(
+            )
-                        "failed to build project {project_id} search documents"
+        })?;
-                    )
+    let removed_projects = missing_project_document_ids(project_ids, &projects);
-                })?,
+    search_backend
-        );
+        .apply_update(SearchIndexUpdate {
-    }
+            projects: &projects,
-
+            removed_projects: &removed_projects,
-    search_backend.index_documents(&documents).await?;
+            ..SearchIndexUpdate::default()
+        })
+        .await?;
 
     Ok(())
 }
 
+fn missing_project_document_ids(
+    project_ids: &[ProjectId],
+    documents: &[UploadSearchProject],
+) -> Vec<ProjectId> {
+    let built_project_ids = documents
+        .iter()
+        .map(|project| project.project_id.as_str())
+        .collect::<HashSet<_>>();
+
+    project_ids
+        .iter()
+        .copied()
+        .filter(|project_id| {
+            !built_project_ids.contains(project_id.to_string().as_str())
+        })
+        .collect()
+}
+
 #[derive(Debug, Deserialize)]
-struct SearchProjectIndexQueueEvent {
+#[serde(untagged)]
-    project_id: ProjectId,
+enum SearchProjectIndexQueueEvent {
+    Current(SearchProjectIndexQueueEventData),
+    Legacy { project_id: ProjectId },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum SearchProjectIndexQueueEventData {
+    #[serde(rename = "project_change")]
+    Change { project_id: ProjectId },
+    #[serde(rename = "project_version_change")]
+    VersionChange {
+        project_id: ProjectId,
+        version_ids: Vec<VersionId>,
+    },
+    #[serde(rename = "project_removal")]
+    Removal { project_id: ProjectId },
 }

apps/labrinth/src/search/indexing.rs

@@ -5,7 +5,7 @@ use futures::TryStreamExt;
 use heck::ToKebabCase;
 use itertools::Itertools;
 use regex::Regex;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 use tracing::{info, warn};
 
@@ -19,11 +19,14 @@ use crate::database::models::{
     LoaderFieldEnumValueId, LoaderFieldId,
 };
 use crate::models::exp;
-use crate::models::ids::ProjectId;
+use crate::models::ids::{ProjectId, VersionId};
 use crate::models::projects::{DependencyType, from_duplicate_version_fields};
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
-use crate::search::{SearchProjectDependency, UploadSearchProject};
+use crate::search::{
+    SearchDocumentBatch, SearchProjectDependency, UploadSearchProject,
+    UploadSearchVersion,
+};
 use crate::util::error::Context;
 use xredis::RedisPool;
 
@@ -68,7 +71,7 @@ pub async fn index_local(
     redis: &RedisPool,
     cursor: i64,
     limit: i64,
-) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
+) -> eyre::Result<(SearchDocumentBatch, i64)> {
     info!("Indexing local projects!");
 
     let searchable_statuses = searchable_statuses();
@@ -111,20 +114,52 @@ pub async fn index_local(
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let Some(largest) = project_ids.iter().max() else {
-        return Ok((vec![], i64::MAX));
+        return Ok((SearchDocumentBatch::default(), i64::MAX));
     };
 
-    let uploads = build_search_documents(pool, redis, db_projects).await?;
+    let documents =
-    Ok((uploads, *largest))
+        build_search_documents(pool, redis, db_projects, None).await?;
+    Ok((documents, *largest))
 }
 
-pub async fn index_project_documents(
+pub async fn build_project_documents(
     pool: &PgPool,
     redis: &RedisPool,
-    project_id: ProjectId,
+    project_ids: &[ProjectId],
 ) -> eyre::Result<Vec<UploadSearchProject>> {
+    let version_ids = HashSet::new();
+    Ok(
+        build_search_document_batch(pool, redis, project_ids, &version_ids)
+            .await?
+            .projects,
+    )
+}
+
+pub async fn build_version_change_documents(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+    version_ids: &[VersionId],
+) -> eyre::Result<SearchDocumentBatch> {
+    let version_ids = version_ids
+        .iter()
+        .copied()
+        .map(DBVersionId::from)
+        .collect::<HashSet<_>>();
+    build_search_document_batch(pool, redis, project_ids, &version_ids).await
+}
+
+async fn build_search_document_batch(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+    version_ids: &HashSet<DBVersionId>,
+) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
-    let project_ids = vec![DBProjectId::from(project_id).0];
+    let project_ids = project_ids
+        .iter()
+        .map(|project_id| DBProjectId::from(*project_id).0)
+        .collect::<Vec<_>>();
 
     let db_projects = sqlx::query!(
         r#"
@@ -158,14 +193,15 @@ pub async fn index_project_documents(
     .await
     .wrap_err("failed to fetch project")?;
 
-    build_search_documents(pool, redis, db_projects).await
+    build_search_documents(pool, redis, db_projects, Some(version_ids)).await
 }
 
 async fn build_search_documents(
     pool: &PgPool,
     redis: &RedisPool,
     db_projects: Vec<PartialProject>,
-) -> eyre::Result<Vec<UploadSearchProject>> {
+    version_ids: Option<&HashSet<DBVersionId>>,
+) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let project_components = db_projects
@@ -281,7 +317,7 @@ async fn build_search_documents(
     .await?;
 
     info!("Indexing local versions!");
-    let mut versions = index_versions(pool, project_ids.clone()).await?;
+    let mut versions = load_project_versions(pool, project_ids.clone()).await?;
 
     info!("Indexing local org owners!");
 
@@ -333,7 +369,7 @@ async fn build_search_documents(
     .await?;
 
     info!("Getting all loader fields!");
-    let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
+    let loader_field_definitions: Vec<QueryLoaderField> = sqlx::query!(
         "
         SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional
         FROM loader_fields lf
@@ -351,7 +387,8 @@ async fn build_search_documents(
     })
     .try_collect()
     .await?;
-    let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect();
+    let loader_field_definitions: Vec<&QueryLoaderField> =
+        loader_field_definitions.iter().collect();
 
     info!("Getting all loader field enum values!");
 
@@ -376,7 +413,8 @@ async fn build_search_documents(
         .await?;
 
     info!("Indexing loaders, project types!");
-    let mut uploads = Vec::new();
+    let mut project_uploads = Vec::new();
+    let mut version_uploads = Vec::new();
 
     let total_len = db_projects.len();
     let mut count = 0;
@@ -453,6 +491,9 @@ async fn build_search_documents(
             } else {
                 (vec![], vec![])
             };
+        let mut project_categories = categories;
+        project_categories.sort();
+        project_categories.dedup();
         let dependencies = dependencies
             .get(&project.id)
             .map(|x| x.clone())
@@ -475,176 +516,231 @@ async fn build_search_documents(
             .collect::<Vec<_>>();
 
         if let Some(versions) = versions.remove(&project.id) {
-            // Aggregated project loader fields
+            let Some(latest_version) = versions.iter().max_by(|a, b| {
+                a.date_published
+                    .cmp(&b.date_published)
+                    .then_with(|| a.id.0.cmp(&b.id.0))
+            }) else {
+                continue;
+            };
+
             let project_version_fields = versions
                 .iter()
                 .flat_map(|x| x.version_fields.clone())
                 .collect::<Vec<_>>();
             let aggregated_version_fields = VersionField::from_query_json(
                 project_version_fields,
-                &loader_fields,
+                &loader_field_definitions,
                 &loader_field_enum_values,
                 true,
             );
-            let project_loader_fields =
+            let unvectorized_loader_fields = aggregated_version_fields
+                .iter()
+                .map(|field| {
+                    (field.field_name.clone(), field.value.serialize_internal())
+                })
+                .collect();
+            let mut loader_fields =
                 from_duplicate_version_fields(aggregated_version_fields);
+            let project_loader_fields = loader_fields.clone();
 
-            // aggregated project loaders
+            let mut project_loaders = versions
-            let project_loaders = versions
                 .iter()
                 .flat_map(|x| x.loaders.clone())
                 .collect::<Vec<_>>();
+            project_loaders.sort();
+            project_loaders.dedup();
 
-            // all valid project types across every version of the project, so that
+            let mut project_types = versions
-            // filters can exclude projects that have *any* version of a given
-            // project type (unlike the version-specific `project_types` field).
-            let mut all_project_types = versions
                 .iter()
                 .flat_map(|x| x.project_types.clone())
                 .collect::<Vec<_>>();
-            all_project_types.sort();
+            project_types.sort();
-            all_project_types.dedup();
+            project_types.dedup();
             exp::compat::correct_project_types(
                 &project.components,
-                &mut all_project_types,
+                &mut project_types,
             );
 
-            for version in versions {
+            let project_id = ProjectId::from(project.id).to_string();
+            version_uploads.extend(versions.iter().filter_map(|version| {
+                if version_ids.is_some_and(|version_ids| {
+                    !version_ids.contains(&version.id)
+                }) {
+                    return None;
+                }
+
                 let version_fields = VersionField::from_query_json(
-                    version.version_fields,
+                    version.version_fields.clone(),
-                    &loader_fields,
+                    &loader_field_definitions,
                     &loader_field_enum_values,
                     false,
                 );
                 let unvectorized_loader_fields = version_fields
                     .iter()
-                    .map(|vf| {
+                    .map(|field| {
-                        (vf.field_name.clone(), vf.value.serialize_internal())
+                        (
+                            field.field_name.clone(),
+                            field.value.serialize_internal(),
+                        )
                     })
                     .collect();
-                let mut loader_fields =
+                let mut fields = from_duplicate_version_fields(version_fields);
-                    from_duplicate_version_fields(version_fields);
+                let mut version_project_types = version.project_types.clone();
-                let mut project_types = version.project_types;
-
                 exp::compat::correct_project_types(
                     &project.components,
-                    &mut project_types,
+                    &mut version_project_types,
                 );
 
-                let mut version_loaders = version.loaders;
-
-                // Uses version loaders, not project loaders.
-                let mut categories = categories.clone();
-                categories.append(&mut version_loaders.clone());
-
-                let display_categories = display_categories.clone();
-                categories.append(&mut version_loaders);
-
                 // SPECIAL BEHAVIOUR
                 // Todo: revisit.
                 // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category.
                 // These were previously considered the loader, and in v2, the loader is a category for searching.
                 // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories.
-                // The loaders are kept in loader_fields as well, so that no information is lost on retrieval.
+                // The loaders are kept in the project document's aggregated loader fields as well, so that no information is lost on retrieval.
-                let mrpack_loaders = loader_fields
+                let mut version_categories = project_categories.clone();
+                version_categories.extend(version.loaders.iter().cloned());
+                let mrpack_loaders = fields
                     .get("mrpack_loaders")
-                    .cloned()
+                    .into_iter()
-                    .map(|x| {
+                    .flatten()
-                        x.into_iter()
+                    .filter_map(|value| value.as_str().map(String::from))
-                            .filter_map(|x| x.as_str().map(String::from))
+                    .collect::<Vec<_>>();
-                            .collect::<Vec<_>>()
+                version_categories.extend(mrpack_loaders);
-                    })
+                if fields.contains_key("mrpack_loaders") {
-                    .unwrap_or_default();
+                    version_categories.retain(|category| category != "mrpack");
-                categories.extend(mrpack_loaders);
-                if loader_fields.contains_key("mrpack_loaders") {
-                    categories.retain(|x| *x != "mrpack");
                 }
+                version_categories.sort();
+                version_categories.dedup();
 
-                // SPECIAL BEHAVIOUR:
-                // For consistency with v2 searching, we manually input the
-                // client_side and server_side fields from the loader fields into
-                // separate loader fields.
-                // 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
                 let (_, v2_og_project_type) =
-                    LegacyProject::get_project_type(&project_types);
+                    LegacyProject::get_project_type(&version_project_types);
                 let (client_side, server_side) =
                     v2_reroute::convert_v3_side_types_to_v2_side_types(
                         &unvectorized_loader_fields,
                         Some(&v2_og_project_type),
                     );
-
                 if let Ok(client_side) = serde_json::to_value(client_side) {
-                    loader_fields
+                    fields.insert("client_side".to_string(), vec![client_side]);
-                        .insert("client_side".to_string(), vec![client_side]);
                 }
                 if let Ok(server_side) = serde_json::to_value(server_side) {
-                    loader_fields
+                    fields.insert("server_side".to_string(), vec![server_side]);
-                        .insert("server_side".to_string(), vec![server_side]);
                 }
-
+                fields.retain(|field, _| {
-                let components = project
+                    matches!(
-                    .components
+                        field.as_str(),
-                    .clone()
+                        "environment"
-                    .into_query(
+                            | "game_versions"
-                        ProjectId::from(project.id),
+                            | "client_side"
-                        &project_query_context,
+                            | "server_side"
                     )
-                    .wrap_err("failed to populate query components")?;
+                });
 
-                let usp = UploadSearchProject {
+                Some(UploadSearchVersion {
-                    version_id: crate::models::ids::VersionId::from(version.id)
+                    version_id: VersionId::from(version.id).to_string(),
-                        .to_string(),
+                    project_id: project_id.clone(),
-                    project_id: crate::models::ids::ProjectId::from(project.id)
+                    categories: version_categories,
-                        .to_string(),
+                    project_types: version_project_types,
-                    name: project.name.clone(),
-                    indexed_name: normalize_for_search(&project.name),
-                    summary: project.summary.clone(),
-                    categories: categories.clone(),
-                    display_categories: display_categories.clone(),
-                    follows: project.follows,
-                    downloads: project.downloads,
-                    log_downloads: (project.downloads.max(1) as f64).ln(),
-                    icon_url: project.icon_url.clone(),
-                    author: username.clone(),
-                    author_id: ariadne::ids::UserId::from(user_id).to_string(),
-                    organization: org_name.clone(),
-                    organization_id: org_id.map(|e| {
-                        crate::models::ids::OrganizationId::from(e).to_string()
-                    }),
-                    indexed_author: normalize_for_search(&username),
-                    date_created: project.approved,
-                    created_timestamp: project.approved.timestamp(),
-                    date_modified: project.updated,
-                    modified_timestamp: project.updated.timestamp(),
                     version_published_timestamp: version
                         .date_published
                         .timestamp(),
-                    license: license.clone(),
+                    loader_fields: fields,
-                    slug: project.slug.clone(),
+                })
-                    // TODO
+            }));
-                    project_types,
-                    all_project_types: all_project_types.clone(),
-                    gallery: gallery.clone(),
-                    featured_gallery: featured_gallery.clone(),
-                    open_source,
-                    color: project.color.map(|x| x as u32),
-                    dependency_project_ids: dependency_project_ids.clone(),
-                    compatible_dependency_project_ids:
-                        compatible_dependency_project_ids.clone(),
-                    dependencies: dependencies.clone(),
-                    loader_fields,
-                    project_loader_fields: project_loader_fields.clone(),
-                    // 'loaders' is aggregate of all versions' loaders
-                    loaders: project_loaders.clone(),
-                    components,
-                };
 
-                uploads.push(usp);
+            let mut categories = project_categories.clone();
+            categories.extend(project_loaders.iter().cloned());
+
+            let mrpack_loaders = loader_fields
+                .get("mrpack_loaders")
+                .into_iter()
+                .flatten()
+                .filter_map(|value| value.as_str().map(String::from))
+                .collect::<Vec<_>>();
+            categories.extend(mrpack_loaders);
+            if loader_fields.contains_key("mrpack_loaders") {
+                categories.retain(|category| category != "mrpack");
             }
+            categories.sort();
+            categories.dedup();
+
+            let (_, v2_og_project_type) =
+                LegacyProject::get_project_type(&project_types);
+            let (client_side, server_side) =
+                v2_reroute::convert_v3_side_types_to_v2_side_types(
+                    &unvectorized_loader_fields,
+                    Some(&v2_og_project_type),
+                );
+
+            if let Ok(client_side) = serde_json::to_value(client_side) {
+                loader_fields
+                    .insert("client_side".to_string(), vec![client_side]);
+            }
+            if let Ok(server_side) = serde_json::to_value(server_side) {
+                loader_fields
+                    .insert("server_side".to_string(), vec![server_side]);
+            }
+
+            let components = project
+                .components
+                .clone()
+                .into_query(ProjectId::from(project.id), &project_query_context)
+                .wrap_err("failed to populate query components")?;
+            let indexed_name = normalize_for_search(&project.name);
+
+            project_uploads.push(UploadSearchProject {
+                version_id: crate::models::ids::VersionId::from(
+                    latest_version.id,
+                )
+                .to_string(),
+                project_id,
+                name: project.name,
+                indexed_name,
+                summary: project.summary,
+                categories,
+                project_categories,
+                display_categories,
+                follows: project.follows,
+                downloads: project.downloads,
+                log_downloads: (project.downloads.max(1) as f64).ln(),
+                icon_url: project.icon_url,
+                author: username.clone(),
+                author_id: ariadne::ids::UserId::from(user_id).to_string(),
+                organization: org_name,
+                organization_id: org_id.map(|id| {
+                    crate::models::ids::OrganizationId::from(id).to_string()
+                }),
+                indexed_author: normalize_for_search(&username),
+                date_created: project.approved,
+                created_timestamp: project.approved.timestamp(),
+                date_modified: project.updated,
+                modified_timestamp: project.updated.timestamp(),
+                version_published_timestamp: latest_version
+                    .date_published
+                    .timestamp(),
+                license,
+                slug: project.slug,
+                project_types: project_types.clone(),
+                all_project_types: project_types,
+                gallery,
+                featured_gallery,
+                open_source,
+                color: project.color.map(|x| x as u32),
+                dependency_project_ids,
+                compatible_dependency_project_ids,
+                dependencies,
+                project_loader_fields,
+                loader_fields,
+                loaders: project_loaders,
+                components,
+            });
         }
     }
 
-    Ok(uploads)
+    Ok(SearchDocumentBatch {
+        projects: project_uploads,
+        versions: version_uploads,
+    })
 }
 
 struct PartialVersion {
@@ -655,7 +751,7 @@ struct PartialVersion {
     date_published: DateTime<Utc>,
 }
 
-async fn index_versions(
+async fn load_project_versions(
     pool: &PgPool,
     project_ids: Vec<i64>,
 ) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>> {

apps/labrinth/src/search/mod.rs

@@ -16,6 +16,7 @@ use utoipa::ToSchema;
 use xredis::RedisPool;
 
 pub mod backend;
+pub mod filter;
 pub mod incremental;
 pub mod indexing;
 
@@ -105,24 +106,17 @@ pub trait SearchBackend: Send + Sync {
         info: &SearchRequest,
     ) -> Result<SearchResults, ApiError>;
 
-    async fn index_projects(
+    async fn rebuild_index(
         &self,
         ro_pool: PgPool,
         redis: RedisPool,
     ) -> eyre::Result<()>;
 
-    async fn index_documents(
+    async fn apply_update(
         &self,
-        documents: &[UploadSearchProject],
+        update: SearchIndexUpdate<'_>,
     ) -> eyre::Result<()>;
 
-    async fn remove_project_documents(
-        &self,
-        ids: &[ProjectId],
-    ) -> eyre::Result<()>;
-
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
-
     async fn tasks(&self) -> eyre::Result<Value>;
 
     async fn tasks_cancel(
@@ -235,8 +229,12 @@ impl FromStr for SearchBackendKind {
     }
 }
 
+/// Nullable fields in Typesense-bound documents should use
+/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
+/// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     //
@@ -246,20 +244,25 @@ pub struct UploadSearchProject {
     pub slug: Option<String>,
     pub author: String,
     pub author_id: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub organization: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub organization_id: Option<String>,
     pub indexed_author: String,
     pub name: String,
     pub indexed_name: String,
     pub summary: String,
     pub categories: Vec<String>,
+    pub project_categories: Vec<String>,
     pub display_categories: Vec<String>,
     pub follows: i32,
     pub downloads: i32,
     pub log_downloads: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub icon_url: Option<String>,
     pub license: String,
     pub gallery: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub featured_gallery: Option<String>,
     /// RFC 3339 formatted creation date of the project
     pub date_created: DateTime<Utc>,
@@ -269,9 +272,10 @@ pub struct UploadSearchProject {
     pub date_modified: DateTime<Utc>,
     /// Unix timestamp of the last major modification
     pub modified_timestamp: i64,
-    /// Unix timestamp of the publication date of the version
+    /// Unix timestamp of the most recently published version.
     pub version_published_timestamp: i64,
     pub open_source: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub color: Option<u32>,
     #[serde(default)]
     pub dependency_project_ids: Vec<String>,
@@ -290,12 +294,45 @@ pub struct UploadSearchProject {
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct UploadSearchVersion {
+    pub version_id: String,
+    pub project_id: String,
+    pub categories: Vec<String>,
+    pub project_types: Vec<String>,
+    pub version_published_timestamp: i64,
+    #[serde(flatten)]
+    pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
+}
+
+#[derive(Debug, Default)]
+pub struct SearchDocumentBatch {
+    pub projects: Vec<UploadSearchProject>,
+    pub versions: Vec<UploadSearchVersion>,
+}
+
+/// A logical search index mutation. Removals are applied before replacements,
+/// so a document may be present in both a removed and replacement field.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct SearchIndexUpdate<'a> {
+    pub projects: &'a [UploadSearchProject],
+    pub versions: &'a [UploadSearchVersion],
+    /// Projects and all of their version documents to remove.
+    pub removed_projects: &'a [ProjectId],
+    pub removed_versions: &'a [VersionId],
+}
+
+/// Nullable fields in Typesense-bound documents should use
+/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
+/// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct SearchProjectDependency {
     pub project_id: String,
     pub dependency_type: DependencyType,
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub slug: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub icon_url: Option<String>,
 }
 
@@ -309,6 +346,7 @@ pub struct SearchResults {
 
 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct ResultSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     pub project_types: Vec<String>,

apps/labrinth/tests/search.rs

@@ -213,18 +213,7 @@ async fn index_swaps() {
                 test_env.api.remove_project("alpha", USER_USER_PAT).await;
             assert_status!(&resp, StatusCode::NO_CONTENT);
 
-            // We should wait for deletions to be indexed
+            // When we reindex, the deleted project should be gone
-            let projects = test_env
-                .api
-                .search_deserialized(
-                    None,
-                    Some(json!([["categories:fabric"]])),
-                    USER_USER_PAT,
-                )
-                .await;
-            assert_eq!(projects.total_hits, 0);
-
-            // When we reindex, it should be still gone
             let resp = test_env.api.reset_search_index().await;
             assert_status!(&resp, StatusCode::NO_CONTENT);
 

packages/app-lib/Cargo.toml

@@ -121,7 +121,7 @@ dunce = { workspace = true }
 [target.'cfg(target_os = "macos")'.dependencies]
 cidre = { workspace = true, features = ["blocks", "nw"] }
 
-[target.'cfg(windows)'.dependencies]
+[target."cfg(windows)".dependencies]
 windows = { workspace = true, features = ["Networking_Connectivity"] }
 windows-core = { workspace = true }
 winreg = { workspace = true }