From 2d089a75264b987ff902cdcbd3ff49c48fa0c542 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:35:08 +0100 Subject: [PATCH] query perf --- .../backend/typesense/filter_rewrite.rs | 22 +++- .../src/search/backend/typesense/mod.rs | 114 +++++++++++++++++- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs b/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs index ca8f221ddc..7580fdb38d 100644 --- a/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs +++ b/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs @@ -68,7 +68,7 @@ pub(super) fn rewrite_filter_for_join( expression.replacen("categories", "project_categories", 1); if is_negative_filter(expression) { clause.project.push(project_expression); - clause.version.push(expression.to_string()); + clause.version.push(version_filter_expression(expression)); Ok(vec![clause]) } else { Ok(vec![ @@ -78,13 +78,13 @@ pub(super) fn rewrite_filter_for_join( }, JoinedFilterClause { project: Vec::new(), - version: vec![expression.to_string()], + version: vec![version_filter_expression(expression)], }, ]) } } else { if is_version_filter_field(field) { - clause.version.push(expression.to_string()); + clause.version.push(version_filter_expression(expression)); } else { clause.project.push(expression.to_string()); } @@ -119,6 +119,16 @@ fn is_version_filter_field(field: &str) -> bool { }) } +fn version_filter_expression(expression: &str) -> String { + let Some((field, value)) = expression.split_once(':') else { + return expression.to_string(); + }; + let Some(value) = value.strip_prefix('=') else { + return expression.to_string(); + }; + format!("{field}:{value}") +} + fn is_negative_filter(expression: &str) -> bool { expression .split_once(':') @@ -403,7 +413,7 @@ mod tests { "versions", ) .unwrap(), - "(project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + "(project_categories:= fabric && $versions(game_versions: 1.21)) || $versions(categories: fabric && game_versions: 1.21)" ); } @@ -415,7 +425,7 @@ mod tests { "versions", ) .unwrap(), - "(license:= MIT && project_categories:= fabric) || (license:= MIT && $versions(categories:= fabric))" + "(license:= MIT && project_categories:= fabric) || (license:= MIT && $versions(categories: fabric))" ); } @@ -427,7 +437,7 @@ mod tests { "versions", ) .unwrap(), - "(license:= MIT && $versions(game_versions:= 1.21)) || (project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + "(license:= MIT && $versions(game_versions: 1.21)) || (project_categories:= fabric && $versions(game_versions: 1.21)) || $versions(categories: fabric && game_versions: 1.21)" ); } diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 28d628644c..d893b8e2c0 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -1,3 +1,113 @@ +//! Search implementation backed by a Typesense cluster. +//! +//! # Search behavior +//! +//! Users expect very specific behavior when searching for projects on the +//! platform. When making changes to search, you must ensure that the query +//! behavior is roughly the same as before your changes - that means test with +//! strings like `sodi`, `sodiu`, `sodium`, `sodium `, etc. and make sure the +//! same results still appear. +//! +//! Before submitting a document into Typesense, we normalize the project name +//! and author using [`normalize_for_search`]. This turns a name like `My Cool +//! Mod` into `my-cool-mod`, which lets us exactly define what we consider to +//! be word separators, and fine-tune search behavior. The normalized/indexed +//! name is never shown to users; it's just used for search ops. +//! +//! # Collections +//! +//! The approach we take is to have two collections in TS: a project and a +//! version collection, which we index separately, and then join together when +//! we query. Project documents contain project metadata like name, downloads, +//! summary, etc.; version documents contain game versions, compatible loaders, +//! etc. This lets us deduplicate large chunks of data (project descriptions), +//! but keep each version doc separate for accurate filtering. +//! +//! If you're wondering why we can't just put all the info required for +//! searching into the project document, it's because collating version info +//! into the project doc will break filtering on multiple fields. Let's say for +//! example we have a project with 2 versions: +//! - NeoForge 1.21.1 +//! - Fabric 1.20.0 +//! +//! If we put both of these versions into the project object, we'd end up with +//! a document like: +//! ```json +//! { +//! "name": "My Project", +//! "game_versions": ["1.21.1", "1.20.0"], +//! "loaders": ["neoforge", "fabric"] +//! } +//! ``` +//! +//! If a user now searches for a mod for NeoForge 1.20.0, this project will +//! match - when in reality it shouldn't, because it has no versions compatible +//! with NeoForge 1.20.0! Only either Neoforge 1.21.1, or Fabric 1.20.0. +//! Keeping the projects/versions document split retains this behavior. +//! +//! What if we put a `versions` object into the project document? This is +//! terrible because: +//! - it makes project documents massive +//! - it forces an entire project document to be rebuilt when one change to a +//! version is made +//! +//! # Performance +//! +//! Our collections are quite large for a platform of our size. To ensure +//! indexing and query performance, we do a few things. +//! +//! ## Indexing +//! +//! We use incremental indexing to update projects and version documents when +//! a user makes a change ASAP[^1]. When a project or version is updated, we +//! push a message "update this project ID/this version ID" into a Kafka topic; +//! the `incremental-index-search` task then consumes all the Kafka messages +//! and applies the actual indexing updates to the Typesense cluster, and +//! commits the Kafka messages. +//! +//! We do also have the `index-search` task which does a full reindex, but this +//! shouldn't be required in normal operation. +//! +//! When batching Typesense update operations, we batch by both project count +//! and document count. Since a project can have an unbounded number of +//! versions, we can (and will) have projects with thousands of versions. If we +//! were to send updates for just this project alone, we will be sending a lot +//! more data in a single HTTP request, which could timeout the operation. +//! +//! [^1]: Okay, not exactly ASAP. Once the Kafka consumer receives the first +//! message, it waits for a small configurable time to batch a bunch of changes +//! together, and updates Typesense in bulk with those. +//! +//! ## Querying +//! +//! Having two collections is cool and all, but joins over such a large dataset +//! causes queries to be a lot slower than if we're just querying one +//! collection. To speed this up, we use some tricks. +//! +//! ### Filtering on multiple version fields +//! +//! If you search for `categories = fabric AND game_versions = 1.21`, then +//! you're searching for *one* project with *both* these conditions set. +//! Therefore, this uses *one* join over versions, rather than two joins +//! each matching different versions of the same project. Using one join here +//! is faster than two. It's even faster if you search for more facets like +//! category, game version, loader, and environment. +//! +//! ### Whole-field tokenization +//! +//! For fields like a version's `categories`, `environment`, `game_versions` +//! etc. it makes no sense to do full-text search on this (unlike name and +//! author), since the full set of categories, environments, etc. is known +//! and finite. All of these values can be represented as whitespace-free, +//! atomic, single tokens in Typesense, if we set +//! `symbols_to_index = ["=", ".", "_"]`. This means that `fabric-api` is +//! treated as one token `fabric-api`, not as `fabric` and `api`. So if we use +//! the filter `categories:fabric-api`, it will match for that *one token* +//! `fabric-api` instead of full text search. +//! +//! For this, we use the `:` operator instead of `:=` to tell +//! Typesense to treat this as an exact token match. + use std::sync::LazyLock; use async_trait::async_trait; @@ -577,6 +687,8 @@ impl SearchField { } } +const VERSION_FILTER_SYMBOLS: &[&str] = &["-", ".", "_"]; + static TYPESENSE_SEARCH_FIELDS: LazyLock> = LazyLock::new(|| { use strum::IntoEnumIterator; @@ -661,7 +773,7 @@ impl Typesense { "type": spec.ty, "facet": spec.facet, "optional": spec.optional, - "token_separators": spec.token_separators, + "symbols_to_index": VERSION_FILTER_SYMBOLS, }) }) .collect::>();