wip: projects/versions collections

apps/labrinth/src/queue/server_ping.rs

@@ -1,9 +1,9 @@
 use crate::database::DBProject;
-use crate::database::models::{DBProjectId, DBVersionId};
+use crate::database::models::DBProjectId;
 use crate::database::redis::RedisPool;
 use crate::env::ENV;
 use crate::models::exp;
-use crate::models::ids::{ProjectId, VersionId};
+use crate::models::ids::ProjectId;
 use crate::models::projects::ProjectStatus;
 use crate::search::incremental::IncrementalSearchQueue;
 use crate::{database::PgPool, util::error::Context};
@@ -175,26 +175,14 @@ impl ServerPingQueue {
                 }
 
                 if updated_project {
-                    let version_ids = sqlx::query_scalar!(
-                        "SELECT id FROM versions WHERE mod_id = $1",
-                        DBProjectId::from(*project_id) as DBProjectId,
-                    )
-                    .fetch_all(&self.db)
-                    .await
-                    .wrap_err("failed to fetch project version IDs")?
-                    .into_iter()
-                    .map(|version_id| VersionId::from(DBVersionId(version_id)))
-                    .collect::<Vec<_>>();
-
                     let clear_cache = DBProject::clear_cache(
                         (*project_id).into(),
                         None,
                         None,
                         &self.redis,
                     );
-                    let queue_search = self
-                        .incremental_search_queue
-                        .push(*project_id, version_ids);
+                    let queue_search =
+                        self.incremental_search_queue.push(*project_id);
 
                     let (clear_cache_result, _) =
                         join(clear_cache, queue_search).await;

apps/labrinth/src/routes/internal/moderation/tech_review.rs

@@ -1161,7 +1161,6 @@ pub async fn submit_report(
 
     if verdict == DelphiVerdict::Unsafe {
         crate::routes::v3::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,

apps/labrinth/src/routes/v3/organizations.rs

@@ -810,7 +810,6 @@ pub async fn organization_delete(
 
     for project_id in organization_project_ids {
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,
@@ -980,7 +979,6 @@ pub async fn organization_projects_add(
         )
         .await?;
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_item.inner.id,
@@ -1173,7 +1171,6 @@ pub async fn organization_projects_remove(
         )
         .await?;
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_item.inner.id,

apps/labrinth/src/routes/v3/project_creation.rs

@@ -358,7 +358,6 @@ pub async fn project_create_internal(
     } else {
         transaction.commit().await?;
         super::projects::clear_project_cache_and_queue_search(
-            &client,
             &redis,
             &search_state,
             project_id.into(),
@@ -425,7 +424,6 @@ pub async fn project_create_with_id(
     } else {
         transaction.commit().await?;
         super::projects::clear_project_cache_and_queue_search(
-            &client,
             &redis,
             &search_state,
             project_id.into(),

apps/labrinth/src/routes/v3/project_creation/new.rs

@@ -345,7 +345,6 @@ pub async fn create(
         .wrap_internal_err("failed to commit transaction")?;
 
     super::super::projects::clear_project_cache_and_queue_search(
-        &db,
         &redis,
         &search_state,
         project_id.into(),

apps/labrinth/src/routes/v3/projects.rs

@@ -72,35 +72,42 @@ pub fn project_config(cfg: &mut actix_web::web::ServiceConfig) {
 }
 
 pub async fn clear_project_cache_and_queue_search(
-    pool: &PgPool,
     redis: &RedisPool,
     search_state: &SearchState,
     project_id: db_ids::DBProjectId,
     slug: Option<String>,
     clear_dependencies: Option<bool>,
 ) -> Result<(), ApiError> {
-    let version_ids = sqlx::query_scalar!(
-        "SELECT id FROM versions WHERE mod_id = $1",
-        project_id as db_ids::DBProjectId,
-    )
-    .fetch_all(pool)
-    .await
-    .wrap_internal_err("failed to fetch project version IDs")?
-    .into_iter()
-    .map(|version_id| VersionId::from(db_ids::DBVersionId(version_id)))
-    .collect::<Vec<_>>();
-
-    clear_project_cache_and_queue_search_versions(
+    clear_project_cache_and_queue_search_inner(
         redis,
         search_state,
         project_id,
         slug,
         clear_dependencies,
-        version_ids,
     )
     .await
 }
 
+pub async fn clear_project_cache_and_queue_search_inner(
+    redis: &RedisPool,
+    search_state: &SearchState,
+    project_id: db_ids::DBProjectId,
+    slug: Option<String>,
+    clear_dependencies: Option<bool>,
+) -> Result<(), ApiError> {
+    db_models::DBProject::clear_cache(
+        project_id,
+        slug,
+        clear_dependencies,
+        redis,
+    )
+    .await?;
+
+    search_state.queue.push(project_id.into()).await;
+
+    Ok(())
+}
+
 pub async fn clear_project_cache_and_queue_search_versions(
     redis: &RedisPool,
     search_state: &SearchState,
@@ -119,7 +126,7 @@ pub async fn clear_project_cache_and_queue_search_versions(
 
     search_state
         .queue
-        .push(project_id.into(), version_ids)
+        .push_versions(project_id.into(), version_ids)
         .await;
 
     Ok(())
@@ -1133,6 +1140,9 @@ pub async fn project_edit_internal(
         Ok(())
     }
 
+    let reindex_version_project_types =
+        new_project.minecraft_java_server.is_some();
+
     update(
         &mut transaction,
         id,
@@ -1202,15 +1212,26 @@ pub async fn project_edit_internal(
 
     transaction.commit().await?;
 
-    clear_project_cache_and_queue_search(
-        &pool,
-        &redis,
-        &search_state,
-        project_item.inner.id,
-        project_item.inner.slug,
-        None,
-    )
-    .await?;
+    if reindex_version_project_types {
+        clear_project_cache_and_queue_search_versions(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+            project_item.versions.iter().copied().map(VersionId::from),
+        )
+        .await?;
+    } else {
+        clear_project_cache_and_queue_search(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+        )
+        .await?;
+    }
 
     // Remove no longer searchable projects from search index
     if let (true, Some(false)) = (
@@ -1764,7 +1785,6 @@ pub async fn projects_edit(
 
     for (project_id, slug) in changed_projects {
         clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,
@@ -1996,7 +2016,6 @@ pub async fn project_icon_edit_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2116,7 +2135,6 @@ pub async fn delete_project_icon_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2317,7 +2335,6 @@ pub async fn add_gallery_item_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2536,7 +2553,6 @@ pub async fn edit_gallery_item_internal(
     transaction.commit().await?;
 
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2685,7 +2701,6 @@ pub async fn delete_gallery_item_internal(
     transaction.commit().await?;
 
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2834,6 +2849,13 @@ pub async fn project_delete_internal(
             &redis,
         )
         .await?;
+        search_state
+            .backend
+            .remove_project_version_documents(&[project.inner.id.into()])
+            .await
+            .wrap_internal_err(
+                "failed to remove project versions from search index",
+            )?;
         search_state
             .backend
             .remove_project_documents(&[project.inner.id.into()])

apps/labrinth/src/routes/v3/versions.rs

@@ -27,6 +27,7 @@ use crate::models::teams::ProjectPermissions;
 use crate::queue::file_scan::get_files_missing_attribution;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
+use crate::search::incremental::consume::reindex_project;
 use crate::search::{SearchBackend, SearchState};
 use crate::util::error::Context;
 use crate::util::img;
@@ -1257,9 +1258,17 @@ pub async fn version_delete(
     )
     .await?;
     search_backend
-        .remove_documents(&[version.inner.id.into()])
+        .remove_version_documents(&[version.inner.id.into()])
         .await
-        .wrap_internal_err("failed to remove documents")?;
+        .wrap_internal_err("failed to remove version search document")?;
+    reindex_project(
+        &pool,
+        &redis,
+        search_backend.as_ref(),
+        version.inner.project_id.into(),
+    )
+    .await
+    .wrap_internal_err("failed to reindex project")?;
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))
     } else {

apps/labrinth/src/search/backend/common.rs

@@ -50,14 +50,7 @@ pub enum SearchIndex {
     MinecraftJavaServerPlayersOnline,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SearchIndexName {
-    Projects,
-    ProjectsFiltered,
-}
-
 pub struct SearchSort {
-    pub index_name: SearchIndexName,
     pub index: SearchIndex,
 }
 
@@ -65,16 +58,12 @@ pub fn parse_search_index(
     index: &str,
     new_filters: Option<&str>,
 ) -> Result<SearchSort, ApiError> {
-    let projects_name = SearchIndexName::Projects;
-    let projects_filtered_name = SearchIndexName::ProjectsFiltered;
-
     // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
     let is_server = new_filters
         .is_some_and(|f| f.contains("project_types = minecraft_java_server"));
 
     Ok(match index {
         "relevance" => SearchSort {
-            index_name: projects_name,
             index: if is_server {
                 SearchIndex::MinecraftJavaServerVerifiedPlays2w
             } else {
@@ -82,27 +71,21 @@ pub fn parse_search_index(
             },
         },
         "downloads" => SearchSort {
-            index_name: projects_filtered_name,
             index: SearchIndex::Downloads,
         },
         "follows" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Follows,
         },
         "updated" | "date_modified" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Updated,
         },
         "newest" | "date_created" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Newest,
         },
         "minecraft_java_server.verified_plays_2w" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
         },
         "minecraft_java_server.ping.data.players_online" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerPlayersOnline,
         },
         i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),

apps/labrinth/src/search/backend/mod.rs

@@ -2,7 +2,7 @@ mod common;
 pub mod typesense;
 
 pub use common::{
-    ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
-    combined_search_filters, parse_search_index, parse_search_request,
+    ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters,
+    parse_search_index, parse_search_request,
 };
 pub use typesense::{Typesense, TypesenseConfig};

apps/labrinth/src/search/incremental.rs

@@ -36,7 +36,11 @@ impl IncrementalSearchQueue {
         }
     }
 
-    pub async fn push(
+    pub async fn push(&self, project_id: ProjectId) {
+        self.operations.lock().await.push_project_change(project_id);
+    }
+
+    pub async fn push_versions(
         &self,
         project_id: ProjectId,
         version_ids: impl IntoIterator<Item = VersionId>,
@@ -44,7 +48,7 @@ impl IncrementalSearchQueue {
         self.operations
             .lock()
             .await
-            .push_project_change(project_id, version_ids);
+            .push_version_change(project_id, version_ids);
     }
 
     pub async fn push_project_removal(&self, project_id: ProjectId) {
@@ -123,22 +127,27 @@ impl PendingSearchIndexOperations {
             && self.removed_project_ids.is_empty()
     }
 
-    fn push_project_change(
+    fn push_project_change(&mut self, project_id: ProjectId) {
+        if !self.removed_project_ids.contains(&project_id) {
+            self.changed_project_ids.insert(project_id);
+        }
+    }
+
+    fn push_version_change(
         &mut self,
         project_id: ProjectId,
         version_ids: impl IntoIterator<Item = VersionId>,
     ) {
-        if !self.removed_project_ids.contains(&project_id) {
-            let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
-            if version_ids.is_empty() {
-                self.changed_project_versions.remove(&project_id);
-                self.changed_project_ids.insert(project_id);
-            } else if !self.changed_project_ids.contains(&project_id) {
-                self.changed_project_versions
-                    .entry(project_id)
-                    .or_default()
-                    .extend(version_ids);
-            }
+        if self.removed_project_ids.contains(&project_id) {
+            return;
+        }
+
+        let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
+        if !version_ids.is_empty() {
+            self.changed_project_versions
+                .entry(project_id)
+                .or_default()
+                .extend(version_ids);
         }
     }
 
@@ -151,16 +160,12 @@ impl PendingSearchIndexOperations {
     fn push_event(&mut self, event: SearchProjectIndexQueueEventData) {
         match event {
             SearchProjectIndexQueueEventData::Change { project_id } => {
-                self.push_project_change(project_id, [])
+                self.push_project_change(project_id)
             }
             SearchProjectIndexQueueEventData::VersionChange {
                 project_id,
                 version_ids,
-            } => {
-                if !version_ids.is_empty() {
-                    self.push_project_change(project_id, version_ids)
-                }
-            }
+            } => self.push_version_change(project_id, version_ids),
             SearchProjectIndexQueueEventData::Removal { project_id } => {
                 self.push_project_removal(project_id)
             }
@@ -188,7 +193,6 @@ impl PendingSearchIndexOperations {
                 }
             },
         ));
-
         events
     }
 }

apps/labrinth/src/search/incremental/consume.rs

@@ -191,7 +191,9 @@ async fn consume_batch(
         .retain(|project_id| !project_ids_to_remove.contains(project_id));
     project_ids_with_version_changes
         .retain(|project_id| !project_ids_to_remove.contains(project_id));
-
+    project_ids_to_change.retain(|project_id| {
+        !project_ids_with_version_changes.contains(project_id)
+    });
     let project_ids_to_change =
         project_ids_to_change.into_iter().collect::<Vec<_>>();
     let project_ids_with_version_changes = project_ids_with_version_changes
@@ -204,7 +206,7 @@ async fn consume_batch(
 
     info!(
         kafka.message_count = messages_to_commit.len(),
-        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with version changes, {} versions to change, and {} projects to remove",
+        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove",
         start.elapsed(),
         project_ids_to_change.len(),
         project_ids_with_version_changes.len(),
@@ -219,6 +221,10 @@ async fn consume_batch(
             project_count = project_ids_to_remove.len(),
             "Removing project documents"
         );
+        search_backend
+            .remove_project_version_documents(&project_ids_to_remove)
+            .await
+            .wrap_err("failed to remove project version documents")?;
         search_backend
             .remove_project_documents(&project_ids_to_remove)
             .await
@@ -232,12 +238,8 @@ async fn consume_batch(
 
     if !version_ids_to_change.is_empty() {
         let operation_start = Instant::now();
-        info!(
-            version_count = version_ids_to_change.len(),
-            "Removing changed version documents",
-        );
         search_backend
-            .remove_documents(&version_ids_to_change)
+            .remove_version_documents(&version_ids_to_change)
             .await
             .wrap_err("failed to remove changed version documents")?;
         info!(
@@ -249,12 +251,7 @@ async fn consume_batch(
 
     if !project_ids_with_version_changes.is_empty() {
         let operation_start = Instant::now();
-        info!(
-            project_count = project_ids_with_version_changes.len(),
-            version_count = version_ids_to_change.len(),
-            "Indexing changed project versions"
-        );
-        index_changed_project_versions(
+        reindex_changed_project_versions(
             ro_pool,
             redis_pool,
             search_backend,
@@ -262,11 +259,11 @@ async fn consume_batch(
             &version_ids_to_change,
         )
         .await
-        .wrap_err("failed to index changed project version batch")?;
+        .wrap_err("failed to reindex changed project versions")?;
         info!(
             project_count = project_ids_with_version_changes.len(),
             version_count = version_ids_to_change.len(),
-            "Indexed changed project versions in {:.2?}",
+            "Reindexed changed project versions in {:.2?}",
             operation_start.elapsed()
         );
     }
@@ -275,19 +272,19 @@ async fn consume_batch(
         let operation_start = Instant::now();
         info!(
             project_count = project_ids_to_change.len(),
-            "Indexing changed projects"
+            "Reindexing changed projects"
         );
-        index_changed_projects(
+        reindex_projects(
             ro_pool,
             redis_pool,
             search_backend,
             &project_ids_to_change,
         )
         .await
-        .wrap_err("failed to index changed project batch")?;
+        .wrap_err("failed to reindex changed project batch")?;
         info!(
             project_count = project_ids_to_change.len(),
-            "Indexed changed projects in {:.2?}",
+            "Reindexed changed projects in {:.2?}",
             operation_start.elapsed()
         );
     }
@@ -356,7 +353,7 @@ async fn index_changed_projects(
     Ok(())
 }
 
-async fn index_changed_project_versions(
+async fn reindex_changed_project_versions(
     ro_pool: &PgPool,
     redis_pool: &RedisPool,
     search_backend: &dyn SearchBackend,
@@ -383,9 +380,11 @@ async fn index_changed_project_versions(
         )
     })?;
 
-    info!("Fetched all project version documents, indexing into backend");
-
-    search_backend.index_documents(&documents).await?;
+    search_backend.remove_project_documents(project_ids).await?;
+    search_backend.index_documents(&documents.projects).await?;
+    search_backend
+        .index_version_documents(&documents.versions)
+        .await?;
 
     Ok(())
 }

apps/labrinth/src/search/indexing.rs

@@ -24,7 +24,10 @@ use crate::models::ids::{ProjectId, VersionId};
 use crate::models::projects::{DependencyType, from_duplicate_version_fields};
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
-use crate::search::{SearchProjectDependency, UploadSearchProject};
+use crate::search::{
+    SearchDocumentBatch, SearchProjectDependency, UploadSearchProject,
+    UploadSearchVersion,
+};
 use crate::util::error::Context;
 
 struct PartialProject {
@@ -68,7 +71,7 @@ pub async fn index_local(
     redis: &RedisPool,
     cursor: i64,
     limit: i64,
-) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
+) -> eyre::Result<(SearchDocumentBatch, i64)> {
     info!("Indexing local projects!");
 
     let searchable_statuses = searchable_statuses();
@@ -111,12 +114,11 @@ pub async fn index_local(
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let Some(largest) = project_ids.iter().max() else {
-        return Ok((vec![], i64::MAX));
+        return Ok((SearchDocumentBatch::default(), i64::MAX));
     };
 
-    let uploads =
-        build_search_documents(pool, redis, db_projects, None).await?;
-    Ok((uploads, *largest))
+    let documents = build_search_documents(pool, redis, db_projects).await?;
+    Ok((documents, *largest))
 }
 
 pub async fn index_project_documents(
@@ -164,7 +166,9 @@ pub async fn index_project_documents(
 
     info!("Fetched partial projects");
 
-    build_search_documents(pool, redis, db_projects, None).await
+    Ok(build_search_documents(pool, redis, db_projects)
+        .await?
+        .projects)
 }
 
 pub async fn index_project_version_documents(
@@ -172,16 +176,33 @@ pub async fn index_project_version_documents(
     redis: &RedisPool,
     project_ids: &[ProjectId],
     version_ids: &[VersionId],
-) -> eyre::Result<Vec<UploadSearchProject>> {
+) -> eyre::Result<SearchDocumentBatch> {
+    let projects =
+        index_project_document_batch(pool, redis, project_ids).await?;
+    let version_ids = version_ids
+        .iter()
+        .map(ToString::to_string)
+        .collect::<HashSet<_>>();
+    Ok(SearchDocumentBatch {
+        projects: projects.projects,
+        versions: projects
+            .versions
+            .into_iter()
+            .filter(|version| version_ids.contains(&version.version_id))
+            .collect(),
+    })
+}
+
+async fn index_project_document_batch(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = project_ids
         .iter()
         .map(|project_id| DBProjectId::from(*project_id).0)
         .collect::<Vec<_>>();
-    let version_ids = version_ids
-        .iter()
-        .map(|version_id| DBVersionId::from(*version_id))
-        .collect::<HashSet<_>>();
 
     let db_projects = sqlx::query!(
         r#"
@@ -215,15 +236,14 @@ pub async fn index_project_version_documents(
     .await
     .wrap_err("failed to fetch project")?;
 
-    build_search_documents(pool, redis, db_projects, Some(&version_ids)).await
+    build_search_documents(pool, redis, db_projects).await
 }
 
 async fn build_search_documents(
     pool: &PgPool,
     redis: &RedisPool,
     db_projects: Vec<PartialProject>,
-    version_ids_to_index: Option<&HashSet<DBVersionId>>,
-) -> eyre::Result<Vec<UploadSearchProject>> {
+) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let project_components = db_projects
@@ -391,7 +411,7 @@ async fn build_search_documents(
     .await?;
 
     info!("Getting all loader fields!");
-    let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
+    let loader_field_definitions: Vec<QueryLoaderField> = sqlx::query!(
         "
         SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional
         FROM loader_fields lf
@@ -409,7 +429,8 @@ async fn build_search_documents(
     })
     .try_collect()
     .await?;
-    let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect();
+    let loader_field_definitions: Vec<&QueryLoaderField> =
+        loader_field_definitions.iter().collect();
 
     info!("Getting all loader field enum values!");
 
@@ -434,7 +455,8 @@ async fn build_search_documents(
         .await?;
 
     info!("Indexing loaders, project types!");
-    let mut uploads = Vec::new();
+    let mut project_uploads = Vec::new();
+    let mut version_uploads = Vec::new();
 
     let total_len = db_projects.len();
     let mut count = 0;
@@ -533,21 +555,34 @@ async fn build_search_documents(
             .collect::<Vec<_>>();
 
         if let Some(versions) = versions.remove(&project.id) {
-            // Aggregated project loader fields
+            let Some(latest_version) = versions.iter().max_by(|a, b| {
+                a.date_published
+                    .cmp(&b.date_published)
+                    .then_with(|| a.id.0.cmp(&b.id.0))
+            }) else {
+                continue;
+            };
+
             let project_version_fields = versions
                 .iter()
                 .flat_map(|x| x.version_fields.clone())
                 .collect::<Vec<_>>();
             let aggregated_version_fields = VersionField::from_query_json(
                 project_version_fields,
-                &loader_fields,
+                &loader_field_definitions,
                 &loader_field_enum_values,
                 true,
             );
-            let project_loader_fields =
+            let unvectorized_loader_fields = aggregated_version_fields
+                .iter()
+                .map(|field| {
+                    (field.field_name.clone(), field.value.serialize_internal())
+                })
+                .collect();
+            let mut loader_fields =
                 from_duplicate_version_fields(aggregated_version_fields);
+            let project_loader_fields = loader_fields.clone();
 
-            // aggregated project loaders
             let mut project_loaders = versions
                 .iter()
                 .flat_map(|x| x.loaders.clone())
@@ -555,162 +590,184 @@ async fn build_search_documents(
             project_loaders.sort();
             project_loaders.dedup();
 
-            // all valid project types across every version of the project, so that
-            // filters can exclude projects that have *any* version of a given
-            // project type (unlike the version-specific `project_types` field).
-            let mut all_project_types = versions
+            let mut project_types = versions
                 .iter()
                 .flat_map(|x| x.project_types.clone())
                 .collect::<Vec<_>>();
-            all_project_types.sort();
-            all_project_types.dedup();
+            project_types.sort();
+            project_types.dedup();
             exp::compat::correct_project_types(
                 &project.components,
-                &mut all_project_types,
+                &mut project_types,
             );
 
-            for version in versions {
-                if let Some(version_ids_to_index) = version_ids_to_index
-                    && !version_ids_to_index.contains(&version.id)
-                {
-                    continue;
-                }
-
+            let project_id = ProjectId::from(project.id).to_string();
+            version_uploads.extend(versions.iter().map(|version| {
                 let version_fields = VersionField::from_query_json(
-                    version.version_fields,
-                    &loader_fields,
+                    version.version_fields.clone(),
+                    &loader_field_definitions,
                     &loader_field_enum_values,
                     false,
                 );
                 let unvectorized_loader_fields = version_fields
                     .iter()
-                    .map(|vf| {
-                        (vf.field_name.clone(), vf.value.serialize_internal())
+                    .map(|field| {
+                        (
+                            field.field_name.clone(),
+                            field.value.serialize_internal(),
+                        )
                     })
                     .collect();
-                let mut loader_fields =
-                    from_duplicate_version_fields(version_fields);
-                let mut project_types = version.project_types;
-
+                let mut fields = from_duplicate_version_fields(version_fields);
+                let mut version_project_types = version.project_types.clone();
                 exp::compat::correct_project_types(
                     &project.components,
-                    &mut project_types,
+                    &mut version_project_types,
                 );
 
-                let mut version_loaders = version.loaders;
-
-                // Uses version loaders, not project loaders.
-                let mut categories = categories.clone();
-                categories.append(&mut version_loaders.clone());
-
-                let display_categories = display_categories.clone();
-                categories.append(&mut version_loaders);
-
-                // SPECIAL BEHAVIOUR
-                // Todo: revisit.
-                // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category.
-                // These were previously considered the loader, and in v2, the loader is a category for searching.
-                // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories.
-                // The loaders are kept in loader_fields as well, so that no information is lost on retrieval.
-                let mrpack_loaders = loader_fields
+                let mut version_categories = version.loaders.clone();
+                let mrpack_loaders = fields
                     .get("mrpack_loaders")
-                    .cloned()
-                    .map(|x| {
-                        x.into_iter()
-                            .filter_map(|x| x.as_str().map(String::from))
-                            .collect::<Vec<_>>()
-                    })
-                    .unwrap_or_default();
-                categories.extend(mrpack_loaders);
-                if loader_fields.contains_key("mrpack_loaders") {
-                    categories.retain(|x| *x != "mrpack");
+                    .into_iter()
+                    .flatten()
+                    .filter_map(|value| value.as_str().map(String::from))
+                    .collect::<Vec<_>>();
+                version_categories.extend(mrpack_loaders);
+                if fields.contains_key("mrpack_loaders") {
+                    version_categories.retain(|category| category != "mrpack");
                 }
+                version_categories.sort();
+                version_categories.dedup();
 
-                // SPECIAL BEHAVIOUR:
-                // For consistency with v2 searching, we manually input the
-                // client_side and server_side fields from the loader fields into
-                // separate loader fields.
-                // 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
                 let (_, v2_og_project_type) =
-                    LegacyProject::get_project_type(&project_types);
+                    LegacyProject::get_project_type(&version_project_types);
                 let (client_side, server_side) =
                     v2_reroute::convert_v3_side_types_to_v2_side_types(
                         &unvectorized_loader_fields,
                         Some(&v2_og_project_type),
                     );
-
                 if let Ok(client_side) = serde_json::to_value(client_side) {
-                    loader_fields
-                        .insert("client_side".to_string(), vec![client_side]);
+                    fields.insert("client_side".to_string(), vec![client_side]);
                 }
                 if let Ok(server_side) = serde_json::to_value(server_side) {
-                    loader_fields
-                        .insert("server_side".to_string(), vec![server_side]);
+                    fields.insert("server_side".to_string(), vec![server_side]);
                 }
-
-                let components = project
-                    .components
-                    .clone()
-                    .into_query(
-                        ProjectId::from(project.id),
-                        &project_query_context,
+                fields.retain(|field, _| {
+                    matches!(
+                        field.as_str(),
+                        "environment"
+                            | "game_versions"
+                            | "client_side"
+                            | "server_side"
                     )
-                    .wrap_err("failed to populate query components")?;
+                });
 
-                let usp = UploadSearchProject {
-                    version_id: crate::models::ids::VersionId::from(version.id)
-                        .to_string(),
-                    project_id: crate::models::ids::ProjectId::from(project.id)
-                        .to_string(),
-                    name: project.name.clone(),
-                    indexed_name: normalize_for_search(&project.name),
-                    summary: project.summary.clone(),
-                    categories: categories.clone(),
-                    display_categories: display_categories.clone(),
-                    follows: project.follows,
-                    downloads: project.downloads,
-                    log_downloads: (project.downloads.max(1) as f64).ln(),
-                    icon_url: project.icon_url.clone(),
-                    author: username.clone(),
-                    author_id: ariadne::ids::UserId::from(user_id).to_string(),
-                    organization: org_name.clone(),
-                    organization_id: org_id.map(|e| {
-                        crate::models::ids::OrganizationId::from(e).to_string()
-                    }),
-                    indexed_author: normalize_for_search(&username),
-                    date_created: project.approved,
-                    created_timestamp: project.approved.timestamp(),
-                    date_modified: project.updated,
-                    modified_timestamp: project.updated.timestamp(),
+                UploadSearchVersion {
+                    version_id: VersionId::from(version.id).to_string(),
+                    project_id: project_id.clone(),
+                    categories: version_categories,
+                    project_types: version_project_types,
                     version_published_timestamp: version
                         .date_published
                         .timestamp(),
-                    license: license.clone(),
-                    slug: project.slug.clone(),
-                    // TODO
-                    project_types,
-                    all_project_types: all_project_types.clone(),
-                    gallery: gallery.clone(),
-                    featured_gallery: featured_gallery.clone(),
-                    open_source,
-                    color: project.color.map(|x| x as u32),
-                    dependency_project_ids: dependency_project_ids.clone(),
-                    compatible_dependency_project_ids:
-                        compatible_dependency_project_ids.clone(),
-                    dependencies: dependencies.clone(),
-                    loader_fields,
-                    project_loader_fields: project_loader_fields.clone(),
-                    // 'loaders' is aggregate of all versions' loaders
-                    loaders: project_loaders.clone(),
-                    components,
-                };
+                    loader_fields: fields,
+                }
+            }));
 
-                uploads.push(usp);
+            let project_categories = categories.clone();
+            let mut categories = categories;
+            categories.extend(project_loaders.iter().cloned());
+
+            let mrpack_loaders = loader_fields
+                .get("mrpack_loaders")
+                .into_iter()
+                .flatten()
+                .filter_map(|value| value.as_str().map(String::from))
+                .collect::<Vec<_>>();
+            categories.extend(mrpack_loaders);
+            if loader_fields.contains_key("mrpack_loaders") {
+                categories.retain(|category| category != "mrpack");
             }
+            categories.sort();
+            categories.dedup();
+
+            let (_, v2_og_project_type) =
+                LegacyProject::get_project_type(&project_types);
+            let (client_side, server_side) =
+                v2_reroute::convert_v3_side_types_to_v2_side_types(
+                    &unvectorized_loader_fields,
+                    Some(&v2_og_project_type),
+                );
+
+            if let Ok(client_side) = serde_json::to_value(client_side) {
+                loader_fields
+                    .insert("client_side".to_string(), vec![client_side]);
+            }
+            if let Ok(server_side) = serde_json::to_value(server_side) {
+                loader_fields
+                    .insert("server_side".to_string(), vec![server_side]);
+            }
+
+            let components = project
+                .components
+                .clone()
+                .into_query(ProjectId::from(project.id), &project_query_context)
+                .wrap_err("failed to populate query components")?;
+            let indexed_name = normalize_for_search(&project.name);
+
+            project_uploads.push(UploadSearchProject {
+                version_id: crate::models::ids::VersionId::from(
+                    latest_version.id,
+                )
+                .to_string(),
+                project_id,
+                name: project.name,
+                indexed_name,
+                summary: project.summary,
+                categories,
+                project_categories,
+                display_categories,
+                follows: project.follows,
+                downloads: project.downloads,
+                log_downloads: (project.downloads.max(1) as f64).ln(),
+                icon_url: project.icon_url,
+                author: username.clone(),
+                author_id: ariadne::ids::UserId::from(user_id).to_string(),
+                organization: org_name,
+                organization_id: org_id.map(|id| {
+                    crate::models::ids::OrganizationId::from(id).to_string()
+                }),
+                indexed_author: normalize_for_search(&username),
+                date_created: project.approved,
+                created_timestamp: project.approved.timestamp(),
+                date_modified: project.updated,
+                modified_timestamp: project.updated.timestamp(),
+                version_published_timestamp: latest_version
+                    .date_published
+                    .timestamp(),
+                license,
+                slug: project.slug,
+                project_types: project_types.clone(),
+                all_project_types: project_types,
+                gallery,
+                featured_gallery,
+                open_source,
+                color: project.color.map(|x| x as u32),
+                dependency_project_ids,
+                compatible_dependency_project_ids,
+                dependencies,
+                project_loader_fields,
+                loader_fields,
+                loaders: project_loaders,
+                components,
+            });
         }
     }
 
-    Ok(uploads)
+    Ok(SearchDocumentBatch {
+        projects: project_uploads,
+        versions: version_uploads,
+    })
 }
 
 struct PartialVersion {

apps/labrinth/src/search/mod.rs

@@ -116,12 +116,25 @@ pub trait SearchBackend: Send + Sync {
         documents: &[UploadSearchProject],
     ) -> eyre::Result<()>;
 
+    async fn index_version_documents(
+        &self,
+        documents: &[UploadSearchVersion],
+    ) -> eyre::Result<()>;
+
     async fn remove_project_documents(
         &self,
         ids: &[ProjectId],
     ) -> eyre::Result<()>;
 
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
+    async fn remove_project_version_documents(
+        &self,
+        ids: &[ProjectId],
+    ) -> eyre::Result<()>;
+
+    async fn remove_version_documents(
+        &self,
+        ids: &[VersionId],
+    ) -> eyre::Result<()>;
 
     async fn tasks(&self) -> eyre::Result<Value>;
 
@@ -238,6 +251,7 @@ impl FromStr for SearchBackendKind {
 /// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     //
@@ -256,6 +270,7 @@ pub struct UploadSearchProject {
     pub indexed_name: String,
     pub summary: String,
     pub categories: Vec<String>,
+    pub project_categories: Vec<String>,
     pub display_categories: Vec<String>,
     pub follows: i32,
     pub downloads: i32,
@@ -274,7 +289,7 @@ pub struct UploadSearchProject {
     pub date_modified: DateTime<Utc>,
     /// Unix timestamp of the last major modification
     pub modified_timestamp: i64,
-    /// Unix timestamp of the publication date of the version
+    /// Unix timestamp of the most recently published version.
     pub version_published_timestamp: i64,
     pub open_source: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -296,6 +311,23 @@ pub struct UploadSearchProject {
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct UploadSearchVersion {
+    pub version_id: String,
+    pub project_id: String,
+    pub categories: Vec<String>,
+    pub project_types: Vec<String>,
+    pub version_published_timestamp: i64,
+    #[serde(flatten)]
+    pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
+}
+
+#[derive(Debug, Default)]
+pub struct SearchDocumentBatch {
+    pub projects: Vec<UploadSearchProject>,
+    pub versions: Vec<UploadSearchVersion>,
+}
+
 /// Nullable fields in Typesense-bound documents should use
 /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
 /// serialized as `null`.
@@ -320,6 +352,7 @@ pub struct SearchResults {
 
 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct ResultSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     pub project_types: Vec<String>,