wip: projects/versions collections

apps/labrinth/src/queue/server_ping.rs

@@ -1,9 +1,9 @@
 use crate::database::DBProject;
-use crate::database::models::{DBProjectId, DBVersionId};
+use crate::database::models::DBProjectId;
 use crate::database::redis::RedisPool;
 use crate::env::ENV;
 use crate::models::exp;
-use crate::models::ids::{ProjectId, VersionId};
+use crate::models::ids::ProjectId;
 use crate::models::projects::ProjectStatus;
 use crate::search::incremental::IncrementalSearchQueue;
 use crate::{database::PgPool, util::error::Context};
@@ -175,26 +175,14 @@ impl ServerPingQueue {
                 }
 
                 if updated_project {
-                    let version_ids = sqlx::query_scalar!(
-                        "SELECT id FROM versions WHERE mod_id = $1",
-                        DBProjectId::from(*project_id) as DBProjectId,
-                    )
-                    .fetch_all(&self.db)
-                    .await
-                    .wrap_err("failed to fetch project version IDs")?
-                    .into_iter()
-                    .map(|version_id| VersionId::from(DBVersionId(version_id)))
-                    .collect::<Vec<_>>();
-
                     let clear_cache = DBProject::clear_cache(
                         (*project_id).into(),
                         None,
                         None,
                         &self.redis,
                     );
-                    let queue_search = self
+                    let queue_search =
-                        .incremental_search_queue
+                        self.incremental_search_queue.push(*project_id);
-                        .push(*project_id, version_ids);
 
                     let (clear_cache_result, _) =
                         join(clear_cache, queue_search).await;

apps/labrinth/src/routes/internal/moderation/tech_review.rs

@@ -1161,7 +1161,6 @@ pub async fn submit_report(
 
     if verdict == DelphiVerdict::Unsafe {
         crate::routes::v3::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,

apps/labrinth/src/routes/v3/organizations.rs

@@ -810,7 +810,6 @@ pub async fn organization_delete(
 
     for project_id in organization_project_ids {
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,
@@ -980,7 +979,6 @@ pub async fn organization_projects_add(
         )
         .await?;
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_item.inner.id,
@@ -1173,7 +1171,6 @@ pub async fn organization_projects_remove(
         )
         .await?;
         super::projects::clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_item.inner.id,

apps/labrinth/src/routes/v3/project_creation.rs

@@ -358,7 +358,6 @@ pub async fn project_create_internal(
     } else {
         transaction.commit().await?;
         super::projects::clear_project_cache_and_queue_search(
-            &client,
             &redis,
             &search_state,
             project_id.into(),
@@ -425,7 +424,6 @@ pub async fn project_create_with_id(
     } else {
         transaction.commit().await?;
         super::projects::clear_project_cache_and_queue_search(
-            &client,
             &redis,
             &search_state,
             project_id.into(),

apps/labrinth/src/routes/v3/project_creation/new.rs

@@ -345,7 +345,6 @@ pub async fn create(
         .wrap_internal_err("failed to commit transaction")?;
 
     super::super::projects::clear_project_cache_and_queue_search(
-        &db,
         &redis,
         &search_state,
         project_id.into(),

apps/labrinth/src/routes/v3/projects.rs

@@ -72,35 +72,42 @@ pub fn project_config(cfg: &mut actix_web::web::ServiceConfig) {
 }
 
 pub async fn clear_project_cache_and_queue_search(
-    pool: &PgPool,
     redis: &RedisPool,
     search_state: &SearchState,
     project_id: db_ids::DBProjectId,
     slug: Option<String>,
     clear_dependencies: Option<bool>,
 ) -> Result<(), ApiError> {
-    let version_ids = sqlx::query_scalar!(
+    clear_project_cache_and_queue_search_inner(
-        "SELECT id FROM versions WHERE mod_id = $1",
-        project_id as db_ids::DBProjectId,
-    )
-    .fetch_all(pool)
-    .await
-    .wrap_internal_err("failed to fetch project version IDs")?
-    .into_iter()
-    .map(|version_id| VersionId::from(db_ids::DBVersionId(version_id)))
-    .collect::<Vec<_>>();
-
-    clear_project_cache_and_queue_search_versions(
         redis,
         search_state,
         project_id,
         slug,
         clear_dependencies,
-        version_ids,
     )
     .await
 }
 
+pub async fn clear_project_cache_and_queue_search_inner(
+    redis: &RedisPool,
+    search_state: &SearchState,
+    project_id: db_ids::DBProjectId,
+    slug: Option<String>,
+    clear_dependencies: Option<bool>,
+) -> Result<(), ApiError> {
+    db_models::DBProject::clear_cache(
+        project_id,
+        slug,
+        clear_dependencies,
+        redis,
+    )
+    .await?;
+
+    search_state.queue.push(project_id.into()).await;
+
+    Ok(())
+}
+
 pub async fn clear_project_cache_and_queue_search_versions(
     redis: &RedisPool,
     search_state: &SearchState,
@@ -119,7 +126,7 @@ pub async fn clear_project_cache_and_queue_search_versions(
 
     search_state
         .queue
-        .push(project_id.into(), version_ids)
+        .push_versions(project_id.into(), version_ids)
         .await;
 
     Ok(())
@@ -1133,6 +1140,9 @@ pub async fn project_edit_internal(
         Ok(())
     }
 
+    let reindex_version_project_types =
+        new_project.minecraft_java_server.is_some();
+
     update(
         &mut transaction,
         id,
@@ -1202,15 +1212,26 @@ pub async fn project_edit_internal(
 
     transaction.commit().await?;
 
-    clear_project_cache_and_queue_search(
+    if reindex_version_project_types {
-        &pool,
+        clear_project_cache_and_queue_search_versions(
-        &redis,
+            &redis,
-        &search_state,
+            &search_state,
-        project_item.inner.id,
+            project_item.inner.id,
-        project_item.inner.slug,
+            project_item.inner.slug,
-        None,
+            None,
-    )
+            project_item.versions.iter().copied().map(VersionId::from),
-    .await?;
+        )
+        .await?;
+    } else {
+        clear_project_cache_and_queue_search(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+        )
+        .await?;
+    }
 
     // Remove no longer searchable projects from search index
     if let (true, Some(false)) = (
@@ -1764,7 +1785,6 @@ pub async fn projects_edit(
 
     for (project_id, slug) in changed_projects {
         clear_project_cache_and_queue_search(
-            &pool,
             &redis,
             &search_state,
             project_id,
@@ -1996,7 +2016,6 @@ pub async fn project_icon_edit_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2116,7 +2135,6 @@ pub async fn delete_project_icon_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2317,7 +2335,6 @@ pub async fn add_gallery_item_internal(
 
     transaction.commit().await?;
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2536,7 +2553,6 @@ pub async fn edit_gallery_item_internal(
     transaction.commit().await?;
 
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2685,7 +2701,6 @@ pub async fn delete_gallery_item_internal(
     transaction.commit().await?;
 
     clear_project_cache_and_queue_search(
-        &pool,
         &redis,
         &search_state,
         project_item.inner.id,
@@ -2834,6 +2849,13 @@ pub async fn project_delete_internal(
             &redis,
         )
         .await?;
+        search_state
+            .backend
+            .remove_project_version_documents(&[project.inner.id.into()])
+            .await
+            .wrap_internal_err(
+                "failed to remove project versions from search index",
+            )?;
         search_state
             .backend
             .remove_project_documents(&[project.inner.id.into()])

apps/labrinth/src/routes/v3/versions.rs

@@ -27,6 +27,7 @@ use crate::models::teams::ProjectPermissions;
 use crate::queue::file_scan::get_files_missing_attribution;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
+use crate::search::incremental::consume::reindex_project;
 use crate::search::{SearchBackend, SearchState};
 use crate::util::error::Context;
 use crate::util::img;
@@ -1257,9 +1258,17 @@ pub async fn version_delete(
     )
     .await?;
     search_backend
-        .remove_documents(&[version.inner.id.into()])
+        .remove_version_documents(&[version.inner.id.into()])
         .await
-        .wrap_internal_err("failed to remove documents")?;
+        .wrap_internal_err("failed to remove version search document")?;
+    reindex_project(
+        &pool,
+        &redis,
+        search_backend.as_ref(),
+        version.inner.project_id.into(),
+    )
+    .await
+    .wrap_internal_err("failed to reindex project")?;
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))
     } else {

apps/labrinth/src/search/backend/common.rs

@@ -50,14 +50,7 @@ pub enum SearchIndex {
     MinecraftJavaServerPlayersOnline,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SearchIndexName {
-    Projects,
-    ProjectsFiltered,
-}
-
 pub struct SearchSort {
-    pub index_name: SearchIndexName,
     pub index: SearchIndex,
 }
 
@@ -65,16 +58,12 @@ pub fn parse_search_index(
     index: &str,
     new_filters: Option<&str>,
 ) -> Result<SearchSort, ApiError> {
-    let projects_name = SearchIndexName::Projects;
-    let projects_filtered_name = SearchIndexName::ProjectsFiltered;
-
     // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
     let is_server = new_filters
         .is_some_and(|f| f.contains("project_types = minecraft_java_server"));
 
     Ok(match index {
         "relevance" => SearchSort {
-            index_name: projects_name,
             index: if is_server {
                 SearchIndex::MinecraftJavaServerVerifiedPlays2w
             } else {
@@ -82,27 +71,21 @@ pub fn parse_search_index(
             },
         },
         "downloads" => SearchSort {
-            index_name: projects_filtered_name,
             index: SearchIndex::Downloads,
         },
         "follows" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Follows,
         },
         "updated" | "date_modified" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Updated,
         },
         "newest" | "date_created" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::Newest,
         },
         "minecraft_java_server.verified_plays_2w" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
         },
         "minecraft_java_server.ping.data.players_online" => SearchSort {
-            index_name: projects_name,
             index: SearchIndex::MinecraftJavaServerPlayersOnline,
         },
         i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),

apps/labrinth/src/search/backend/mod.rs

@@ -2,7 +2,7 @@ mod common;
 pub mod typesense;
 
 pub use common::{
-    ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
+    ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters,
-    combined_search_filters, parse_search_index, parse_search_request,
+    parse_search_index, parse_search_request,
 };
 pub use typesense::{Typesense, TypesenseConfig};

apps/labrinth/src/search/incremental.rs

@@ -36,7 +36,11 @@ impl IncrementalSearchQueue {
         }
     }
 
-    pub async fn push(
+    pub async fn push(&self, project_id: ProjectId) {
+        self.operations.lock().await.push_project_change(project_id);
+    }
+
+    pub async fn push_versions(
         &self,
         project_id: ProjectId,
         version_ids: impl IntoIterator<Item = VersionId>,
@@ -44,7 +48,7 @@ impl IncrementalSearchQueue {
         self.operations
             .lock()
             .await
-            .push_project_change(project_id, version_ids);
+            .push_version_change(project_id, version_ids);
     }
 
     pub async fn push_project_removal(&self, project_id: ProjectId) {
@@ -123,22 +127,27 @@ impl PendingSearchIndexOperations {
             && self.removed_project_ids.is_empty()
     }
 
-    fn push_project_change(
+    fn push_project_change(&mut self, project_id: ProjectId) {
+        if !self.removed_project_ids.contains(&project_id) {
+            self.changed_project_ids.insert(project_id);
+        }
+    }
+
+    fn push_version_change(
         &mut self,
         project_id: ProjectId,
         version_ids: impl IntoIterator<Item = VersionId>,
     ) {
-        if !self.removed_project_ids.contains(&project_id) {
+        if self.removed_project_ids.contains(&project_id) {
-            let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
+            return;
-            if version_ids.is_empty() {
+        }
-                self.changed_project_versions.remove(&project_id);
+
-                self.changed_project_ids.insert(project_id);
+        let version_ids = version_ids.into_iter().collect::<HashSet<_>>();
-            } else if !self.changed_project_ids.contains(&project_id) {
+        if !version_ids.is_empty() {
-                self.changed_project_versions
+            self.changed_project_versions
-                    .entry(project_id)
+                .entry(project_id)
-                    .or_default()
+                .or_default()
-                    .extend(version_ids);
+                .extend(version_ids);
-            }
         }
     }
 
@@ -151,16 +160,12 @@ impl PendingSearchIndexOperations {
     fn push_event(&mut self, event: SearchProjectIndexQueueEventData) {
         match event {
             SearchProjectIndexQueueEventData::Change { project_id } => {
-                self.push_project_change(project_id, [])
+                self.push_project_change(project_id)
             }
             SearchProjectIndexQueueEventData::VersionChange {
                 project_id,
                 version_ids,
-            } => {
+            } => self.push_version_change(project_id, version_ids),
-                if !version_ids.is_empty() {
-                    self.push_project_change(project_id, version_ids)
-                }
-            }
             SearchProjectIndexQueueEventData::Removal { project_id } => {
                 self.push_project_removal(project_id)
             }
@@ -188,7 +193,6 @@ impl PendingSearchIndexOperations {
                 }
             },
         ));
-
         events
     }
 }

apps/labrinth/src/search/incremental/consume.rs

@@ -191,7 +191,9 @@ async fn consume_batch(
         .retain(|project_id| !project_ids_to_remove.contains(project_id));
     project_ids_with_version_changes
         .retain(|project_id| !project_ids_to_remove.contains(project_id));
-
+    project_ids_to_change.retain(|project_id| {
+        !project_ids_with_version_changes.contains(project_id)
+    });
     let project_ids_to_change =
         project_ids_to_change.into_iter().collect::<Vec<_>>();
     let project_ids_with_version_changes = project_ids_with_version_changes
@@ -204,7 +206,7 @@ async fn consume_batch(
 
     info!(
         kafka.message_count = messages_to_commit.len(),
-        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with version changes, {} versions to change, and {} projects to remove",
+        "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove",
         start.elapsed(),
         project_ids_to_change.len(),
         project_ids_with_version_changes.len(),
@@ -219,6 +221,10 @@ async fn consume_batch(
             project_count = project_ids_to_remove.len(),
             "Removing project documents"
         );
+        search_backend
+            .remove_project_version_documents(&project_ids_to_remove)
+            .await
+            .wrap_err("failed to remove project version documents")?;
         search_backend
             .remove_project_documents(&project_ids_to_remove)
             .await
@@ -232,12 +238,8 @@ async fn consume_batch(
 
     if !version_ids_to_change.is_empty() {
         let operation_start = Instant::now();
-        info!(
-            version_count = version_ids_to_change.len(),
-            "Removing changed version documents",
-        );
         search_backend
-            .remove_documents(&version_ids_to_change)
+            .remove_version_documents(&version_ids_to_change)
             .await
             .wrap_err("failed to remove changed version documents")?;
         info!(
@@ -249,12 +251,7 @@ async fn consume_batch(
 
     if !project_ids_with_version_changes.is_empty() {
         let operation_start = Instant::now();
-        info!(
+        reindex_changed_project_versions(
-            project_count = project_ids_with_version_changes.len(),
-            version_count = version_ids_to_change.len(),
-            "Indexing changed project versions"
-        );
-        index_changed_project_versions(
             ro_pool,
             redis_pool,
             search_backend,
@@ -262,11 +259,11 @@ async fn consume_batch(
             &version_ids_to_change,
         )
         .await
-        .wrap_err("failed to index changed project version batch")?;
+        .wrap_err("failed to reindex changed project versions")?;
         info!(
             project_count = project_ids_with_version_changes.len(),
             version_count = version_ids_to_change.len(),
-            "Indexed changed project versions in {:.2?}",
+            "Reindexed changed project versions in {:.2?}",
             operation_start.elapsed()
         );
     }
@@ -275,19 +272,19 @@ async fn consume_batch(
         let operation_start = Instant::now();
         info!(
             project_count = project_ids_to_change.len(),
-            "Indexing changed projects"
+            "Reindexing changed projects"
         );
-        index_changed_projects(
+        reindex_projects(
             ro_pool,
             redis_pool,
             search_backend,
             &project_ids_to_change,
         )
         .await
-        .wrap_err("failed to index changed project batch")?;
+        .wrap_err("failed to reindex changed project batch")?;
         info!(
             project_count = project_ids_to_change.len(),
-            "Indexed changed projects in {:.2?}",
+            "Reindexed changed projects in {:.2?}",
             operation_start.elapsed()
         );
     }
@@ -356,7 +353,7 @@ async fn index_changed_projects(
     Ok(())
 }
 
-async fn index_changed_project_versions(
+async fn reindex_changed_project_versions(
     ro_pool: &PgPool,
     redis_pool: &RedisPool,
     search_backend: &dyn SearchBackend,
@@ -383,9 +380,11 @@ async fn index_changed_project_versions(
         )
     })?;
 
-    info!("Fetched all project version documents, indexing into backend");
+    search_backend.remove_project_documents(project_ids).await?;
-
+    search_backend.index_documents(&documents.projects).await?;
-    search_backend.index_documents(&documents).await?;
+    search_backend
+        .index_version_documents(&documents.versions)
+        .await?;
 
     Ok(())
 }

apps/labrinth/src/search/indexing.rs

@@ -24,7 +24,10 @@ use crate::models::ids::{ProjectId, VersionId};
 use crate::models::projects::{DependencyType, from_duplicate_version_fields};
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
-use crate::search::{SearchProjectDependency, UploadSearchProject};
+use crate::search::{
+    SearchDocumentBatch, SearchProjectDependency, UploadSearchProject,
+    UploadSearchVersion,
+};
 use crate::util::error::Context;
 
 struct PartialProject {
@@ -68,7 +71,7 @@ pub async fn index_local(
     redis: &RedisPool,
     cursor: i64,
     limit: i64,
-) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
+) -> eyre::Result<(SearchDocumentBatch, i64)> {
     info!("Indexing local projects!");
 
     let searchable_statuses = searchable_statuses();
@@ -111,12 +114,11 @@ pub async fn index_local(
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let Some(largest) = project_ids.iter().max() else {
-        return Ok((vec![], i64::MAX));
+        return Ok((SearchDocumentBatch::default(), i64::MAX));
     };
 
-    let uploads =
+    let documents = build_search_documents(pool, redis, db_projects).await?;
-        build_search_documents(pool, redis, db_projects, None).await?;
+    Ok((documents, *largest))
-    Ok((uploads, *largest))
 }
 
 pub async fn index_project_documents(
@@ -164,7 +166,9 @@ pub async fn index_project_documents(
 
     info!("Fetched partial projects");
 
-    build_search_documents(pool, redis, db_projects, None).await
+    Ok(build_search_documents(pool, redis, db_projects)
+        .await?
+        .projects)
 }
 
 pub async fn index_project_version_documents(
@@ -172,16 +176,33 @@ pub async fn index_project_version_documents(
     redis: &RedisPool,
     project_ids: &[ProjectId],
     version_ids: &[VersionId],
-) -> eyre::Result<Vec<UploadSearchProject>> {
+) -> eyre::Result<SearchDocumentBatch> {
+    let projects =
+        index_project_document_batch(pool, redis, project_ids).await?;
+    let version_ids = version_ids
+        .iter()
+        .map(ToString::to_string)
+        .collect::<HashSet<_>>();
+    Ok(SearchDocumentBatch {
+        projects: projects.projects,
+        versions: projects
+            .versions
+            .into_iter()
+            .filter(|version| version_ids.contains(&version.version_id))
+            .collect(),
+    })
+}
+
+async fn index_project_document_batch(
+    pool: &PgPool,
+    redis: &RedisPool,
+    project_ids: &[ProjectId],
+) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = project_ids
         .iter()
         .map(|project_id| DBProjectId::from(*project_id).0)
         .collect::<Vec<_>>();
-    let version_ids = version_ids
-        .iter()
-        .map(|version_id| DBVersionId::from(*version_id))
-        .collect::<HashSet<_>>();
 
     let db_projects = sqlx::query!(
         r#"
@@ -215,15 +236,14 @@ pub async fn index_project_version_documents(
     .await
     .wrap_err("failed to fetch project")?;
 
-    build_search_documents(pool, redis, db_projects, Some(&version_ids)).await
+    build_search_documents(pool, redis, db_projects).await
 }
 
 async fn build_search_documents(
     pool: &PgPool,
     redis: &RedisPool,
     db_projects: Vec<PartialProject>,
-    version_ids_to_index: Option<&HashSet<DBVersionId>>,
+) -> eyre::Result<SearchDocumentBatch> {
-) -> eyre::Result<Vec<UploadSearchProject>> {
     let searchable_statuses = searchable_statuses();
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let project_components = db_projects
@@ -391,7 +411,7 @@ async fn build_search_documents(
     .await?;
 
     info!("Getting all loader fields!");
-    let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
+    let loader_field_definitions: Vec<QueryLoaderField> = sqlx::query!(
         "
         SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional
         FROM loader_fields lf
@@ -409,7 +429,8 @@ async fn build_search_documents(
     })
     .try_collect()
     .await?;
-    let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect();
+    let loader_field_definitions: Vec<&QueryLoaderField> =
+        loader_field_definitions.iter().collect();
 
     info!("Getting all loader field enum values!");
 
@@ -434,7 +455,8 @@ async fn build_search_documents(
         .await?;
 
     info!("Indexing loaders, project types!");
-    let mut uploads = Vec::new();
+    let mut project_uploads = Vec::new();
+    let mut version_uploads = Vec::new();
 
     let total_len = db_projects.len();
     let mut count = 0;
@@ -533,21 +555,34 @@ async fn build_search_documents(
             .collect::<Vec<_>>();
 
         if let Some(versions) = versions.remove(&project.id) {
-            // Aggregated project loader fields
+            let Some(latest_version) = versions.iter().max_by(|a, b| {
+                a.date_published
+                    .cmp(&b.date_published)
+                    .then_with(|| a.id.0.cmp(&b.id.0))
+            }) else {
+                continue;
+            };
+
             let project_version_fields = versions
                 .iter()
                 .flat_map(|x| x.version_fields.clone())
                 .collect::<Vec<_>>();
             let aggregated_version_fields = VersionField::from_query_json(
                 project_version_fields,
-                &loader_fields,
+                &loader_field_definitions,
                 &loader_field_enum_values,
                 true,
             );
-            let project_loader_fields =
+            let unvectorized_loader_fields = aggregated_version_fields
+                .iter()
+                .map(|field| {
+                    (field.field_name.clone(), field.value.serialize_internal())
+                })
+                .collect();
+            let mut loader_fields =
                 from_duplicate_version_fields(aggregated_version_fields);
+            let project_loader_fields = loader_fields.clone();
 
-            // aggregated project loaders
             let mut project_loaders = versions
                 .iter()
                 .flat_map(|x| x.loaders.clone())
@@ -555,162 +590,184 @@ async fn build_search_documents(
             project_loaders.sort();
             project_loaders.dedup();
 
-            // all valid project types across every version of the project, so that
+            let mut project_types = versions
-            // filters can exclude projects that have *any* version of a given
-            // project type (unlike the version-specific `project_types` field).
-            let mut all_project_types = versions
                 .iter()
                 .flat_map(|x| x.project_types.clone())
                 .collect::<Vec<_>>();
-            all_project_types.sort();
+            project_types.sort();
-            all_project_types.dedup();
+            project_types.dedup();
             exp::compat::correct_project_types(
                 &project.components,
-                &mut all_project_types,
+                &mut project_types,
             );
 
-            for version in versions {
+            let project_id = ProjectId::from(project.id).to_string();
-                if let Some(version_ids_to_index) = version_ids_to_index
+            version_uploads.extend(versions.iter().map(|version| {
-                    && !version_ids_to_index.contains(&version.id)
-                {
-                    continue;
-                }
-
                 let version_fields = VersionField::from_query_json(
-                    version.version_fields,
+                    version.version_fields.clone(),
-                    &loader_fields,
+                    &loader_field_definitions,
                     &loader_field_enum_values,
                     false,
                 );
                 let unvectorized_loader_fields = version_fields
                     .iter()
-                    .map(|vf| {
+                    .map(|field| {
-                        (vf.field_name.clone(), vf.value.serialize_internal())
+                        (
+                            field.field_name.clone(),
+                            field.value.serialize_internal(),
+                        )
                     })
                     .collect();
-                let mut loader_fields =
+                let mut fields = from_duplicate_version_fields(version_fields);
-                    from_duplicate_version_fields(version_fields);
+                let mut version_project_types = version.project_types.clone();
-                let mut project_types = version.project_types;
-
                 exp::compat::correct_project_types(
                     &project.components,
-                    &mut project_types,
+                    &mut version_project_types,
                 );
 
-                let mut version_loaders = version.loaders;
+                let mut version_categories = version.loaders.clone();
-
+                let mrpack_loaders = fields
-                // Uses version loaders, not project loaders.
-                let mut categories = categories.clone();
-                categories.append(&mut version_loaders.clone());
-
-                let display_categories = display_categories.clone();
-                categories.append(&mut version_loaders);
-
-                // SPECIAL BEHAVIOUR
-                // Todo: revisit.
-                // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category.
-                // These were previously considered the loader, and in v2, the loader is a category for searching.
-                // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories.
-                // The loaders are kept in loader_fields as well, so that no information is lost on retrieval.
-                let mrpack_loaders = loader_fields
                     .get("mrpack_loaders")
-                    .cloned()
+                    .into_iter()
-                    .map(|x| {
+                    .flatten()
-                        x.into_iter()
+                    .filter_map(|value| value.as_str().map(String::from))
-                            .filter_map(|x| x.as_str().map(String::from))
+                    .collect::<Vec<_>>();
-                            .collect::<Vec<_>>()
+                version_categories.extend(mrpack_loaders);
-                    })
+                if fields.contains_key("mrpack_loaders") {
-                    .unwrap_or_default();
+                    version_categories.retain(|category| category != "mrpack");
-                categories.extend(mrpack_loaders);
-                if loader_fields.contains_key("mrpack_loaders") {
-                    categories.retain(|x| *x != "mrpack");
                 }
+                version_categories.sort();
+                version_categories.dedup();
 
-                // SPECIAL BEHAVIOUR:
-                // For consistency with v2 searching, we manually input the
-                // client_side and server_side fields from the loader fields into
-                // separate loader fields.
-                // 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
                 let (_, v2_og_project_type) =
-                    LegacyProject::get_project_type(&project_types);
+                    LegacyProject::get_project_type(&version_project_types);
                 let (client_side, server_side) =
                     v2_reroute::convert_v3_side_types_to_v2_side_types(
                         &unvectorized_loader_fields,
                         Some(&v2_og_project_type),
                     );
-
                 if let Ok(client_side) = serde_json::to_value(client_side) {
-                    loader_fields
+                    fields.insert("client_side".to_string(), vec![client_side]);
-                        .insert("client_side".to_string(), vec![client_side]);
                 }
                 if let Ok(server_side) = serde_json::to_value(server_side) {
-                    loader_fields
+                    fields.insert("server_side".to_string(), vec![server_side]);
-                        .insert("server_side".to_string(), vec![server_side]);
                 }
-
+                fields.retain(|field, _| {
-                let components = project
+                    matches!(
-                    .components
+                        field.as_str(),
-                    .clone()
+                        "environment"
-                    .into_query(
+                            | "game_versions"
-                        ProjectId::from(project.id),
+                            | "client_side"
-                        &project_query_context,
+                            | "server_side"
                     )
-                    .wrap_err("failed to populate query components")?;
+                });
 
-                let usp = UploadSearchProject {
+                UploadSearchVersion {
-                    version_id: crate::models::ids::VersionId::from(version.id)
+                    version_id: VersionId::from(version.id).to_string(),
-                        .to_string(),
+                    project_id: project_id.clone(),
-                    project_id: crate::models::ids::ProjectId::from(project.id)
+                    categories: version_categories,
-                        .to_string(),
+                    project_types: version_project_types,
-                    name: project.name.clone(),
-                    indexed_name: normalize_for_search(&project.name),
-                    summary: project.summary.clone(),
-                    categories: categories.clone(),
-                    display_categories: display_categories.clone(),
-                    follows: project.follows,
-                    downloads: project.downloads,
-                    log_downloads: (project.downloads.max(1) as f64).ln(),
-                    icon_url: project.icon_url.clone(),
-                    author: username.clone(),
-                    author_id: ariadne::ids::UserId::from(user_id).to_string(),
-                    organization: org_name.clone(),
-                    organization_id: org_id.map(|e| {
-                        crate::models::ids::OrganizationId::from(e).to_string()
-                    }),
-                    indexed_author: normalize_for_search(&username),
-                    date_created: project.approved,
-                    created_timestamp: project.approved.timestamp(),
-                    date_modified: project.updated,
-                    modified_timestamp: project.updated.timestamp(),
                     version_published_timestamp: version
                         .date_published
                         .timestamp(),
-                    license: license.clone(),
+                    loader_fields: fields,
-                    slug: project.slug.clone(),
+                }
-                    // TODO
+            }));
-                    project_types,
-                    all_project_types: all_project_types.clone(),
-                    gallery: gallery.clone(),
-                    featured_gallery: featured_gallery.clone(),
-                    open_source,
-                    color: project.color.map(|x| x as u32),
-                    dependency_project_ids: dependency_project_ids.clone(),
-                    compatible_dependency_project_ids:
-                        compatible_dependency_project_ids.clone(),
-                    dependencies: dependencies.clone(),
-                    loader_fields,
-                    project_loader_fields: project_loader_fields.clone(),
-                    // 'loaders' is aggregate of all versions' loaders
-                    loaders: project_loaders.clone(),
-                    components,
-                };
 
-                uploads.push(usp);
+            let project_categories = categories.clone();
+            let mut categories = categories;
+            categories.extend(project_loaders.iter().cloned());
+
+            let mrpack_loaders = loader_fields
+                .get("mrpack_loaders")
+                .into_iter()
+                .flatten()
+                .filter_map(|value| value.as_str().map(String::from))
+                .collect::<Vec<_>>();
+            categories.extend(mrpack_loaders);
+            if loader_fields.contains_key("mrpack_loaders") {
+                categories.retain(|category| category != "mrpack");
             }
+            categories.sort();
+            categories.dedup();
+
+            let (_, v2_og_project_type) =
+                LegacyProject::get_project_type(&project_types);
+            let (client_side, server_side) =
+                v2_reroute::convert_v3_side_types_to_v2_side_types(
+                    &unvectorized_loader_fields,
+                    Some(&v2_og_project_type),
+                );
+
+            if let Ok(client_side) = serde_json::to_value(client_side) {
+                loader_fields
+                    .insert("client_side".to_string(), vec![client_side]);
+            }
+            if let Ok(server_side) = serde_json::to_value(server_side) {
+                loader_fields
+                    .insert("server_side".to_string(), vec![server_side]);
+            }
+
+            let components = project
+                .components
+                .clone()
+                .into_query(ProjectId::from(project.id), &project_query_context)
+                .wrap_err("failed to populate query components")?;
+            let indexed_name = normalize_for_search(&project.name);
+
+            project_uploads.push(UploadSearchProject {
+                version_id: crate::models::ids::VersionId::from(
+                    latest_version.id,
+                )
+                .to_string(),
+                project_id,
+                name: project.name,
+                indexed_name,
+                summary: project.summary,
+                categories,
+                project_categories,
+                display_categories,
+                follows: project.follows,
+                downloads: project.downloads,
+                log_downloads: (project.downloads.max(1) as f64).ln(),
+                icon_url: project.icon_url,
+                author: username.clone(),
+                author_id: ariadne::ids::UserId::from(user_id).to_string(),
+                organization: org_name,
+                organization_id: org_id.map(|id| {
+                    crate::models::ids::OrganizationId::from(id).to_string()
+                }),
+                indexed_author: normalize_for_search(&username),
+                date_created: project.approved,
+                created_timestamp: project.approved.timestamp(),
+                date_modified: project.updated,
+                modified_timestamp: project.updated.timestamp(),
+                version_published_timestamp: latest_version
+                    .date_published
+                    .timestamp(),
+                license,
+                slug: project.slug,
+                project_types: project_types.clone(),
+                all_project_types: project_types,
+                gallery,
+                featured_gallery,
+                open_source,
+                color: project.color.map(|x| x as u32),
+                dependency_project_ids,
+                compatible_dependency_project_ids,
+                dependencies,
+                project_loader_fields,
+                loader_fields,
+                loaders: project_loaders,
+                components,
+            });
         }
     }
 
-    Ok(uploads)
+    Ok(SearchDocumentBatch {
+        projects: project_uploads,
+        versions: version_uploads,
+    })
 }
 
 struct PartialVersion {

apps/labrinth/src/search/mod.rs

@@ -116,12 +116,25 @@ pub trait SearchBackend: Send + Sync {
         documents: &[UploadSearchProject],
     ) -> eyre::Result<()>;
 
+    async fn index_version_documents(
+        &self,
+        documents: &[UploadSearchVersion],
+    ) -> eyre::Result<()>;
+
     async fn remove_project_documents(
         &self,
         ids: &[ProjectId],
     ) -> eyre::Result<()>;
 
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
+    async fn remove_project_version_documents(
+        &self,
+        ids: &[ProjectId],
+    ) -> eyre::Result<()>;
+
+    async fn remove_version_documents(
+        &self,
+        ids: &[VersionId],
+    ) -> eyre::Result<()>;
 
     async fn tasks(&self) -> eyre::Result<Value>;
 
@@ -238,6 +251,7 @@ impl FromStr for SearchBackendKind {
 /// serialized as `null`.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     //
@@ -256,6 +270,7 @@ pub struct UploadSearchProject {
     pub indexed_name: String,
     pub summary: String,
     pub categories: Vec<String>,
+    pub project_categories: Vec<String>,
     pub display_categories: Vec<String>,
     pub follows: i32,
     pub downloads: i32,
@@ -274,7 +289,7 @@ pub struct UploadSearchProject {
     pub date_modified: DateTime<Utc>,
     /// Unix timestamp of the last major modification
     pub modified_timestamp: i64,
-    /// Unix timestamp of the publication date of the version
+    /// Unix timestamp of the most recently published version.
     pub version_published_timestamp: i64,
     pub open_source: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -296,6 +311,23 @@ pub struct UploadSearchProject {
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct UploadSearchVersion {
+    pub version_id: String,
+    pub project_id: String,
+    pub categories: Vec<String>,
+    pub project_types: Vec<String>,
+    pub version_published_timestamp: i64,
+    #[serde(flatten)]
+    pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
+}
+
+#[derive(Debug, Default)]
+pub struct SearchDocumentBatch {
+    pub projects: Vec<UploadSearchProject>,
+    pub versions: Vec<UploadSearchVersion>,
+}
+
 /// Nullable fields in Typesense-bound documents should use
 /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
 /// serialized as `null`.
@@ -320,6 +352,7 @@ pub struct SearchResults {
 
 #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
 pub struct ResultSearchProject {
+    /// ID of the most recently published version.
     pub version_id: String,
     pub project_id: String,
     pub project_types: Vec<String>,

scripts/convert-typesense-project-docs.py

@@ -0,0 +1,443 @@
+#!/usr/bin/env python3
+"""Split legacy Typesense JSONL into project and version documents."""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+import zlib
+from collections import OrderedDict
+
+
+VERSION_FILTER_PATHS = (
+	"project_types",
+	"environment",
+	"game_versions",
+	"client_side",
+	"server_side",
+)
+
+BASE62_DIGITS = {
+	character: index
+	for index, character in enumerate(
+		"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	)
+}
+
+
+class ConversionError(Exception):
+	pass
+
+
+def parse_args():
+	parser = argparse.ArgumentParser(
+		description=(
+			"Convert a legacy Typesense JSONL export from one document per "
+			"version into separate project and version collections."
+		)
+	)
+	parser.add_argument("input", help="Legacy Typesense JSONL export")
+	parser.add_argument("projects_output", help="Destination project-document JSONL")
+	parser.add_argument("versions_output", help="Destination version-document JSONL")
+	parser.add_argument(
+		"--shards",
+		type=int,
+		default=512,
+		help="Temporary hash shards used to bound memory usage (default: 512)",
+	)
+	parser.add_argument(
+		"--max-open-shards",
+		type=int,
+		default=64,
+		help="Maximum temporary shard files held open at once (default: 64)",
+	)
+	parser.add_argument(
+		"--progress-interval",
+		type=float,
+		default=10.0,
+		help="Seconds between progress reports (default: 10)",
+	)
+	parser.add_argument(
+		"--keep-temporary",
+		action="store_true",
+		help="Keep temporary shard files after completion or failure",
+	)
+	return parser.parse_args()
+
+
+def base62_value(value):
+	result = 0
+	try:
+		for character in value:
+			result = result * 62 + BASE62_DIGITS[character]
+	except KeyError:
+		return -1
+	return result
+
+
+def get_path(document, path):
+	value = document
+	for segment in path.split("."):
+		if not isinstance(value, dict) or segment not in value:
+			return None, False
+		value = value[segment]
+	return value, True
+
+
+def set_path(document, path, value):
+	segments = path.split(".")
+	current = document
+	for segment in segments[:-1]:
+		child = current.get(segment)
+		if not isinstance(child, dict):
+			child = {}
+			current[segment] = child
+		current = child
+	current[segments[-1]] = value
+
+
+def unique_sorted(values):
+	return sorted(set(values))
+
+
+def extend_values(target, value):
+	if isinstance(value, list):
+		target.extend(value)
+	elif value is not None:
+		target.append(value)
+
+
+def version_filter_document(document):
+	version_id = document.get("version_id") or document.get("id")
+	result = {
+		"id": str(version_id),
+		"version_id": str(version_id),
+		"project_id": str(document["project_id"]),
+		"version_published_timestamp": document.get(
+			"version_published_timestamp", -1
+		),
+	}
+	loaders = list(document.get("loaders") or [])
+	mrpack_loaders = list(document.get("mrpack_loaders") or [])
+	loaders.extend(mrpack_loaders)
+	if mrpack_loaders:
+		loaders = [loader for loader in loaders if loader != "mrpack"]
+	result["categories"] = unique_sorted(loaders)
+	for path in VERSION_FILTER_PATHS:
+		value, exists = get_path(document, path)
+		if exists and value is not None:
+			set_path(result, path, value)
+	return result
+
+
+class ProjectAccumulator:
+	def __init__(self, document):
+		self.latest_document = document
+		self.latest_key = self.version_key(document)
+		self.versions = {}
+		self.categories = []
+		self.version_categories = []
+		self.loaders = []
+		self.project_types = []
+		self.client_side = []
+		self.server_side = []
+		self.add(document)
+
+	@staticmethod
+	def version_key(document):
+		return (
+			document.get("version_published_timestamp", -1),
+			base62_value(str(document.get("version_id", ""))),
+		)
+
+	def add(self, document):
+		project_id = document.get("project_id")
+		if project_id != self.latest_document.get("project_id"):
+			raise ConversionError("attempted to combine different projects")
+
+		version_id = document.get("version_id") or document.get("id")
+		if not version_id:
+			raise ConversionError(f"project `{project_id}` has a version without an ID")
+
+		key = self.version_key(document)
+		if key > self.latest_key:
+			self.latest_document = document
+			self.latest_key = key
+
+		version_document = version_filter_document(document)
+		self.versions[str(version_id)] = (key, version_document)
+		extend_values(self.categories, document.get("categories"))
+		extend_values(self.version_categories, version_document.get("categories"))
+		extend_values(self.loaders, document.get("loaders"))
+		extend_values(self.project_types, document.get("project_types"))
+		extend_values(self.project_types, document.get("all_project_types"))
+		extend_values(self.client_side, document.get("client_side"))
+		extend_values(self.server_side, document.get("server_side"))
+
+	def finish(self):
+		result = dict(self.latest_document)
+		project_id = str(result["project_id"])
+		all_project_types = unique_sorted(self.project_types)
+
+		result["id"] = project_id
+		result["version_id"] = str(
+			self.latest_document.get("version_id")
+			or self.latest_document.get("id")
+		)
+		result["categories"] = unique_sorted(self.categories)
+		result["project_categories"] = unique_sorted(
+			set(self.categories) - set(self.version_categories)
+		)
+		result["loaders"] = unique_sorted(self.loaders)
+		result["project_types"] = all_project_types
+		result["all_project_types"] = all_project_types
+
+		project_loader_fields = result.get("project_loader_fields")
+		if not isinstance(project_loader_fields, dict):
+			project_loader_fields = {}
+			result["project_loader_fields"] = project_loader_fields
+		for field, value in project_loader_fields.items():
+			result[field] = value
+
+		if self.client_side:
+			result["client_side"] = unique_sorted(self.client_side)
+		if self.server_side:
+			result["server_side"] = unique_sorted(self.server_side)
+
+		versions = [
+			version
+			for _, version in sorted(
+				self.versions.values(), key=lambda item: item[0]
+			)
+		]
+		result.pop("versions", None)
+		return result, versions
+
+
+class ShardWriter:
+	def __init__(self, directory, shard_count, max_open):
+		self.directory = directory
+		self.shard_count = shard_count
+		self.max_open = max_open
+		self.handles = OrderedDict()
+
+	def path(self, shard):
+		return os.path.join(self.directory, f"shard-{shard:04d}.jsonl")
+
+	def write(self, project_id, line):
+		shard = zlib.crc32(project_id.encode("utf-8")) % self.shard_count
+		handle = self.handles.pop(shard, None)
+		if handle is None:
+			if len(self.handles) >= self.max_open:
+				_, oldest = self.handles.popitem(last=False)
+				oldest.close()
+			handle = open(self.path(shard), "ab")
+		self.handles[shard] = handle
+		handle.write(line)
+
+	def close(self):
+		for handle in self.handles.values():
+			handle.close()
+		self.handles.clear()
+
+
+def shard_input(args, temporary_directory):
+	writer = ShardWriter(
+		temporary_directory,
+		args.shards,
+		args.max_open_shards,
+	)
+	input_size = os.path.getsize(args.input)
+	bytes_read = 0
+	document_count = 0
+	started_at = time.monotonic()
+	last_report = started_at
+
+	try:
+		with open(args.input, "rb") as input_file:
+			for line_number, line in enumerate(input_file, start=1):
+				bytes_read += len(line)
+				if not line.strip():
+					continue
+				try:
+					document = json.loads(line)
+				except json.JSONDecodeError as error:
+					raise ConversionError(
+						f"invalid JSON on input line {line_number}: {error}"
+					) from error
+				project_id = document.get("project_id")
+				if not project_id:
+					raise ConversionError(
+						f"input line {line_number} has no `project_id`"
+					)
+				writer.write(str(project_id), line)
+				document_count += 1
+
+				now = time.monotonic()
+				if now - last_report >= args.progress_interval:
+					percent = bytes_read / input_size * 100 if input_size else 100
+					print(
+						f"sharding: {percent:.1f}% ({document_count:,} documents)",
+						flush=True,
+					)
+					last_report = now
+	finally:
+		writer.close()
+
+	print(
+		f"sharding complete: {document_count:,} version documents in "
+		f"{time.monotonic() - started_at:.1f}s",
+		flush=True,
+	)
+	return document_count
+
+
+def convert_shards(
+	args,
+	temporary_directory,
+	partial_projects_output,
+	partial_versions_output,
+):
+	project_count = 0
+	version_count = 0
+	started_at = time.monotonic()
+	last_report = started_at
+
+	with (
+		open(partial_projects_output, "w", encoding="utf-8") as projects_file,
+		open(partial_versions_output, "w", encoding="utf-8") as versions_file,
+	):
+		for shard in range(args.shards):
+			path = os.path.join(temporary_directory, f"shard-{shard:04d}.jsonl")
+			if not os.path.exists(path):
+				continue
+
+			projects = {}
+			with open(path, "rb") as shard_file:
+				for line_number, line in enumerate(shard_file, start=1):
+					try:
+						document = json.loads(line)
+					except json.JSONDecodeError as error:
+						raise ConversionError(
+							f"invalid JSON in shard {shard}, line {line_number}: {error}"
+						) from error
+					project_id = str(document["project_id"])
+					if project_id in projects:
+						projects[project_id].add(document)
+					else:
+						projects[project_id] = ProjectAccumulator(document)
+					version_count += 1
+
+			for project_id in sorted(projects):
+				project, versions = projects[project_id].finish()
+				json.dump(
+					project,
+					projects_file,
+					separators=(",", ":"),
+					ensure_ascii=False,
+				)
+				projects_file.write("\n")
+				for version in versions:
+					json.dump(
+						version,
+						versions_file,
+						separators=(",", ":"),
+						ensure_ascii=False,
+					)
+					versions_file.write("\n")
+				project_count += 1
+
+			os.remove(path)
+			now = time.monotonic()
+			if now - last_report >= args.progress_interval:
+				print(
+					f"converting: shard {shard + 1}/{args.shards}, "
+					f"{project_count:,} projects",
+					flush=True,
+				)
+				last_report = now
+
+	print(
+		f"conversion complete: {version_count:,} versions into "
+		f"{project_count:,} projects in {time.monotonic() - started_at:.1f}s",
+		flush=True,
+	)
+	return project_count, version_count
+
+
+def main():
+	args = parse_args()
+	if args.shards <= 0:
+		raise ConversionError("--shards must be greater than zero")
+	if args.max_open_shards <= 0:
+		raise ConversionError("--max-open-shards must be greater than zero")
+	if args.progress_interval <= 0:
+		raise ConversionError("--progress-interval must be greater than zero")
+	if not os.path.isfile(args.input):
+		raise ConversionError(f"input file does not exist: {args.input}")
+	paths = {
+		os.path.abspath(args.input),
+		os.path.abspath(args.projects_output),
+		os.path.abspath(args.versions_output),
+	}
+	if len(paths) != 3:
+		raise ConversionError("input and output paths must all differ")
+
+	projects_directory = os.path.dirname(os.path.abspath(args.projects_output))
+	versions_directory = os.path.dirname(os.path.abspath(args.versions_output))
+	os.makedirs(projects_directory, exist_ok=True)
+	os.makedirs(versions_directory, exist_ok=True)
+	temporary_directory = tempfile.mkdtemp(
+		prefix="typesense-project-convert-",
+		dir=projects_directory,
+	)
+	partial_projects_output = f"{args.projects_output}.partial"
+	partial_versions_output = f"{args.versions_output}.partial"
+
+	try:
+		expected_versions = shard_input(args, temporary_directory)
+		project_count, version_count = convert_shards(
+			args,
+			temporary_directory,
+			partial_projects_output,
+			partial_versions_output,
+		)
+		if version_count != expected_versions:
+			raise ConversionError(
+				f"sharded {expected_versions} versions but converted {version_count}"
+			)
+		os.replace(partial_projects_output, args.projects_output)
+		os.replace(partial_versions_output, args.versions_output)
+		print(
+			f"wrote {project_count:,} project documents to "
+			f"{args.projects_output} and {version_count:,} version documents to "
+			f"{args.versions_output}",
+			flush=True,
+		)
+	except Exception:
+		for partial_output in (
+			partial_projects_output,
+			partial_versions_output,
+		):
+			if os.path.exists(partial_output):
+				os.remove(partial_output)
+		raise
+	finally:
+		if args.keep_temporary:
+			print(f"temporary shards kept at {temporary_directory}", file=sys.stderr)
+		else:
+			shutil.rmtree(temporary_directory, ignore_errors=True)
+
+
+if __name__ == "__main__":
+	try:
+		main()
+	except KeyboardInterrupt:
+		print("interrupted", file=sys.stderr)
+		sys.exit(130)
+	except (ConversionError, OSError) as error:
+		print(error, file=sys.stderr)
+		sys.exit(1)