cleanup pass

apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        UPDATE mods\n        SET queued = NOW()\n        WHERE id = $1 AND status = $2\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981"
-}

apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id FROM versions WHERE mod_id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24"
-}

apps/labrinth/src/background_task.rs

@@ -19,7 +19,7 @@ use crate::util::anrok;
 use actix_web::web;
 use clap::ValueEnum;
 use eyre::WrapErr;
-use tracing::{info, instrument};
+use tracing::info;
 
 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
 #[clap(rename_all = "kebab_case")]

apps/labrinth/src/search/backend/typesense/mod.rs

@@ -814,74 +814,16 @@ impl Typesense {
             .transpose()
     }
 
-    async fn ensure_collections(&self) -> Result<()> {
-        let projects_alias = self.config.get_alias_name("projects");
-        let projects_collection = if let Some(collection) =
-            self.client.get_alias(&projects_alias).await?
-        {
-            collection
-        } else {
-            let collection =
-                self.config.get_next_collection_name(&projects_alias, false);
-            if !self.client.collection_exists(&collection).await? {
-                self.client
-                    .create_collection(&Self::project_collection_schema(
-                        &collection,
-                    ))
-                    .await?;
-            }
-            self.client
-                .upsert_alias(&projects_alias, &collection)
-                .await?;
-            collection
-        };
-
-        let versions_alias = self.config.get_alias_name("versions");
-        if self.client.get_alias(&versions_alias).await?.is_none() {
-            let collection =
-                self.config.get_next_collection_name(&versions_alias, false);
-            if !self.client.collection_exists(&collection).await? {
-                self.client
-                    .create_collection(&Self::version_collection_schema(
-                        &collection,
-                        &projects_collection,
-                    ))
-                    .await?;
-            }
-            self.client
-                .upsert_alias(&versions_alias, &collection)
-                .await?;
-        }
-        Ok(())
-    }
-
-    async fn delete_documents_by_filter_if_exists(
-        &self,
-        collection: &str,
-        filter: &str,
-    ) -> Result<()> {
-        if self.client.collection_exists(collection).await? {
-            self.client
-                .delete_documents_by_filter(
-                    collection,
-                    filter,
-                    self.config.delete_batch_size,
-                )
-                .await?;
-        }
-
-        Ok(())
-    }
-
-    async fn import_document_batches(
+    async fn import_document_batches<T>(
         &self,
         collections: &[String],
-        documents: &[UploadSearchProject],
+        documents: &[T],
+        serialize: fn(&[T]) -> Result<String>,
     ) -> Result<()> {
         let batch_size = self.config.import_batch_size.max(1);
 
         for batch in documents.chunks(batch_size) {
-            let jsonl = documents_to_jsonl(batch)?;
+            let jsonl = serialize(batch)?;
 
             for collection in collections {
                 info!(
@@ -899,43 +841,21 @@ impl Typesense {
         Ok(())
     }
 
-    async fn import_version_document_batches(
-        &self,
-        collections: &[String],
-        documents: &[UploadSearchVersion],
-    ) -> Result<()> {
-        let batch_size = self.config.import_batch_size.max(1);
-
-        for batch in documents.chunks(batch_size) {
-            let jsonl = version_documents_to_jsonl(batch)?;
-
-            for collection in collections {
-                info!(
-                    collection,
-                    document_count = batch.len(),
-                    content_length_bytes = jsonl.len(),
-                    "sending Typesense version document import"
-                );
-                self.client
-                    .import_documents(collection, jsonl.clone())
-                    .await?;
-            }
-        }
-
-        Ok(())
-    }
-
     async fn existing_write_collections(
         &self,
         alias: &str,
     ) -> Result<Vec<String>> {
-        let mut collections = Vec::new();
+        let mut collections = self
+            .client
+            .get_alias(alias)
+            .await?
+            .into_iter()
+            .collect_vec();
 
-        let live = self.client.get_alias(alias).await?;
-        let shadow_alt = self.config.get_next_collection_name(alias, true);
-        let shadow_current = self.config.get_next_collection_name(alias, false);
-
-        for collection in live.into_iter().chain([shadow_alt, shadow_current]) {
+        for collection in [
+            self.config.get_next_collection_name(alias, true),
+            self.config.get_next_collection_name(alias, false),
+        ] {
             if !collections.contains(&collection)
                 && self.client.collection_exists(&collection).await?
             {
@@ -946,27 +866,24 @@ impl Typesense {
         Ok(collections)
     }
 
-    async fn delete_from_write_collections(
-        &self,
-        alias: &str,
-        filter: &str,
-    ) -> Result<()> {
-        for collection in self.existing_write_collections(alias).await? {
-            self.delete_documents_by_filter_if_exists(&collection, filter)
-                .await?;
-        }
-        Ok(())
-    }
-
     async fn delete_ids_from_write_collections(
         &self,
         alias: &str,
         field: &str,
         ids: &[String],
     ) -> Result<()> {
+        let collections = self.existing_write_collections(alias).await?;
         for ids in ids.chunks(DELETE_FILTER_ID_BATCH_SIZE) {
             let filter = format!("{field}:[{}]", ids.iter().join(", "));
-            self.delete_from_write_collections(alias, &filter).await?;
+            for collection in &collections {
+                self.client
+                    .delete_documents_by_filter(
+                        collection,
+                        &filter,
+                        self.config.delete_batch_size,
+                    )
+                    .await?;
+            }
         }
         Ok(())
     }
@@ -1177,8 +1094,6 @@ impl SearchBackend for Typesense {
         let projects_alias = self.config.get_alias_name("projects");
         let versions_alias = self.config.get_alias_name("versions");
 
-        self.ensure_collections().await?;
-
         let projects_current = self.client.get_alias(&projects_alias).await?;
         let versions_current = self.client.get_alias(&versions_alias).await?;
 
@@ -1246,11 +1161,13 @@ impl SearchBackend for Typesense {
             self.import_document_batches(
                 std::slice::from_ref(&projects_next),
                 &documents.projects,
+                documents_to_jsonl,
             )
             .await?;
-            self.import_version_document_batches(
+            self.import_document_batches(
                 std::slice::from_ref(&versions_next),
                 &documents.versions,
+                version_documents_to_jsonl,
             )
             .await?;
         }
@@ -1348,8 +1265,12 @@ impl SearchBackend for Typesense {
                 num_documents = update.projects.len(),
                 "Replacing project documents in collections",
             );
-            self.import_document_batches(&collections, update.projects)
-                .await?;
+            self.import_document_batches(
+                &collections,
+                update.projects,
+                documents_to_jsonl,
+            )
+            .await?;
         }
 
         if !update.versions.is_empty() {
@@ -1360,8 +1281,12 @@ impl SearchBackend for Typesense {
                 num_documents = update.versions.len(),
                 "Replacing version documents in collections",
             );
-            self.import_version_document_batches(&collections, update.versions)
-                .await?;
+            self.import_document_batches(
+                &collections,
+                update.versions,
+                version_documents_to_jsonl,
+            )
+            .await?;
         }
 
         debug!("Done applying search index update");
@@ -1540,15 +1465,10 @@ fn rewrite_filter_for_join(
 }
 
 fn is_version_filter_field(field: &str) -> bool {
-    matches!(
-        field,
-        "categories"
-            | "project_types"
-            | "environment"
-            | "game_versions"
-            | "client_side"
-            | "server_side"
-    )
+    <SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
+        search_field.is_version_field()
+            && search_field.typesense_spec().path == field
+    })
 }
 
 fn is_negative_filter(expression: &str) -> bool {

apps/labrinth/src/search/incremental/consume.rs

@@ -168,7 +168,14 @@ async fn consume_batch(
             }
         };
 
-        match event.into_data() {
+        let event = match event {
+            SearchProjectIndexQueueEvent::Current(event) => event,
+            SearchProjectIndexQueueEvent::Legacy { project_id } => {
+                SearchProjectIndexQueueEventData::Change { project_id }
+            }
+        };
+
+        match event {
             SearchProjectIndexQueueEventData::Change { project_id } => {
                 project_ids_to_change.insert(project_id);
             }
@@ -200,7 +207,7 @@ async fn consume_batch(
     let project_ids_with_version_changes = project_ids_with_version_changes
         .into_iter()
         .collect::<Vec<_>>();
-    let project_ids_to_remove =
+    let mut project_ids_to_remove =
         project_ids_to_remove.into_iter().collect::<Vec<_>>();
     let version_ids_to_change =
         version_ids_to_change.into_iter().collect::<Vec<_>>();
@@ -238,6 +245,10 @@ async fn consume_batch(
                 version_ids_to_change.len()
             )
         })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_with_version_changes,
+            &changed_documents.projects,
+        ));
         documents.projects.extend(changed_documents.projects);
         documents.versions.extend(changed_documents.versions);
         info!(
@@ -254,15 +265,27 @@ async fn consume_batch(
             project_count = project_ids_to_change.len(),
             "Building changed projects"
         );
-        documents.projects.extend(
-            build_changed_project_documents(
-                ro_pool,
-                redis_pool,
-                &project_ids_to_change,
+        let changed_project_documents = build_project_documents(
+            ro_pool,
+            redis_pool,
+            &project_ids_to_change,
+        )
+        .instrument(info_span!(
+            "index",
+            batch_size = project_ids_to_change.len()
+        ))
+        .await
+        .wrap_err_with(|| {
+            format!(
+                "failed to build search documents for {} projects",
+                project_ids_to_change.len()
             )
-            .await
-            .wrap_err("failed to build changed projects")?,
-        );
+        })?;
+        project_ids_to_remove.extend(missing_project_document_ids(
+            &project_ids_to_change,
+            &changed_project_documents,
+        ));
+        documents.projects.extend(changed_project_documents);
         info!(
             project_count = project_ids_to_change.len(),
             "Built changed projects in {:.2?}",
@@ -327,25 +350,7 @@ pub async fn reindex_project_documents(
     project_ids: &[ProjectId],
 ) -> eyre::Result<()> {
     info!("Creating project documents");
-    let projects =
-        build_changed_project_documents(ro_pool, redis_pool, project_ids)
-            .await?;
-    search_backend
-        .apply_update(SearchIndexUpdate {
-            projects: &projects,
-            ..SearchIndexUpdate::default()
-        })
-        .await?;
-
-    Ok(())
-}
-
-async fn build_changed_project_documents(
-    ro_pool: &PgPool,
-    redis_pool: &RedisPool,
-    project_ids: &[ProjectId],
-) -> eyre::Result<Vec<UploadSearchProject>> {
-    let documents = build_project_documents(ro_pool, redis_pool, project_ids)
+    let projects = build_project_documents(ro_pool, redis_pool, project_ids)
         .instrument(info_span!("index", batch_size = project_ids.len()))
         .await
         .wrap_err_with(|| {
@@ -354,9 +359,34 @@ async fn build_changed_project_documents(
                 project_ids.len()
             )
         })?;
+    let removed_projects = missing_project_document_ids(project_ids, &projects);
+    search_backend
+        .apply_update(SearchIndexUpdate {
+            projects: &projects,
+            removed_projects: &removed_projects,
+            ..SearchIndexUpdate::default()
+        })
+        .await?;
 
-    info!("Fetched all project documents");
-    Ok(documents)
+    Ok(())
+}
+
+fn missing_project_document_ids(
+    project_ids: &[ProjectId],
+    documents: &[UploadSearchProject],
+) -> Vec<ProjectId> {
+    let built_project_ids = documents
+        .iter()
+        .map(|project| project.project_id.as_str())
+        .collect::<HashSet<_>>();
+
+    project_ids
+        .iter()
+        .copied()
+        .filter(|project_id| {
+            !built_project_ids.contains(project_id.to_string().as_str())
+        })
+        .collect()
 }
 
 #[derive(Debug, Deserialize)]
@@ -366,17 +396,6 @@ enum SearchProjectIndexQueueEvent {
     Legacy { project_id: ProjectId },
 }
 
-impl SearchProjectIndexQueueEvent {
-    fn into_data(self) -> SearchProjectIndexQueueEventData {
-        match self {
-            Self::Current(data) => data,
-            Self::Legacy { project_id } => {
-                SearchProjectIndexQueueEventData::Change { project_id }
-            }
-        }
-    }
-}
-
 #[derive(Debug, Deserialize)]
 #[serde(tag = "type", rename_all = "snake_case")]
 enum SearchProjectIndexQueueEventData {

apps/labrinth/src/search/indexing.rs

@@ -117,7 +117,8 @@ pub async fn index_local(
         return Ok((SearchDocumentBatch::default(), i64::MAX));
     };
 
-    let documents = build_search_documents(pool, redis, db_projects).await?;
+    let documents =
+        build_search_documents(pool, redis, db_projects, None).await?;
     Ok((documents, *largest))
 }
 
@@ -126,49 +127,12 @@ pub async fn build_project_documents(
     redis: &RedisPool,
     project_ids: &[ProjectId],
 ) -> eyre::Result<Vec<UploadSearchProject>> {
-    let searchable_statuses = searchable_statuses();
-    let project_ids = project_ids
-        .iter()
-        .map(|project_id| DBProjectId::from(*project_id).0)
-        .collect::<Vec<_>>();
-
-    let db_projects = sqlx::query!(
-        r#"
-		SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,
-		m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color,
-		m.components AS "components: sqlx::types::Json<exp::ProjectSerial>"
-		FROM mods m
-		WHERE m.status = ANY($1) AND m.id = ANY($2)
-		GROUP BY m.id
-		ORDER BY m.id ASC;
-		"#,
-        &searchable_statuses,
-        &project_ids,
+    let version_ids = HashSet::new();
+    Ok(
+        build_search_document_batch(pool, redis, project_ids, &version_ids)
+            .await?
+            .projects,
     )
-    .fetch(pool)
-    .map_ok(|m| PartialProject {
-        id: DBProjectId(m.id),
-        name: m.name,
-        summary: m.summary,
-        downloads: m.downloads,
-        follows: m.follows,
-        icon_url: m.icon_url,
-        updated: m.updated,
-        approved: m.approved.unwrap_or(m.published),
-        slug: m.slug,
-        color: m.color,
-        license: m.license,
-        components: m.components.0,
-    })
-    .try_collect::<Vec<PartialProject>>()
-    .await
-    .wrap_err("failed to fetch project")?;
-
-    info!("Fetched partial projects");
-
-    Ok(build_search_documents(pool, redis, db_projects)
-        .await?
-        .projects)
 }
 
 pub async fn build_version_change_documents(
@@ -177,26 +141,19 @@ pub async fn build_version_change_documents(
     project_ids: &[ProjectId],
     version_ids: &[VersionId],
 ) -> eyre::Result<SearchDocumentBatch> {
-    let projects =
-        build_search_document_batch(pool, redis, project_ids).await?;
     let version_ids = version_ids
         .iter()
-        .map(ToString::to_string)
+        .copied()
+        .map(DBVersionId::from)
         .collect::<HashSet<_>>();
-    Ok(SearchDocumentBatch {
-        projects: projects.projects,
-        versions: projects
-            .versions
-            .into_iter()
-            .filter(|version| version_ids.contains(&version.version_id))
-            .collect(),
-    })
+    build_search_document_batch(pool, redis, project_ids, &version_ids).await
 }
 
 async fn build_search_document_batch(
     pool: &PgPool,
     redis: &RedisPool,
     project_ids: &[ProjectId],
+    version_ids: &HashSet<DBVersionId>,
 ) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = project_ids
@@ -236,13 +193,14 @@ async fn build_search_document_batch(
     .await
     .wrap_err("failed to fetch project")?;
 
-    build_search_documents(pool, redis, db_projects).await
+    build_search_documents(pool, redis, db_projects, Some(version_ids)).await
 }
 
 async fn build_search_documents(
     pool: &PgPool,
     redis: &RedisPool,
     db_projects: Vec<PartialProject>,
+    version_ids: Option<&HashSet<DBVersionId>>,
 ) -> eyre::Result<SearchDocumentBatch> {
     let searchable_statuses = searchable_statuses();
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
@@ -602,7 +560,13 @@ async fn build_search_documents(
             );
 
             let project_id = ProjectId::from(project.id).to_string();
-            version_uploads.extend(versions.iter().map(|version| {
+            version_uploads.extend(versions.iter().filter_map(|version| {
+                if version_ids.is_some_and(|version_ids| {
+                    !version_ids.contains(&version.id)
+                }) {
+                    return None;
+                }
+
                 let version_fields = VersionField::from_query_json(
                     version.version_fields.clone(),
                     &loader_field_definitions,
@@ -662,7 +626,7 @@ async fn build_search_documents(
                     )
                 });
 
-                UploadSearchVersion {
+                Some(UploadSearchVersion {
                     version_id: VersionId::from(version.id).to_string(),
                     project_id: project_id.clone(),
                     categories: version_categories,
@@ -671,7 +635,7 @@ async fn build_search_documents(
                         .date_published
                         .timestamp(),
                     loader_fields: fields,
-                }
+                })
             }));
 
             let mut project_categories = categories;