fix: search indexing performance and batching (#6521)

* Add consume batching delay

* maybe fix

* max batch size

* more logging

* parallelize remove tasks

* delete/upsert project/version messages

* prepare

* more logging

* log number of docs

* try more targeted, homogenous version change ops

* ensure only necessary fields are serialized into typesense

* disable index background task

* wip: script changes

* don't facet by project id

* fix

* fix clippy

* batch by document and dedup loaders

* fix test

* wip: projects/versions collections

* clean up SearchBackend interface

* cleanup pass

* cleanup pass 2

* standardise fn names

* cleanup pass

* fix compile

* factor out filter rewriting

* query perf

* put categories into the project doc

* wip: search filter AST

* doc comment

* more AST normalization

* (temp) convert search request errors into internal errors

* implement unary NOT

* tombi fmt

* fix tests

* revert request error

* try increase stack size

apps/labrinth/src/routes/v3/version_creation.rs

@@ -184,19 +184,20 @@ pub async fn version_create(
         if let Err(e) = rollback_result {
             return Err(e.into());
         }
-    } else if let Ok((_, project_id)) = &result {
+    } else if let Ok((_, project_id, version_id)) = &result {
         transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
-            *project_id,
-            None,
-            Some(true),
-        )
-        .await?;
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
+            .await?;
+        search_state
+            .queue
+            .push_version_changes(
+                (*project_id).into(),
+                [VersionId::from(*version_id)],
+            )
+            .await;
     }
 
-    result.map(|(response, _)| response)
+    result.map(|(response, _, _)| response)
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -210,7 +211,8 @@ async fn version_create_inner(
     pool: &PgPool,
     session_queue: &AuthQueue,
     http: &reqwest::Client,
-) -> Result<(HttpResponse, models::DBProjectId), CreateError> {
+) -> Result<(HttpResponse, models::DBProjectId, models::DBVersionId), CreateError>
+{
     let mut initial_version_data = None;
     let mut version_builder = None;
     let mut selected_loaders = None;
@@ -568,7 +570,11 @@ async fn version_create_inner(
         }
     }
 
-    Ok((HttpResponse::Ok().json(response), project_id))
+    Ok((
+        HttpResponse::Ok().json(response),
+        project_id,
+        models::DBVersionId::from(version_id),
+    ))
 }
 
 /// Add files to an existing version.
@@ -635,17 +641,18 @@ pub async fn upload_file_to_version(
     let mut transaction = client.begin().await?;
     let mut uploaded_files = Vec::new();
 
-    let version_id = models::DBVersionId::from(url_data.into_inner().0);
+    let version_id = url_data.into_inner().0;
+    let db_version_id = models::DBVersionId::from(version_id);
 
     let result = upload_file_to_version_inner(
         req,
         &mut payload,
-        client,
+        client.clone(),
         &mut transaction,
         redis.clone(),
         &**file_host,
         &mut uploaded_files,
-        version_id,
+        db_version_id,
         &session_queue,
         &http,
     )
@@ -665,14 +672,12 @@ pub async fn upload_file_to_version(
         }
     } else if let Ok((_, project_id)) = &result {
         transaction.commit().await?;
-        super::projects::clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
-            *project_id,
-            None,
-            Some(true),
-        )
-        .await?;
+        models::DBProject::clear_cache(*project_id, None, Some(true), &redis)
+            .await?;
+        search_state
+            .queue
+            .push_version_changes((*project_id).into(), [version_id])
+            .await;
     }
 
     result.map(|(response, _)| response)