Incremental search indexing (#6453)

* wip: initial kafka support

* wip: kafka search queue

* wip: kafka consumer

* wip: create/delete docs

* reindexing works

* reindex in more places

* also incrementally update server pings

* cleanup

* tombi

* fix ci

* update tombi

* prepare

* fix ci

* fix

* fix PLEASE

* 🙏

* add route to force reindex project

apps/labrinth/src/test/database.rs

@@ -170,11 +170,11 @@ impl TemporaryDatabase {
                             "Dummy data updated, so template DB tables will be dropped and re-created"
                         );
                         // Drop all tables in the database so they can be re-created and later filled with updated dummy data
-                        sqlx::query("DROP SCHEMA public CASCADE;")
+                        sqlx::query!("DROP SCHEMA public CASCADE;")
                             .execute(&pool)
                             .await
                             .unwrap();
-                        sqlx::query("CREATE SCHEMA public;")
+                        sqlx::query!("CREATE SCHEMA public;")
                             .execute(&pool)
                             .await
                             .unwrap();
@@ -211,14 +211,14 @@ impl TemporaryDatabase {
                     &temp_database_name, TEMPLATE_DATABASE_NAME
                 );
 
-                sqlx::query(&create_db_query)
+                sqlx::raw_sql(&create_db_query)
                     .execute(&main_pool)
                     .await
                     .expect("Database creation failed");
 
                 // Release the advisory lock
-                sqlx::query("SELECT pg_advisory_unlock(1)")
-                    .execute(&main_pool)
+                sqlx::query_scalar!("SELECT pg_advisory_unlock(1)")
+                    .fetch_one(&main_pool)
                     .await
                     .unwrap();
 
@@ -248,7 +248,7 @@ impl TemporaryDatabase {
             "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()",
             &self.database_name
         );
-        sqlx::query(&terminate_query)
+        sqlx::raw_sql(&terminate_query)
             .execute(&self.pool)
             .await
             .unwrap();
@@ -256,7 +256,7 @@ impl TemporaryDatabase {
         // Execute the deletion query asynchronously
         let drop_db_query =
             format!("DROP DATABASE IF EXISTS {}", &self.database_name);
-        sqlx::query(&drop_db_query)
+        sqlx::raw_sql(&drop_db_query)
             .execute(&self.pool)
             .await
             .expect("Database deletion failed");
@@ -265,7 +265,7 @@ impl TemporaryDatabase {
 
 async fn create_template_database(pool: &PgPool) {
     let create_db_query = format!("CREATE DATABASE {TEMPLATE_DATABASE_NAME}");
-    sqlx::query(&create_db_query)
+    sqlx::raw_sql(&create_db_query)
         .execute(pool)
         .await
         .expect("Database creation failed");

apps/labrinth/src/test/dummy_data.rs

@@ -296,11 +296,12 @@ pub async fn add_dummy_data(api: &ApiV3, db: TemporaryDatabase) -> DummyData {
 
     let oauth_client_alpha = get_oauth_client_alpha(api).await;
 
-    sqlx::query("INSERT INTO dummy_data (update_id) VALUES ($1)")
-        .bind(DUMMY_DATA_UPDATE)
-        .execute(pool)
-        .await
-        .unwrap();
+    sqlx::raw_sql(&format!(
+        "INSERT INTO dummy_data (update_id) VALUES ({DUMMY_DATA_UPDATE})"
+    ))
+    .execute(pool)
+    .await
+    .unwrap();
 
     DummyData::new(
         alpha_project,

apps/labrinth/src/test/mod.rs

@@ -42,6 +42,10 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         EmailQueue::init(pool.clone(), redis_pool.clone()).unwrap();
     let gotenberg_client = GotenbergClient::from_env(redis_pool.clone())
         .expect("Failed to create Gotenberg client");
+    let kafka_client = actix_web::web::Data::new(
+        crate::util::kafka::KafkaClientState::new()
+            .expect("Kafka connection failed"),
+    );
 
     crate::app_setup(
         pool.clone(),
@@ -54,6 +58,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         anrok_client,
         email_queue,
         gotenberg_client,
+        kafka_client,
         false,
     )
 }