Incremental search indexing (#6453)

* wip: initial kafka support

* wip: kafka search queue

* wip: kafka consumer

* wip: create/delete docs

* reindexing works

* reindex in more places

* also incrementally update server pings

* cleanup

* tombi

* fix ci

* update tombi

* prepare

* fix ci

* fix

* fix PLEASE

* 🙏

* add route to force reindex project

apps/labrinth/src/background_task.rs

@@ -29,6 +29,7 @@ pub enum BackgroundTask {
     SyncPayoutStatuses,
     IndexBilling,
     IndexSubscriptions,
+    IncrementalIndexSearch,
     Migrations,
     Mail,
     /// Queries server project analytics (e.g. number of verified plays in last
@@ -49,6 +50,7 @@ impl BackgroundTask {
         ro_pool: PgPool,
         redis_pool: RedisPool,
         search_backend: web::Data<dyn SearchBackend>,
+        kafka_client: web::Data<crate::util::kafka::KafkaClientState>,
         clickhouse: clickhouse::Client,
         stripe_client: stripe::Client,
         anrok_client: anrok::Client,
@@ -88,12 +90,26 @@ impl BackgroundTask {
                 .await;
                 Ok(())
             }
+            IncrementalIndexSearch => {
+                crate::search::incremental::consume::run(
+                    ro_pool,
+                    redis_pool,
+                    search_backend,
+                )
+                .await
+            }
             Mail => run_email(email_queue).await,
             CacheAnalytics => {
                 cache_analytics(&pool, &redis_pool, &clickhouse).await
             }
             PingMinecraftJavaServers => {
-                ping_minecraft_java_servers(pool, redis_pool, clickhouse).await
+                ping_minecraft_java_servers(
+                    pool,
+                    redis_pool,
+                    clickhouse,
+                    kafka_client,
+                )
+                .await
             }
             DiscordRoleEmailCampaign => {
                 discord_role_email_campaign(pool, redis_pool).await
@@ -322,17 +338,27 @@ pub async fn ping_minecraft_java_servers(
     pool: PgPool,
     redis_pool: RedisPool,
     clickhouse: clickhouse::Client,
+    kafka_client: web::Data<crate::util::kafka::KafkaClientState>,
 ) -> eyre::Result<()> {
     info!("Started pinging Minecraft Java servers");
 
+    let incremental_search_queue =
+        crate::search::incremental::IncrementalSearchQueue::new(kafka_client);
     let server_ping_queue = crate::queue::server_ping::ServerPingQueue::new(
-        pool, redis_pool, clickhouse,
+        pool,
+        redis_pool,
+        clickhouse,
+        incremental_search_queue.clone(),
     );
 
     server_ping_queue
         .ping_minecraft_java_servers()
         .await
         .wrap_err("failed to ping Minecraft Java servers")?;
+    incremental_search_queue
+        .drain()
+        .await
+        .wrap_err("failed to drain incremental search queue")?;
     info!("Successfully pinged Minecraft Java servers");
 
     info!("Done pinging Minecraft Java servers");