Incremental search indexing (#6453)

* wip: initial kafka support

* wip: kafka search queue

* wip: kafka consumer

* wip: create/delete docs

* reindexing works

* reindex in more places

* also incrementally update server pings

* cleanup

* tombi

* fix ci

* update tombi

* prepare

* fix ci

* fix

* fix PLEASE

* 🙏

* add route to force reindex project

apps/labrinth/src/main.rs

@@ -166,6 +166,10 @@ async fn app() -> std::io::Result<()> {
         .expect("Failed to create Gotenberg client");
     let muralpay = labrinth::queue::payouts::create_muralpay_client()
         .expect("Failed to create MuralPay client");
+    let kafka_client = actix_web::web::Data::new(
+        labrinth::util::kafka::KafkaClientState::new()
+            .expect("Kafka connection failed"),
+    );
 
     if let Some(task) = args.run_background_task {
         info!("Running task {task:?} and exiting");
@@ -174,6 +178,7 @@ async fn app() -> std::io::Result<()> {
             ro_pool.into_inner(),
             redis_pool,
             search_backend,
+            kafka_client,
             clickhouse,
             stripe_client,
             anrok_client.clone(),
@@ -216,6 +221,7 @@ async fn app() -> std::io::Result<()> {
         anrok_client.clone(),
         email_queue,
         gotenberg_client,
+        kafka_client,
         !args.no_background_tasks,
     );