Search backend refactor with typesense impl (#5528)

* initial elasticsearch impl

* working elastic cluster

* replace SearchError with ApiError for preparation of search backend

* start factoring meili out to trait

* move meili to backend

* update routes to use search backend trait

* wip

* Update projects.rs

* search backend is only init'd once in config

* wip

* wip: backend agnostic

* change search internal routes to delegate to backend

* initial elasticsearch impl

* fix filtering

* elastic impl

* refactor indexing into its own module

* clean up elastic code

* fix ci

* fix tests

* fix elastic health check

* fix up env rebase

* fix compile

* dummy commit to update github pr

* Fix rebase

* Elastic basic https auth

* Fix duplicate projects showing up

* Fix up tests

* Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors

* clean up agents files

* make index chunk size configurable

* make `match_phrase` in elastic case-insensitive

* use current/next indices and swap between them

* test case for error body

* Fix failing case

* da merge

* factor out common stuff from search backends

* allow fetching hit metadata from search results

* allow customising elasticsearch search config

* bit of docs

* add mappings to indices for elastic

* Implement Typesense

* wip

* fix up some sort fields stuff

* use different approach to filterable field sets

* remove a bunch of search fields which weren't used for filtering

* bucket text matches

* Bucketing by text_match for typesense

* fix tombi lint

* fix some sentry errors and dont prioritise 2+ term matches

* tweak ts query settings

* expose some more search settings

* query sort changes

* small fixes

* should fix pagination stuff

* fix healthcheck maybe

* ragebait ci

* tests

* tests

* revert environment

apps/labrinth/src/routes/internal/admin.rs

@@ -7,7 +7,7 @@ use crate::models::pats::Scopes;
 use crate::queue::analytics::AnalyticsQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
-use crate::search::SearchConfig;
+use crate::search::SearchBackend;
 use crate::util::date::get_current_tenths_of_ms;
 use crate::util::error::Context;
 use crate::util::guards::admin_key_guard;
@@ -154,11 +154,11 @@ pub async fn count_download(
 pub async fn force_reindex(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
-    use crate::search::indexing::index_projects;
     let redis = redis.get_ref();
-    index_projects(pool.as_ref().clone(), redis.clone(), &config)
+    search_backend
+        .index_projects(pool.as_ref().clone(), redis.clone())
         .await
         .wrap_internal_err("failed to index projects")?;
     Ok(HttpResponse::NoContent().finish())

apps/labrinth/src/routes/internal/search.rs

@@ -1,12 +1,9 @@
-use crate::routes::ApiError;
-use crate::search::SearchConfig;
 use crate::util::guards::admin_key_guard;
-use actix_web::{HttpResponse, delete, get, web};
-use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::time::Duration;
-use utoipa::ToSchema;
+use crate::{
+    routes::ApiError,
+    search::{SearchBackend, TasksCancelFilter},
+};
+use actix_web::{delete, get, web};
 
 pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
     cfg.service(tasks).service(tasks_cancel);
@@ -15,107 +12,20 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
 #[utoipa::path]
 #[get("tasks", guard = "admin_key_guard")]
 pub async fn tasks(
-    config: web::Data<SearchConfig>,
-) -> Result<HttpResponse, ApiError> {
-    let client = config.make_batch_client()?;
-    let tasks = client
-        .with_all_clients("get_tasks", async |client| {
-            let tasks = client.get_tasks().await?;
-
-            Ok(tasks.results)
-        })
-        .await?;
-
-    #[derive(Serialize, ToSchema)]
-    struct MeiliTask<Time> {
-        uid: u32,
-        status: &'static str,
-        duration: Option<Duration>,
-        enqueued_at: Option<Time>,
-    }
-
-    #[derive(Serialize, ToSchema)]
-    struct TaskList<Time> {
-        by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
-    }
-
-    let response = tasks
-        .into_iter()
-        .enumerate()
-        .map(|(idx, instance_tasks)| {
-            let tasks = instance_tasks
-                .into_iter()
-                .filter_map(|task| {
-                    Some(match task {
-                        Task::Enqueued { content } => MeiliTask {
-                            uid: content.uid,
-                            status: "enqueued",
-                            duration: None,
-                            enqueued_at: Some(content.enqueued_at),
-                        },
-                        Task::Processing { content } => MeiliTask {
-                            uid: content.uid,
-                            status: "processing",
-                            duration: None,
-                            enqueued_at: Some(content.enqueued_at),
-                        },
-                        Task::Failed { content } => MeiliTask {
-                            uid: content.task.uid,
-                            status: "failed",
-                            duration: Some(content.task.duration),
-                            enqueued_at: Some(content.task.enqueued_at),
-                        },
-                        Task::Succeeded { content: _ } => return None,
-                    })
-                })
-                .collect();
-
-            (idx.to_string(), tasks)
-        })
-        .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
-
-    Ok(HttpResponse::Ok().json(TaskList {
-        by_instance: response,
-    }))
-}
-
-#[derive(Deserialize, Serialize, ToSchema)]
-#[serde(tag = "type", rename_all = "snake_case")]
-enum TasksCancelFilter {
-    All,
-    AllEnqueued,
-    Indexes { indexes: Vec<String> },
+    search: web::Data<dyn SearchBackend>,
+) -> Result<web::Json<serde_json::Value>, ApiError> {
+    Ok(web::Json(search.tasks().await.map_err(ApiError::Internal)?))
 }
 
 #[utoipa::path]
 #[delete("tasks", guard = "admin_key_guard")]
 pub async fn tasks_cancel(
-    config: web::Data<SearchConfig>,
+    search: web::Data<dyn SearchBackend>,
     body: web::Json<TasksCancelFilter>,
-) -> Result<HttpResponse, ApiError> {
-    let client = config.make_batch_client()?;
-    let all_results = client
-        .with_all_clients("cancel_tasks", async |client| {
-            let mut q = TasksCancelQuery::new(client);
-            match &body.0 {
-                TasksCancelFilter::All => {}
-                TasksCancelFilter::Indexes { indexes } => {
-                    q.with_index_uids(indexes.iter().map(|s| s.as_str()));
-                }
-                TasksCancelFilter::AllEnqueued => {
-                    q.with_statuses(["enqueued"]);
-                }
-            };
-
-            let result = client.cancel_tasks_with(&q).await;
-
-            Ok(result)
-        })
-        .await?;
-
-    for r in all_results {
-        r?;
-    }
-
-    Ok(HttpResponse::Ok().finish())
+) -> Result<(), ApiError> {
+    search
+        .tasks_cancel(&body)
+        .await
+        .map_err(ApiError::Internal)?;
+    Ok(())
 }