Add refresh button in files tab behind feature flag

Cargo.lock

@@ -2631,29 +2631,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "elasticsearch"
-version = "9.1.0-alpha.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12bb303aa6e1d28c0c86b6fbfe484fd0fd3f512629aeed1ac4f6b85f81d9834a"
-dependencies = [
- "base64 0.22.1",
- "bytes",
- "dyn-clone",
- "flate2",
- "lazy_static",
- "parking_lot",
- "percent-encoding",
- "reqwest",
- "rustc_version",
- "serde",
- "serde_json",
- "serde_with",
- "tokio",
- "url",
- "void",
-]
-
 [[package]]
 name = "elliptic-curve"
 version = "0.13.8"
@@ -4771,7 +4748,6 @@ dependencies = [
  "dotenv-build",
  "dotenvy",
  "either",
- "elasticsearch",
  "eyre",
  "futures",
  "futures-util",

Cargo.toml

@@ -69,7 +69,6 @@ dotenv-build = "0.1.1"
 dotenvy = "0.15.7"
 dunce = "1.0.5"
 either = "1.15.0"
-elasticsearch = "9.1.0-alpha.1"
 encoding_rs = "0.8.35"
 enumset = "1.1.10"
 eyre = "0.6.12"

apps/app-frontend/src/pages/library/Index.vue

@@ -48,7 +48,7 @@ onUnmounted(() => {
 			]"
 		/>
 		<template v-if="instances.length > 0">
-			<RouterView :instances="instances" />
+			<RouterView v-if="route.path.startsWith('/library')" :instances="instances" />
 		</template>
 		<div v-else class="no-instance">
 			<div class="icon">

apps/frontend/src/components/ui/dashboard/withdraw-stages/MuralpayDetailsStage.vue

@@ -100,7 +100,7 @@
 			/>
 		</div>
 
-		<div v-for="field in selectedRail?.fields" :key="field.name" class="flex flex-col gap-2.5">
+		<div v-for="field in visibleFields" :key="field.name" class="flex flex-col gap-2.5">
 			<label>
 				<span class="text-md font-semibold text-contrast">
 					{{ formatMessage(field.label) }}
@@ -398,6 +398,26 @@ const isBusinessEntity = computed(() => {
 	return providerDataValue.kycData?.type === 'business'
 })
 
+const visibleFields = computed(() => {
+	const rail = selectedRail.value
+	if (!rail) return []
+
+	return rail.fields.filter((field) => {
+		if (!field.dependsOn) return true
+
+		const { field: dependsOnField, value: dependsOnValue } = field.dependsOn
+		const currentValue = formData.value[dependsOnField]
+
+		if (!currentValue) return false
+
+		if (Array.isArray(dependsOnValue)) {
+			return dependsOnValue.includes(currentValue)
+		} else {
+			return currentValue === dependsOnValue
+		}
+	})
+})
+
 const allRequiredFieldsFilled = computed(() => {
 	const rail = selectedRail.value
 	if (!rail) return false
@@ -407,7 +427,7 @@ const allRequiredFieldsFilled = computed(() => {
 
 	if (rail.requiresBankName && !formData.value.bankName) return false
 
-	const requiredFields = rail.fields.filter((f) => f.required)
+	const requiredFields = visibleFields.value.filter((f) => f.required)
 	const allRequiredPresent = requiredFields.every((f) => {
 		const value = formData.value[f.name]
 		return value !== undefined && value !== null && value !== ''

apps/frontend/src/composables/featureFlags.ts

@@ -32,6 +32,7 @@ export const DEFAULT_FEATURE_FLAGS = validateValues({
 	projectBackground: false,
 	searchBackground: false,
 	advancedDebugInfo: false,
+	FilesRefreshButton: false,
 	showProjectPageDownloadModalServersPromo: false,
 	showProjectPageCreateServersTooltip: true,
 	showProjectPageQuickServerButton: false,

apps/frontend/src/pages/hosting/manage/[id]/files.vue

@@ -10,5 +10,8 @@ useHead({
 </script>
 
 <template>
-	<ServersManageFilesPage :show-debug-info="flags.advancedDebugInfo" />
+	<ServersManageFilesPage
+		:show-debug-info="flags.advancedDebugInfo"
+		:show-refresh-button="flags.FilesRefreshButton"
+	/>
 </template>

apps/frontend/src/utils/muralpay-rails.ts

@@ -13,6 +13,10 @@ export interface FieldConfig {
 	pattern?: string
 	validate?: (value: string) => string | null
 	autocomplete?: string
+	dependsOn?: {
+		field: string
+		value?: string | string[]
+	}
 }
 
 export interface RailConfig {
@@ -330,6 +334,10 @@ export const MURALPAY_RAILS: Record<string, RailConfig> = {
 					defaultMessage: 'Enter PIX email',
 				}),
 				autocomplete: 'email',
+				dependsOn: {
+					field: 'pixAccountType',
+					value: 'EMAIL',
+				},
 			},
 			{
 				name: 'pixPhone',
@@ -341,6 +349,10 @@ export const MURALPAY_RAILS: Record<string, RailConfig> = {
 					defaultMessage: '+55...',
 				}),
 				autocomplete: 'tel',
+				dependsOn: {
+					field: 'pixAccountType',
+					value: 'PHONE',
+				},
 			},
 			{
 				name: 'branchCode',
@@ -352,6 +364,10 @@ export const MURALPAY_RAILS: Record<string, RailConfig> = {
 					defaultMessage: 'Enter branch code',
 				}),
 				autocomplete: 'off',
+				dependsOn: {
+					field: 'pixAccountType',
+					value: 'BANK_ACCOUNT',
+				},
 			},
 			{
 				name: 'documentNumber',
@@ -367,6 +383,10 @@ export const MURALPAY_RAILS: Record<string, RailConfig> = {
 					defaultMessage: 'Brazilian tax identification number',
 				}),
 				autocomplete: 'off',
+				dependsOn: {
+					field: 'pixAccountType',
+					value: 'DOCUMENT',
+				},
 			},
 		],
 	},

apps/labrinth/.env.docker-compose

@@ -16,14 +16,9 @@ DATABASE_URL=postgresql://labrinth:labrinth@labrinth-postgres/labrinth
 DATABASE_MIN_CONNECTIONS=0
 DATABASE_MAX_CONNECTIONS=16
 
-SEARCH_BACKEND=meilisearch
 MEILISEARCH_READ_ADDR=http://localhost:7700
 MEILISEARCH_WRITE_ADDRS=http://localhost:7700
 MEILISEARCH_KEY=modrinth
-ELASTICSEARCH_URL=http://localhost:9200
-ELASTICSEARCH_INDEX_PREFIX=labrinth
-ELASTICSEARCH_USERNAME=elastic
-ELASTICSEARCH_PASSWORD=elastic
 
 REDIS_URL=redis://labrinth-redis
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/.env.local

@@ -16,29 +16,16 @@ DATABASE_URL=postgresql://labrinth:labrinth@localhost/labrinth
 DATABASE_MIN_CONNECTIONS=0
 DATABASE_MAX_CONNECTIONS=16
 
-SEARCH_BACKEND=meilisearch
-
-# Meilisearch configuration
 MEILISEARCH_READ_ADDR=http://localhost:7700
 MEILISEARCH_WRITE_ADDRS=http://localhost:7700
 # 5 minutes in milliseconds
 SEARCH_OPERATION_TIMEOUT=300000
 
-ELASTICSEARCH_URL=http://localhost:9200
-ELASTICSEARCH_INDEX_PREFIX=labrinth
-
 # # For a sharded Meilisearch setup (sharded-meilisearch docker compose profile)
 # MEILISEARCH_READ_ADDR=http://localhost:7710
 # MEILISEARCH_WRITE_ADDRS=http://localhost:7700,http://localhost:7701
 
 MEILISEARCH_KEY=modrinth
-MEILISEARCH_META_NAMESPACE=
-
-# Elasticsearch configuration
-ELASTICSEARCH_URL=http://localhost:9200
-ELASTICSEARCH_INDEX_PREFIX=labrinth
-ELASTICSEARCH_USERNAME=
-ELASTICSEARCH_PASSWORD=
 
 REDIS_URL=redis://localhost
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/Cargo.toml

@@ -25,7 +25,7 @@ async-stripe = { workspace = true, features = [
   "billing",
   "checkout",
   "connect",
-  "webhook-events"
+  "webhook-events",
 ] }
 async-trait = { workspace = true }
 base64 = { workspace = true }
@@ -43,7 +43,6 @@ deadpool-redis.workspace = true
 derive_more = { workspace = true, features = ["deref", "deref_mut"] }
 dotenvy = { workspace = true }
 either = { workspace = true }
-elasticsearch = { workspace = true, features = ["experimental-apis"] }
 eyre = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }
@@ -86,11 +85,11 @@ reqwest = { workspace = true, features = [
   "http2",
   "json",
   "multipart",
-  "rustls-tls-webpki-roots"
+  "rustls-tls-webpki-roots",
 ] }
 rust_decimal = { workspace = true, features = [
   "serde-with-float",
-  "serde-with-str"
+  "serde-with-str",
 ] }
 rust_iso3166 = { workspace = true }
 rust-s3 = { workspace = true }

apps/labrinth/src/auth/mod.rs

@@ -20,6 +20,8 @@ use thiserror::Error;
 pub enum AuthenticationError {
     #[error(transparent)]
     Internal(#[from] eyre::Report),
+    #[error("Environment Error")]
+    Env(#[from] dotenvy::Error),
     #[error("An unknown database error occurred: {0}")]
     Sqlx(#[from] sqlx::Error),
     #[error("Database Error: {0}")]
@@ -56,6 +58,7 @@ impl actix_web::ResponseError for AuthenticationError {
             AuthenticationError::Internal(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
+            AuthenticationError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
             AuthenticationError::Sqlx(..) => StatusCode::INTERNAL_SERVER_ERROR,
             AuthenticationError::Database(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
@@ -91,6 +94,7 @@ impl AuthenticationError {
     pub fn error_name(&self) -> &'static str {
         match self {
             AuthenticationError::Internal(..) => "internal_error",
+            AuthenticationError::Env(..) => "environment_error",
             AuthenticationError::Sqlx(..) => "database_error",
             AuthenticationError::Database(..) => "database_error",
             AuthenticationError::SerDe(..) => "invalid_input",

apps/labrinth/src/background_task.rs

@@ -1,4 +1,3 @@
-use crate::database;
 use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::queue::billing::{index_billing, index_subscriptions};
@@ -8,9 +7,9 @@ use crate::queue::payouts::{
     insert_bank_balances_and_webhook, process_affiliate_payouts,
     process_payout, remove_payouts_for_refunded_charges,
 };
-use crate::search::SearchBackend;
+use crate::search::indexing::index_projects;
 use crate::util::anrok;
-use actix_web::web;
+use crate::{database, search};
 use clap::ValueEnum;
 use tracing::{error, info, warn};
 
@@ -35,19 +34,18 @@ impl BackgroundTask {
         pool: PgPool,
         ro_pool: PgPool,
         redis_pool: RedisPool,
-        search_backend: web::Data<dyn SearchBackend>,
+        search_config: search::SearchConfig,
         clickhouse: clickhouse::Client,
         stripe_client: stripe::Client,
         anrok_client: anrok::Client,
         email_queue: EmailQueue,
         mural_client: muralpay::Client,
-    ) -> eyre::Result<()> {
+    ) {
         use BackgroundTask::*;
-        // TODO: all of these tasks should return `eyre::Result`s
         match self {
             Migrations => run_migrations().await,
             IndexSearch => {
-                return index_search(ro_pool, redis_pool, search_backend).await;
+                index_search(ro_pool, redis_pool, search_config).await
             }
             ReleaseScheduled => release_scheduled(pool).await,
             UpdateVersions => update_versions(pool, redis_pool).await,
@@ -79,7 +77,6 @@ impl BackgroundTask {
                 run_email(email_queue).await;
             }
         }
-        Ok(())
     }
 }
 
@@ -125,10 +122,14 @@ pub async fn run_migrations() {
 pub async fn index_search(
     ro_pool: PgPool,
     redis_pool: RedisPool,
-    search_backend: web::Data<dyn SearchBackend>,
-) -> eyre::Result<()> {
+    search_config: search::SearchConfig,
+) {
     info!("Indexing local database");
-    search_backend.index_projects(ro_pool, redis_pool).await
+    let result = index_projects(ro_pool, redis_pool, &search_config).await;
+    if let Err(e) = result {
+        warn!("Local project indexing failed: {:?}", e);
+    }
+    info!("Done indexing local database");
 }
 
 pub async fn release_scheduled(pool: PgPool) {

apps/labrinth/src/env.rs

@@ -82,7 +82,6 @@ where
 }
 
 pub fn init() -> eyre::Result<()> {
-    dotenvy::dotenv().ok();
     EnvVars::from_env()?;
     LazyLock::force(&ENV);
     Ok(())
@@ -129,6 +128,9 @@ vars! {
     LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
     RATE_LIMIT_IGNORE_KEY: String;
     DATABASE_URL: String;
+    MEILISEARCH_READ_ADDR: String;
+    MEILISEARCH_WRITE_ADDRS: StringCsv;
+    MEILISEARCH_KEY: String;
     REDIS_URL: String;
     BIND_ADDR: String;
     SELF_ADDR: String;
@@ -140,17 +142,6 @@ vars! {
     ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
     ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
 
-    // search
-    SEARCH_BACKEND: crate::search::SearchBackendKind;
-    MEILISEARCH_READ_ADDR: String;
-    MEILISEARCH_WRITE_ADDRS: StringCsv;
-    MEILISEARCH_KEY: String;
-    ELASTICSEARCH_URL: String;
-    ELASTICSEARCH_INDEX_PREFIX: String;
-    ELASTICSEARCH_USERNAME: String = "";
-    ELASTICSEARCH_PASSWORD: String = "";
-    ELASTICSEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
-
     // storage
     STORAGE_BACKEND: crate::file_hosting::FileHostKind;
 

apps/labrinth/src/lib.rs

@@ -56,7 +56,7 @@ pub struct LabrinthConfig {
     pub file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
     pub scheduler: Arc<scheduler::Scheduler>,
     pub ip_salt: Pepper,
-    pub search_backend: web::Data<dyn search::SearchBackend>,
+    pub search_config: search::SearchConfig,
     pub session_queue: web::Data<AuthQueue>,
     pub payouts_queue: web::Data<PayoutsQueue>,
     pub analytics_queue: Arc<AnalyticsQueue>,
@@ -75,7 +75,7 @@ pub fn app_setup(
     pool: PgPool,
     ro_pool: ReadOnlyPgPool,
     redis_pool: RedisPool,
-    search_backend: actix_web::web::Data<dyn search::SearchBackend>,
+    search_config: search::SearchConfig,
     clickhouse: &mut Client,
     file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
     stripe_client: stripe::Client,
@@ -113,22 +113,19 @@ pub fn app_setup(
         let local_index_interval =
             Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
         let pool_ref = pool.clone();
+        let search_config_ref = search_config.clone();
         let redis_pool_ref = redis_pool.clone();
-        let search_backend_ref = search_backend.clone();
         scheduler.run(local_index_interval, move || {
             let pool_ref = pool_ref.clone();
             let redis_pool_ref = redis_pool_ref.clone();
-            let search_backend = search_backend_ref.clone();
+            let search_config_ref = search_config_ref.clone();
             async move {
-                if let Err(err) = background_task::index_search(
+                background_task::index_search(
                     pool_ref,
                     redis_pool_ref,
-                    search_backend,
+                    search_config_ref,
                 )
-                .await
-                {
-                    warn!("Failed to index search: {err:?}");
-                }
+                .await;
             }
         });
 
@@ -272,7 +269,7 @@ pub fn app_setup(
         file_host,
         scheduler: Arc::new(scheduler),
         ip_salt,
-        search_backend,
+        search_config,
         session_queue,
         payouts_queue: web::Data::new(PayoutsQueue::new()),
         analytics_queue,
@@ -310,7 +307,7 @@ pub fn app_config(
     .app_data(web::Data::new(labrinth_config.pool.clone()))
     .app_data(web::Data::new(labrinth_config.ro_pool.clone()))
     .app_data(web::Data::new(labrinth_config.file_host.clone()))
-    .app_data(labrinth_config.search_backend.clone())
+    .app_data(web::Data::new(labrinth_config.search_config.clone()))
     .app_data(web::Data::new(labrinth_config.gotenberg_client.clone()))
     .app_data(labrinth_config.session_queue.clone())
     .app_data(labrinth_config.payouts_queue.clone())

apps/labrinth/src/main.rs

@@ -17,7 +17,6 @@ use labrinth::utoipa_app_config;
 use labrinth::{app_config, env};
 use labrinth::{clickhouse, database, file_hosting};
 use std::ffi::CStr;
-use std::io;
 use std::sync::Arc;
 use tracing::{Instrument, info, info_span};
 use tracing_actix_web::TracingLogger;
@@ -57,6 +56,7 @@ struct Args {
 
 fn main() -> std::io::Result<()> {
     color_eyre::install().expect("failed to install `color-eyre`");
+    dotenvy::dotenv().ok();
     modrinth_util::log::init().expect("failed to initialize logging");
     env::init().expect("failed to initialize environment variables");
 
@@ -152,8 +152,7 @@ async fn app() -> std::io::Result<()> {
     info!("Initializing clickhouse connection");
     let mut clickhouse = clickhouse::init_client().await.unwrap();
 
-    let search_backend =
-        actix_web::web::Data::from(Arc::from(search::backend(None)));
+    let search_config = search::SearchConfig::new(None);
 
     let stripe_client = stripe::Client::new(ENV.STRIPE_API_KEY.clone());
 
@@ -168,20 +167,19 @@ async fn app() -> std::io::Result<()> {
 
     if let Some(task) = args.run_background_task {
         info!("Running task {task:?} and exiting");
-        return task
-            .run(
-                pool,
-                ro_pool.into_inner(),
-                redis_pool,
-                search_backend,
-                clickhouse,
-                stripe_client,
-                anrok_client.clone(),
-                email_queue,
-                muralpay,
-            )
-            .await
-            .map_err(io::Error::other);
+        task.run(
+            pool,
+            ro_pool.into_inner(),
+            redis_pool,
+            search_config,
+            clickhouse,
+            stripe_client,
+            anrok_client.clone(),
+            email_queue,
+            muralpay,
+        )
+        .await;
+        return Ok(());
     }
 
     let prometheus = PrometheusMetricsBuilder::new("labrinth")
@@ -208,7 +206,7 @@ async fn app() -> std::io::Result<()> {
         pool.clone(),
         ro_pool.clone(),
         redis_pool.clone(),
-        search_backend.clone(),
+        search_config.clone(),
         &mut clickhouse,
         file_host.clone(),
         stripe_client,

apps/labrinth/src/queue/email.rs

@@ -102,6 +102,8 @@ impl Mailer {
 
 #[derive(Error, Debug)]
 pub enum MailError {
+    #[error("Environment Error")]
+    Env(#[from] dotenvy::Error),
     #[error("Mail Error: {0}")]
     Mail(#[from] lettre::error::Error),
     #[error("Address Parse Error: {0}")]
@@ -134,7 +136,7 @@ impl EmailQueue {
             pg,
             redis,
             mailer: Arc::new(TokioMutex::new(Mailer::Uninitialized)),
-            identity: templates::MailingIdentity::from_env(),
+            identity: templates::MailingIdentity::from_env()?,
             client: Client::builder()
                 .user_agent("Modrinth")
                 .build()

apps/labrinth/src/queue/email/templates.rs

@@ -95,8 +95,8 @@ pub struct MailingIdentity {
 }
 
 impl MailingIdentity {
-    pub fn from_env() -> Self {
-        Self {
+    pub fn from_env() -> dotenvy::Result<Self> {
+        Ok(Self {
             from_name: ENV.SMTP_FROM_NAME.clone(),
             from_address: ENV.SMTP_FROM_ADDRESS.clone(),
             reply_name: if ENV.SMTP_REPLY_TO_NAME.is_empty() {
@@ -109,7 +109,7 @@ impl MailingIdentity {
             } else {
                 Some(ENV.SMTP_REPLY_TO_ADDRESS.clone())
             },
-        }
+        })
     }
 }
 

apps/labrinth/src/routes/internal/admin.rs

@@ -7,7 +7,7 @@ use crate::models::pats::Scopes;
 use crate::queue::analytics::AnalyticsQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
-use crate::search::SearchBackend;
+use crate::search::SearchConfig;
 use crate::util::date::get_current_tenths_of_ms;
 use crate::util::guards::admin_key_guard;
 use actix_web::{HttpRequest, HttpResponse, patch, post, web};
@@ -152,12 +152,10 @@ pub async fn count_download(
 pub async fn force_reindex(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_backend: web::Data<dyn SearchBackend>,
+    config: web::Data<SearchConfig>,
 ) -> Result<HttpResponse, ApiError> {
+    use crate::search::indexing::index_projects;
     let redis = redis.get_ref();
-    search_backend
-        .index_projects(pool.as_ref().clone(), redis.clone())
-        .await
-        .map_err(ApiError::Internal)?;
+    index_projects(pool.as_ref().clone(), redis.clone(), &config).await?;
     Ok(HttpResponse::NoContent().finish())
 }

apps/labrinth/src/routes/internal/search.rs

@@ -1,9 +1,12 @@
+use crate::routes::ApiError;
+use crate::search::SearchConfig;
 use crate::util::guards::admin_key_guard;
-use crate::{
-    routes::ApiError,
-    search::{SearchBackend, TasksCancelFilter},
-};
-use actix_web::{delete, get, web};
+use actix_web::{HttpResponse, delete, get, web};
+use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::time::Duration;
+use utoipa::ToSchema;
 
 pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
     cfg.service(tasks).service(tasks_cancel);
@@ -12,20 +15,107 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
 #[utoipa::path]
 #[get("tasks", guard = "admin_key_guard")]
 pub async fn tasks(
-    search: web::Data<dyn SearchBackend>,
-) -> Result<web::Json<serde_json::Value>, ApiError> {
-    Ok(web::Json(search.tasks().await.map_err(ApiError::Internal)?))
+    config: web::Data<SearchConfig>,
+) -> Result<HttpResponse, ApiError> {
+    let client = config.make_batch_client()?;
+    let tasks = client
+        .with_all_clients("get_tasks", async |client| {
+            let tasks = client.get_tasks().await?;
+
+            Ok(tasks.results)
+        })
+        .await?;
+
+    #[derive(Serialize, ToSchema)]
+    struct MeiliTask<Time> {
+        uid: u32,
+        status: &'static str,
+        duration: Option<Duration>,
+        enqueued_at: Option<Time>,
+    }
+
+    #[derive(Serialize, ToSchema)]
+    struct TaskList<Time> {
+        by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
+    }
+
+    let response = tasks
+        .into_iter()
+        .enumerate()
+        .map(|(idx, instance_tasks)| {
+            let tasks = instance_tasks
+                .into_iter()
+                .filter_map(|task| {
+                    Some(match task {
+                        Task::Enqueued { content } => MeiliTask {
+                            uid: content.uid,
+                            status: "enqueued",
+                            duration: None,
+                            enqueued_at: Some(content.enqueued_at),
+                        },
+                        Task::Processing { content } => MeiliTask {
+                            uid: content.uid,
+                            status: "processing",
+                            duration: None,
+                            enqueued_at: Some(content.enqueued_at),
+                        },
+                        Task::Failed { content } => MeiliTask {
+                            uid: content.task.uid,
+                            status: "failed",
+                            duration: Some(content.task.duration),
+                            enqueued_at: Some(content.task.enqueued_at),
+                        },
+                        Task::Succeeded { content: _ } => return None,
+                    })
+                })
+                .collect();
+
+            (idx.to_string(), tasks)
+        })
+        .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
+
+    Ok(HttpResponse::Ok().json(TaskList {
+        by_instance: response,
+    }))
+}
+
+#[derive(Deserialize, Serialize, ToSchema)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum TasksCancelFilter {
+    All,
+    AllEnqueued,
+    Indexes { indexes: Vec<String> },
 }
 
 #[utoipa::path]
 #[delete("tasks", guard = "admin_key_guard")]
 pub async fn tasks_cancel(
-    search: web::Data<dyn SearchBackend>,
+    config: web::Data<SearchConfig>,
     body: web::Json<TasksCancelFilter>,
-) -> Result<(), ApiError> {
-    search
-        .tasks_cancel(&body)
-        .await
-        .map_err(ApiError::Internal)?;
-    Ok(())
+) -> Result<HttpResponse, ApiError> {
+    let client = config.make_batch_client()?;
+    let all_results = client
+        .with_all_clients("cancel_tasks", async |client| {
+            let mut q = TasksCancelQuery::new(client);
+            match &body.0 {
+                TasksCancelFilter::All => {}
+                TasksCancelFilter::Indexes { indexes } => {
+                    q.with_index_uids(indexes.iter().map(|s| s.as_str()));
+                }
+                TasksCancelFilter::AllEnqueued => {
+                    q.with_statuses(["enqueued"]);
+                }
+            };
+
+            let result = client.cancel_tasks_with(&q).await;
+
+            Ok(result)
+        })
+        .await?;
+
+    for r in all_results {
+        r?;
+    }
+
+    Ok(HttpResponse::Ok().finish())
 }

apps/labrinth/src/routes/mod.rs

@@ -95,6 +95,8 @@ pub enum ApiError {
     Auth(eyre::Report),
     #[error("Invalid input: {0}")]
     InvalidInput(String),
+    #[error("Environment error")]
+    Env(#[from] dotenvy::Error),
     #[error("Error while uploading file: {0}")]
     FileHosting(#[from] FileHostingError),
     #[error("database error")]
@@ -117,6 +119,8 @@ pub enum ApiError {
     Validation(String),
     #[error("Search error: {0}")]
     Search(#[from] meilisearch_sdk::errors::Error),
+    #[error("search indexing error")]
+    Indexing(#[from] crate::search::indexing::IndexingError),
     #[error("Payments error: {0}")]
     Payments(String),
     #[error("Discord error: {0}")]
@@ -174,6 +178,7 @@ impl ApiError {
                 Self::Internal(..) => "internal_error",
                 Self::Request(..) => "request_error",
                 Self::Auth(..) => "auth_error",
+                Self::Env(..) => "environment_error",
                 Self::Database(..) => "database_error",
                 Self::SqlxDatabase(..) => "database_error",
                 Self::RedisDatabase(..) => "database_error",
@@ -182,6 +187,7 @@ impl ApiError {
                 Self::Xml(..) => "xml_error",
                 Self::Json(..) => "json_error",
                 Self::Search(..) => "search_error",
+                Self::Indexing(..) => "indexing_error",
                 Self::FileHosting(..) => "file_hosting_error",
                 Self::InvalidInput(..) => "invalid_input",
                 Self::Validation(..) => "invalid_input",
@@ -237,6 +243,7 @@ impl actix_web::ResponseError for ApiError {
             Self::Request(..) => StatusCode::BAD_REQUEST,
             Self::Auth(..) => StatusCode::UNAUTHORIZED,
             Self::InvalidInput(..) => StatusCode::BAD_REQUEST,
+            Self::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::SqlxDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::RedisDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
@@ -246,6 +253,7 @@ impl actix_web::ResponseError for ApiError {
             Self::Xml(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Json(..) => StatusCode::BAD_REQUEST,
             Self::Search(..) => StatusCode::INTERNAL_SERVER_ERROR,
+            Self::Indexing(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::FileHosting(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Validation(..) => StatusCode::BAD_REQUEST,
             Self::Payments(..) => StatusCode::FAILED_DEPENDENCY,

apps/labrinth/src/routes/v2/projects.rs

@@ -14,7 +14,7 @@ use crate::queue::moderation::AutomatedModerationQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::v3::projects::ProjectIds;
 use crate::routes::{ApiError, v2_reroute, v3};
-use crate::search::SearchBackend;
+use crate::search::{SearchConfig, SearchError, search_for_project};
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -53,8 +53,8 @@ pub fn config(cfg: &mut web::ServiceConfig) {
 #[get("search")]
 pub async fn project_search(
     web::Query(info): web::Query<SearchRequest>,
-    search_backend: web::Data<dyn SearchBackend>,
-) -> Result<HttpResponse, ApiError> {
+    config: web::Data<SearchConfig>,
+) -> Result<HttpResponse, SearchError> {
     // Search now uses loader_fields instead of explicit 'client_side' and 'server_side' fields
     // While the backend for this has changed, it doesnt affect much
     // in the API calls except that 'versions:x' is now 'game_versions:x'
@@ -99,10 +99,7 @@ pub async fn project_search(
         ..info
     };
 
-    let results = search_backend
-        .search_for_project(&info)
-        .await
-        .map_err(ApiError::Internal)?;
+    let results = search_for_project(&info, &config).await?;
 
     let results = LegacySearchResults::from(results);
 
@@ -414,7 +411,7 @@ pub async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
     new_project: web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -524,7 +521,7 @@ pub async fn project_edit(
         req.clone(),
         info,
         pool.clone(),
-        search_backend,
+        search_config,
         web::Json(new_project),
         redis.clone(),
         session_queue.clone(),
@@ -918,7 +915,7 @@ pub async fn project_delete(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
@@ -927,7 +924,7 @@ pub async fn project_delete(
         info,
         pool,
         redis,
-        search_backend,
+        search_config,
         session_queue,
     )
     .await

apps/labrinth/src/routes/v2/versions.rs

@@ -11,7 +11,7 @@ use crate::models::projects::{
 use crate::models::v2::projects::LegacyVersion;
 use crate::queue::session::AuthQueue;
 use crate::routes::{v2_reroute, v3};
-use crate::search::SearchBackend;
+use crate::search::SearchConfig;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
@@ -349,7 +349,7 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so we don't need to convert the response
     v3::versions::version_delete(
@@ -358,7 +358,7 @@ pub async fn version_delete(
         pool,
         redis,
         session_queue,
-        search_backend,
+        search_config,
     )
     .await
     .or_else(v2_reroute::flatten_404_error)

apps/labrinth/src/routes/v3/project_creation.rs

@@ -21,6 +21,7 @@ use crate::models::teams::{OrganizationPermissions, ProjectPermissions};
 use crate::models::threads::ThreadType;
 use crate::models::v3::user_limits::UserLimits;
 use crate::queue::session::AuthQueue;
+use crate::search::indexing::IndexingError;
 use crate::util::guards::admin_key_guard;
 use crate::util::img::upload_image_optimized;
 use crate::util::routes::read_from_field;
@@ -48,10 +49,14 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
 
 #[derive(Error, Debug)]
 pub enum CreateError {
+    #[error("Environment Error")]
+    EnvError(#[from] dotenvy::Error),
     #[error("An unknown database error occurred")]
     SqlxDatabaseError(#[from] sqlx::Error),
     #[error("Database Error: {0}")]
     DatabaseError(#[from] models::DatabaseError),
+    #[error("Indexing Error: {0}")]
+    IndexingError(#[from] IndexingError),
     #[error("Error while parsing multipart payload: {0}")]
     MultipartError(#[from] actix_multipart::MultipartError),
     #[error("Error while parsing JSON: {0}")]
@@ -91,10 +96,12 @@ pub enum CreateError {
 impl actix_web::ResponseError for CreateError {
     fn status_code(&self) -> StatusCode {
         match self {
+            CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR,
             CreateError::SqlxDatabaseError(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
             CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
+            CreateError::IndexingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
             CreateError::FileHostingError(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
@@ -122,8 +129,10 @@ impl actix_web::ResponseError for CreateError {
     fn error_response(&self) -> HttpResponse {
         HttpResponse::build(self.status_code()).json(ApiError {
             error: match self {
+                CreateError::EnvError(..) => "environment_error",
                 CreateError::SqlxDatabaseError(..) => "database_error",
                 CreateError::DatabaseError(..) => "database_error",
+                CreateError::IndexingError(..) => "indexing_error",
                 CreateError::FileHostingError(..) => "file_hosting_error",
                 CreateError::SerDeError(..) => "invalid_input",
                 CreateError::MultipartError(..) => "invalid_input",

apps/labrinth/src/routes/v3/projects.rs

@@ -29,7 +29,8 @@ use crate::queue::moderation::AutomatedModerationQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::routes::internal::delphi;
-use crate::search::{SearchBackend, SearchResults};
+use crate::search::indexing::remove_documents;
+use crate::search::{SearchConfig, SearchError, search_for_project};
 use crate::util::error::Context;
 use crate::util::img;
 use crate::util::img::{delete_old_images, upload_image_optimized};
@@ -263,7 +264,7 @@ pub async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
     new_project: web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -973,16 +974,16 @@ pub async fn project_edit(
         project_item.inner.status.is_searchable(),
         new_project.status.map(|status| status.is_searchable()),
     ) {
-        search_backend
-            .remove_documents(
-                &project_item
-                    .versions
-                    .into_iter()
-                    .map(|x| x.into())
-                    .collect::<Vec<_>>(),
-            )
-            .await
-            .wrap_internal_err("failed to remove documents")?;
+        remove_documents(
+            &project_item
+                .versions
+                .into_iter()
+                .map(|x| x.into())
+                .collect::<Vec<_>>(),
+            &search_config,
+        )
+        .await
+        .wrap_internal_err("failed to remove documents")?;
     }
 
     Ok(HttpResponse::NoContent().body(""))
@@ -1038,12 +1039,9 @@ pub async fn edit_project_categories(
 
 pub async fn project_search(
     web::Query(info): web::Query<SearchRequest>,
-    search_backend: web::Data<dyn SearchBackend>,
-) -> Result<web::Json<SearchResults>, ApiError> {
-    let results = search_backend
-        .search_for_project(&info)
-        .await
-        .map_err(ApiError::Internal)?;
+    config: web::Data<SearchConfig>,
+) -> Result<HttpResponse, SearchError> {
+    let results = search_for_project(&info, &config).await?;
 
     // TODO: add this back
     // let results = ReturnSearchResults {
@@ -1057,7 +1055,7 @@ pub async fn project_search(
     //     total_hits: results.total_hits,
     // };
 
-    Ok(web::Json(results))
+    Ok(HttpResponse::Ok().json(results))
 }
 
 //checks the validity of a project id or slug
@@ -2160,7 +2158,7 @@ pub async fn project_delete(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<(), ApiError> {
     let (_, user) = get_user_from_headers(
@@ -2272,16 +2270,16 @@ pub async fn project_delete(
         .await
         .wrap_internal_err("failed to commit transaction")?;
 
-    search_backend
-        .remove_documents(
-            &project
-                .versions
-                .into_iter()
-                .map(|x| x.into())
-                .collect::<Vec<_>>(),
-        )
-        .await
-        .wrap_internal_err("failed to remove project version documents")?;
+    remove_documents(
+        &project
+            .versions
+            .into_iter()
+            .map(|x| x.into())
+            .collect::<Vec<_>>(),
+        &search_config,
+    )
+    .await
+    .wrap_internal_err("failed to remove project version documents")?;
 
     if result.is_some() {
         Ok(())

apps/labrinth/src/routes/v3/versions.rs

@@ -26,7 +26,8 @@ use crate::models::projects::{Loader, skip_nulls};
 use crate::models::teams::ProjectPermissions;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
-use crate::search::SearchBackend;
+use crate::search::SearchConfig;
+use crate::search::indexing::remove_documents;
 use crate::util::error::Context;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
@@ -893,7 +894,7 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_backend: web::Data<dyn SearchBackend>,
+    search_config: web::Data<SearchConfig>,
 ) -> Result<HttpResponse, ApiError> {
     let user = get_user_from_headers(
         &req,
@@ -1000,10 +1001,10 @@ pub async fn version_delete(
         &redis,
     )
     .await?;
-    search_backend
-        .remove_documents(&[version.inner.id.into()])
+    remove_documents(&[version.inner.id.into()], &search_config)
         .await
         .wrap_internal_err("failed to remove documents")?;
+
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))
     } else {

apps/labrinth/src/search/backend/meilisearch/mod.rs

@@ -1,426 +0,0 @@
-use crate::database::PgPool;
-use crate::database::redis::RedisPool;
-use crate::env::ENV;
-use crate::models::ids::VersionId;
-use crate::models::projects::SearchRequest;
-use crate::routes::ApiError;
-use crate::search::{
-    ResultSearchProject, SearchBackend, SearchResults, TasksCancelFilter,
-};
-use crate::util::error::Context;
-use async_trait::async_trait;
-use eyre::eyre;
-use futures::TryStreamExt;
-use futures::stream::FuturesOrdered;
-use itertools::Itertools;
-use meilisearch_sdk::client::Client;
-use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
-use serde::Serialize;
-use serde_json::Value;
-use std::borrow::Cow;
-use std::collections::HashMap;
-use std::fmt::Write;
-use std::time::Duration;
-use tracing::{Instrument, info_span};
-
-pub mod indexing;
-
-#[derive(Debug, Clone)]
-pub struct MeilisearchReadClient {
-    pub client: Client,
-}
-
-impl std::ops::Deref for MeilisearchReadClient {
-    type Target = Client;
-
-    fn deref(&self) -> &Self::Target {
-        &self.client
-    }
-}
-
-pub struct BatchClient {
-    pub clients: Vec<Client>,
-}
-
-impl BatchClient {
-    pub fn new(clients: Vec<Client>) -> Self {
-        Self { clients }
-    }
-
-    pub async fn with_all_clients<'a, T, G, Fut>(
-        &'a self,
-        task_name: &str,
-        generator: G,
-    ) -> Result<Vec<T>, indexing::IndexingError>
-    where
-        G: Fn(&'a Client) -> Fut,
-        Fut: Future<Output = Result<T, indexing::IndexingError>> + 'a,
-    {
-        let mut tasks = FuturesOrdered::new();
-        for (idx, client) in self.clients.iter().enumerate() {
-            tasks.push_back(generator(client).instrument(info_span!(
-                "client_task",
-                task.name = task_name,
-                client.idx = idx,
-            )));
-        }
-
-        let results = tasks.try_collect::<Vec<T>>().await?;
-        Ok(results)
-    }
-
-    pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
-    where
-        F: FnMut(T, &Client) -> R,
-    {
-        assert_eq!(
-            data.len(),
-            self.clients.len(),
-            "mismatch between data len and meilisearch client count"
-        );
-        self.clients
-            .iter()
-            .zip(data)
-            .map(|(client, item)| predicate(item, client))
-            .collect()
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct MeilisearchConfig {
-    pub addresses: Vec<String>,
-    pub read_lb_address: String,
-    pub key: String,
-    pub meta_namespace: String,
-}
-
-impl MeilisearchConfig {
-    pub fn new(meta_namespace: Option<String>) -> Self {
-        Self {
-            addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
-            key: ENV.MEILISEARCH_KEY.clone(),
-            meta_namespace: meta_namespace.unwrap_or_default(),
-            read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
-        }
-    }
-
-    pub fn make_loadbalanced_read_client(
-        &self,
-    ) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
-        Ok(MeilisearchReadClient {
-            client: Client::new(&self.read_lb_address, Some(&self.key))?,
-        })
-    }
-
-    pub fn make_batch_client(
-        &self,
-    ) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
-        Ok(BatchClient::new(
-            self.addresses
-                .iter()
-                .map(|address| {
-                    Client::new(address.as_str(), Some(self.key.as_str()))
-                })
-                .collect::<Result<Vec<_>, _>>()?,
-        ))
-    }
-
-    pub fn get_index_name(&self, index: &str, next: bool) -> String {
-        let alt = if next { "_alt" } else { "" };
-        format!("{}_{}_{}", self.meta_namespace, index, alt)
-    }
-}
-
-pub struct Meilisearch {
-    pub config: MeilisearchConfig,
-}
-
-impl Meilisearch {
-    pub fn new(config: MeilisearchConfig) -> Self {
-        Self { config }
-    }
-
-    fn get_sort_index(
-        &self,
-        index: &str,
-    ) -> Result<(String, &'static [&'static str]), ApiError> {
-        let projects_name = self.config.get_index_name("projects", false);
-        let projects_filtered_name =
-            self.config.get_index_name("projects_filtered", false);
-        Ok(match index {
-            "relevance" => (projects_name, &["downloads:desc"]),
-            "downloads" => (projects_filtered_name, &["downloads:desc"]),
-            "follows" => (projects_name, &["follows:desc"]),
-            "updated" => (projects_name, &["date_modified:desc"]),
-            "newest" => (projects_name, &["date_created:desc"]),
-            _ => {
-                return Err(ApiError::Request(eyre!(
-                    "invalid index `{index}`"
-                )));
-            }
-        })
-    }
-}
-
-#[async_trait]
-impl SearchBackend for Meilisearch {
-    async fn search_for_project(
-        &self,
-        info: &SearchRequest,
-    ) -> eyre::Result<SearchResults> {
-        let offset: usize = info
-            .offset
-            .as_deref()
-            .unwrap_or("0")
-            .parse()
-            .wrap_request_err("invalid offset")?;
-        let index = info.index.as_deref().unwrap_or("relevance");
-        let limit = info
-            .limit
-            .as_deref()
-            .unwrap_or("10")
-            .parse::<usize>()
-            .wrap_request_err("invalid limit")?
-            .min(100);
-
-        let (index_name, sort_name) = self.get_sort_index(index)?;
-        let client = self
-            .config
-            .make_loadbalanced_read_client()
-            .wrap_internal_err("failed to make load-balanced read client")?;
-        let meilisearch_index = client
-            .get_index(index_name)
-            .await
-            .wrap_internal_err("failed to get index")?;
-
-        let mut filter_string = String::new();
-
-        let hits_per_page = if limit == 0 { 1 } else { limit };
-
-        let page = offset / hits_per_page + 1;
-
-        let results = {
-            let mut query = meilisearch_index.search();
-            query
-                .with_page(page)
-                .with_hits_per_page(hits_per_page)
-                .with_query(info.query.as_deref().unwrap_or_default())
-                .with_sort(sort_name);
-
-            if let Some(new_filters) = info.new_filters.as_deref() {
-                query.with_filter(new_filters);
-            } else {
-                let facets = if let Some(facets) = &info.facets {
-                    let facets =
-                        serde_json::from_str::<Vec<Vec<Value>>>(facets)
-                            .wrap_request_err("failed to parse facets")?;
-                    Some(facets)
-                } else {
-                    None
-                };
-
-                let filters: Cow<_> =
-                    match (info.filters.as_deref(), info.version.as_deref()) {
-                        (Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
-                        (Some(f), None) => f.into(),
-                        (None, Some(v)) => v.into(),
-                        (None, None) => "".into(),
-                    };
-
-                if let Some(facets) = facets {
-                    let facets: Vec<Vec<Vec<String>>> =
-                        facets
-                            .into_iter()
-                            .map(|facets| {
-                                facets
-								.into_iter()
-								.map(|facet| {
-									if facet.is_array() {
-										serde_json::from_value::<Vec<String>>(facet)
-											.unwrap_or_default()
-									} else {
-										vec![
-											serde_json::from_value::<String>(facet)
-												.unwrap_or_default(),
-										]
-									}
-								})
-								.collect_vec()
-                            })
-                            .collect_vec();
-
-                    filter_string.push('(');
-                    for (index, facet_outer_list) in facets.iter().enumerate() {
-                        filter_string.push('(');
-
-                        for (facet_outer_index, facet_inner_list) in
-                            facet_outer_list.iter().enumerate()
-                        {
-                            filter_string.push('(');
-                            for (facet_inner_index, facet) in
-                                facet_inner_list.iter().enumerate()
-                            {
-                                filter_string
-                                    .push_str(&facet.replace(':', " = "));
-                                if facet_inner_index
-                                    != (facet_inner_list.len() - 1)
-                                {
-                                    filter_string.push_str(" AND ")
-                                }
-                            }
-                            filter_string.push(')');
-
-                            if facet_outer_index != (facet_outer_list.len() - 1)
-                            {
-                                filter_string.push_str(" OR ")
-                            }
-                        }
-
-                        filter_string.push(')');
-
-                        if index != (facets.len() - 1) {
-                            filter_string.push_str(" AND ")
-                        }
-                    }
-                    filter_string.push(')');
-
-                    if !filters.is_empty() {
-                        write!(filter_string, " AND ({filters})")
-                            .expect("write should not fail");
-                    }
-                } else {
-                    filter_string.push_str(&filters);
-                }
-
-                if !filter_string.is_empty() {
-                    query.with_filter(&filter_string);
-                }
-            }
-
-            query.execute::<ResultSearchProject>().await?
-        };
-
-        Ok(SearchResults {
-            hits: results.hits.into_iter().map(|r| r.result).collect(),
-            page: results.page.unwrap_or_default(),
-            hits_per_page: results.hits_per_page.unwrap_or_default(),
-            total_hits: results.total_hits.unwrap_or_default(),
-        })
-    }
-
-    async fn index_projects(
-        &self,
-        ro_pool: PgPool,
-        redis: RedisPool,
-    ) -> eyre::Result<()> {
-        indexing::index_projects(ro_pool, redis, &self.config).await?;
-        Ok(())
-    }
-
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()> {
-        indexing::remove_documents(ids, &self.config).await?;
-        Ok(())
-    }
-
-    async fn tasks(&self) -> eyre::Result<Value> {
-        let client = self
-            .config
-            .make_batch_client()
-            .wrap_internal_err("failed to make batch client")?;
-        let tasks = client
-            .with_all_clients("get_tasks", async |client| {
-                let tasks = client.get_tasks().await?;
-                Ok(tasks.results)
-            })
-            .await
-            .wrap_internal_err("failed to get tasks")?;
-
-        #[derive(Serialize)]
-        struct MeiliTask<Time> {
-            uid: u32,
-            status: &'static str,
-            duration: Option<Duration>,
-            enqueued_at: Option<Time>,
-        }
-
-        #[derive(Serialize)]
-        struct TaskList<Time> {
-            by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
-        }
-
-        let response = tasks
-            .into_iter()
-            .enumerate()
-            .map(|(idx, instance_tasks)| {
-                let tasks = instance_tasks
-                    .into_iter()
-                    .filter_map(|task| {
-                        Some(match task {
-                            Task::Enqueued { content } => MeiliTask {
-                                uid: content.uid,
-                                status: "enqueued",
-                                duration: None,
-                                enqueued_at: Some(content.enqueued_at),
-                            },
-                            Task::Processing { content } => MeiliTask {
-                                uid: content.uid,
-                                status: "processing",
-                                duration: None,
-                                enqueued_at: Some(content.enqueued_at),
-                            },
-                            Task::Failed { content } => MeiliTask {
-                                uid: content.task.uid,
-                                status: "failed",
-                                duration: Some(content.task.duration),
-                                enqueued_at: Some(content.task.enqueued_at),
-                            },
-                            Task::Succeeded { .. } => return None,
-                        })
-                    })
-                    .collect();
-
-                (idx.to_string(), tasks)
-            })
-            .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
-
-        let response = serde_json::to_value(TaskList {
-            by_instance: response,
-        })
-        .wrap_internal_err("failed to serialize tasks response")?;
-        Ok(response)
-    }
-
-    async fn tasks_cancel(
-        &self,
-        filter: &TasksCancelFilter,
-    ) -> eyre::Result<()> {
-        let client = self
-            .config
-            .make_batch_client()
-            .wrap_internal_err("failed to make batch client")?;
-        let all_results = client
-            .with_all_clients("cancel_tasks", async |client| {
-                let mut q = TasksCancelQuery::new(client);
-                match filter {
-                    TasksCancelFilter::All => {}
-                    TasksCancelFilter::Indexes { indexes } => {
-                        q.with_index_uids(indexes.iter().map(|s| s.as_str()));
-                    }
-                    TasksCancelFilter::AllEnqueued => {
-                        q.with_statuses(["enqueued"]);
-                    }
-                };
-
-                let result = client.cancel_tasks_with(&q).await;
-                Ok(result)
-            })
-            .await
-            .wrap_internal_err("failed to cancel tasks")?;
-
-        for r in all_results {
-            r.wrap_internal_err("failed to cancel tasks")?;
-        }
-
-        Ok(())
-    }
-}

apps/labrinth/src/search/backend/mod.rs

@@ -1,5 +0,0 @@
-mod elasticsearch;
-mod meilisearch;
-
-pub use elasticsearch::Elasticsearch;
-pub use meilisearch::{Meilisearch, MeilisearchConfig};

apps/labrinth/src/search/indexing/local_import.rs

@@ -5,6 +5,7 @@ use itertools::Itertools;
 use std::collections::HashMap;
 use tracing::info;
 
+use super::IndexingError;
 use crate::database::PgPool;
 use crate::database::models::loader_fields::{
     QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
@@ -18,13 +19,12 @@ use crate::models::projects::from_duplicate_version_fields;
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
 use crate::search::UploadSearchProject;
-use crate::util::error::Context;
 
 pub async fn index_local(
     pool: &PgPool,
     cursor: i64,
     limit: i64,
-) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
+) -> Result<(Vec<UploadSearchProject>, i64), IndexingError> {
     info!("Indexing local projects!");
 
     // todo: loaders, project type, game versions
@@ -76,8 +76,7 @@ pub async fn index_local(
             }
         })
         .try_collect::<Vec<PartialProject>>()
-        .await
-        .wrap_err("failed to fetch projects")?;
+        .await?;
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
 
@@ -439,7 +438,7 @@ struct PartialVersion {
 async fn index_versions(
     pool: &PgPool,
     project_ids: Vec<i64>,
-) -> eyre::Result<HashMap<DBProjectId, Vec<PartialVersion>>> {
+) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>, IndexingError> {
     let versions: HashMap<DBProjectId, Vec<DBVersionId>> = sqlx::query!(
         "
         SELECT v.id, v.mod_id
@@ -458,8 +457,7 @@ async fn index_versions(
             async move { Ok(acc) }
         },
     )
-    .await
-    .wrap_err("failed to fetch versions")?;
+    .await?;
 
     // Get project types, loaders
     #[derive(Default)]
@@ -500,8 +498,7 @@ async fn index_versions(
         (version_id, version_loader_data)
     })
     .try_collect()
-    .await
-    .wrap_err("failed to fetch loaders and project types")?;
+    .await?;
 
     // Get version fields
     let version_fields: DashMap<DBVersionId, Vec<QueryVersionField>> =
@@ -533,10 +530,7 @@ async fn index_versions(
                 async move { Ok(acc) }
             },
         )
-        .await
-        .wrap_err("failed to fetch version fields")?;
-
-    // Get version fields
+        .await?;
 
     // Convert to partial versions
     let mut res_versions: HashMap<DBProjectId, Vec<PartialVersion>> =

apps/labrinth/src/search/indexing/mod.rs

@@ -1,16 +1,18 @@
+/// This module is used for the indexing from any source.
+pub mod local_import;
+
 use std::time::Duration;
 
 use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::env::ENV;
-use crate::search::UploadSearchProject;
-use crate::search::backend::meilisearch::MeilisearchConfig;
-use crate::search::indexing::index_local;
+use crate::search::{SearchConfig, UploadSearchProject};
 use crate::util::error::Context;
 use ariadne::ids::base62_impl::to_base62;
 use eyre::eyre;
 use futures::StreamExt;
 use futures::stream::FuturesOrdered;
+use local_import::index_local;
 use meilisearch_sdk::client::{Client, SwapIndexes};
 use meilisearch_sdk::indexes::Index;
 use meilisearch_sdk::settings::{PaginationSetting, Settings};
@@ -32,8 +34,6 @@ pub enum IndexingError {
     Env(#[from] dotenvy::Error),
     #[error("Error while awaiting index creation task")]
     Task,
-    #[error(transparent)]
-    Other(#[from] eyre::Report),
 }
 
 // // The chunk size for adding projects to the indexing database. If the request size
@@ -49,8 +49,8 @@ fn search_operation_timeout() -> std::time::Duration {
 
 pub async fn remove_documents(
     ids: &[crate::models::ids::VersionId],
-    config: &MeilisearchConfig,
-) -> Result<(), IndexingError> {
+    config: &SearchConfig,
+) -> eyre::Result<()> {
     let mut indexes = get_indexes_for_indexing(config, false, false)
         .await
         .wrap_err("failed to get current indexes")?;
@@ -106,7 +106,7 @@ pub async fn remove_documents(
 pub async fn index_projects(
     ro_pool: PgPool,
     redis: RedisPool,
-    config: &MeilisearchConfig,
+    config: &SearchConfig,
 ) -> Result<(), IndexingError> {
     info!("Indexing projects.");
 
@@ -188,7 +188,7 @@ pub async fn index_projects(
 }
 
 pub async fn swap_index(
-    config: &MeilisearchConfig,
+    config: &SearchConfig,
     index_name: &str,
 ) -> Result<(), IndexingError> {
     let client = config.make_batch_client()?;
@@ -224,7 +224,7 @@ pub async fn swap_index(
 
 #[instrument(skip(config))]
 pub async fn get_indexes_for_indexing(
-    config: &MeilisearchConfig,
+    config: &SearchConfig,
     next: bool, // Get the 'next' one
     update_settings: bool,
 ) -> Result<Vec<Vec<Index>>, IndexingError> {
@@ -500,7 +500,7 @@ pub async fn add_projects_batch_client(
     indices: &[Vec<Index>],
     projects: Vec<UploadSearchProject>,
     additional_fields: Vec<String>,
-    config: &MeilisearchConfig,
+    config: &SearchConfig,
 ) -> Result<(), IndexingError> {
     let client = config.make_batch_client()?;
 

apps/labrinth/src/search/mod.rs

@@ -1,71 +1,178 @@
-use crate::database::redis::RedisPool;
-use crate::models::ids::VersionId;
+use crate::env::ENV;
 use crate::models::projects::SearchRequest;
-use crate::{database::PgPool, env::ENV};
-use async_trait::async_trait;
+use crate::{models::error::ApiError, search::indexing::IndexingError};
+use actix_web::HttpResponse;
+use actix_web::http::StatusCode;
 use chrono::{DateTime, Utc};
+use futures::TryStreamExt;
+use futures::stream::FuturesOrdered;
+use itertools::Itertools;
+use meilisearch_sdk::client::Client;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::{collections::HashMap, str::FromStr};
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::fmt::Write;
 use thiserror::Error;
-use utoipa::ToSchema;
+use tracing::{Instrument, info_span};
 
-pub mod backend;
 pub mod indexing;
 
-#[async_trait]
-pub trait SearchBackend: Send + Sync {
-    async fn search_for_project(
-        &self,
-        info: &SearchRequest,
-    ) -> eyre::Result<SearchResults>;
-
-    async fn index_projects(
-        &self,
-        ro_pool: PgPool,
-        redis: RedisPool,
-    ) -> eyre::Result<()>;
-
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
-
-    async fn tasks(&self) -> eyre::Result<Value>;
-
-    async fn tasks_cancel(
-        &self,
-        filter: &TasksCancelFilter,
-    ) -> eyre::Result<()>;
+#[derive(Error, Debug)]
+pub enum SearchError {
+    #[error("MeiliSearch Error: {0}")]
+    MeiliSearch(#[from] meilisearch_sdk::errors::Error),
+    #[error("Error while serializing or deserializing JSON: {0}")]
+    Serde(#[from] serde_json::Error),
+    #[error("Error while parsing an integer: {0}")]
+    IntParsing(#[from] std::num::ParseIntError),
+    #[error("Error while formatting strings: {0}")]
+    FormatError(#[from] std::fmt::Error),
+    #[error("Environment Error")]
+    Env(#[from] dotenvy::Error),
+    #[error("Invalid index to sort by: {0}")]
+    InvalidIndex(String),
 }
 
-#[derive(Deserialize, Serialize, ToSchema)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum TasksCancelFilter {
-    All,
-    AllEnqueued,
-    Indexes { indexes: Vec<String> },
-}
+impl actix_web::ResponseError for SearchError {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            SearchError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
+            SearchError::MeiliSearch(..) => StatusCode::BAD_REQUEST,
+            SearchError::Serde(..) => StatusCode::BAD_REQUEST,
+            SearchError::IntParsing(..) => StatusCode::BAD_REQUEST,
+            SearchError::InvalidIndex(..) => StatusCode::BAD_REQUEST,
+            SearchError::FormatError(..) => StatusCode::BAD_REQUEST,
+        }
+    }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum SearchBackendKind {
-    Meilisearch,
-    Elasticsearch,
-}
-
-#[derive(Debug, Error)]
-#[error("invalid search backend kind")]
-pub struct InvalidSearchBackendKind;
-
-impl FromStr for SearchBackendKind {
-    type Err = InvalidSearchBackendKind;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(match s {
-            "meilisearch" => SearchBackendKind::Meilisearch,
-            "elasticsearch" => SearchBackendKind::Elasticsearch,
-            _ => return Err(InvalidSearchBackendKind),
+    fn error_response(&self) -> HttpResponse {
+        HttpResponse::build(self.status_code()).json(ApiError {
+            error: match self {
+                SearchError::Env(..) => "environment_error",
+                SearchError::MeiliSearch(..) => "meilisearch_error",
+                SearchError::Serde(..) => "invalid_input",
+                SearchError::IntParsing(..) => "invalid_input",
+                SearchError::InvalidIndex(..) => "invalid_input",
+                SearchError::FormatError(..) => "invalid_input",
+            },
+            description: self.to_string(),
+            details: None,
         })
     }
 }
 
+#[derive(Debug, Clone)]
+pub struct MeilisearchReadClient {
+    pub client: Client,
+}
+
+impl std::ops::Deref for MeilisearchReadClient {
+    type Target = Client;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}
+
+pub struct BatchClient {
+    pub clients: Vec<Client>,
+}
+
+impl BatchClient {
+    pub fn new(clients: Vec<Client>) -> Self {
+        Self { clients }
+    }
+
+    pub async fn with_all_clients<'a, T, G, Fut>(
+        &'a self,
+        task_name: &str,
+        generator: G,
+    ) -> Result<Vec<T>, IndexingError>
+    where
+        G: Fn(&'a Client) -> Fut,
+        Fut: Future<Output = Result<T, IndexingError>> + 'a,
+    {
+        let mut tasks = FuturesOrdered::new();
+        for (idx, client) in self.clients.iter().enumerate() {
+            tasks.push_back(generator(client).instrument(info_span!(
+                "client_task",
+                task.name = task_name,
+                client.idx = idx,
+            )));
+        }
+
+        let results = tasks.try_collect::<Vec<T>>().await?;
+        Ok(results)
+    }
+
+    pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
+    where
+        F: FnMut(T, &Client) -> R,
+    {
+        assert_eq!(
+            data.len(),
+            self.clients.len(),
+            "mismatch between data len and meilisearch client count"
+        );
+        self.clients
+            .iter()
+            .zip(data)
+            .map(|(client, item)| predicate(item, client))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct SearchConfig {
+    pub addresses: Vec<String>,
+    pub read_lb_address: String,
+    pub key: String,
+    pub meta_namespace: String,
+}
+
+impl SearchConfig {
+    // Panics if the environment variables are not set,
+    // but these are already checked for on startup.
+    pub fn new(meta_namespace: Option<String>) -> Self {
+        Self {
+            addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
+            key: ENV.MEILISEARCH_KEY.clone(),
+            meta_namespace: meta_namespace.unwrap_or_default(),
+            read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
+        }
+    }
+
+    pub fn make_loadbalanced_read_client(
+        &self,
+    ) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
+        Ok(MeilisearchReadClient {
+            client: Client::new(&self.read_lb_address, Some(&self.key))?,
+        })
+    }
+
+    pub fn make_batch_client(
+        &self,
+    ) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
+        Ok(BatchClient::new(
+            self.addresses
+                .iter()
+                .map(|address| {
+                    Client::new(address.as_str(), Some(self.key.as_str()))
+                })
+                .collect::<Result<Vec<_>, _>>()?,
+        ))
+    }
+
+    // Next: true if we want the next index (we are preparing the next swap), false if we want the current index (searching)
+    pub fn get_index_name(&self, index: &str, next: bool) -> String {
+        let alt = if next { "_alt" } else { "" };
+        format!("{}_{}_{}", self.meta_namespace, index, alt)
+    }
+}
+
+/// A project document used for uploading projects to MeiliSearch's indices.
+/// This contains some extra data that is not returned by search results.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
     pub version_id: String,
@@ -142,14 +249,147 @@ pub struct ResultSearchProject {
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
 
-pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
-    match ENV.SEARCH_BACKEND {
-        SearchBackendKind::Meilisearch => {
-            let config = backend::MeilisearchConfig::new(meta_namespace);
-            Box::new(backend::Meilisearch::new(config))
+pub fn get_sort_index(
+    config: &SearchConfig,
+    index: &str,
+) -> Result<(String, [&'static str; 1]), SearchError> {
+    let projects_name = config.get_index_name("projects", false);
+    let projects_filtered_name =
+        config.get_index_name("projects_filtered", false);
+    Ok(match index {
+        "relevance" => (projects_name, ["downloads:desc"]),
+        "downloads" => (projects_filtered_name, ["downloads:desc"]),
+        "follows" => (projects_name, ["follows:desc"]),
+        "updated" => (projects_name, ["date_modified:desc"]),
+        "newest" => (projects_name, ["date_created:desc"]),
+        i => return Err(SearchError::InvalidIndex(i.to_string())),
+    })
+}
+
+pub async fn search_for_project(
+    info: &SearchRequest,
+    config: &SearchConfig,
+) -> Result<SearchResults, SearchError> {
+    let offset: usize = info.offset.as_deref().unwrap_or("0").parse()?;
+    let index = info.index.as_deref().unwrap_or("relevance");
+    let limit = info
+        .limit
+        .as_deref()
+        .unwrap_or("10")
+        .parse::<usize>()?
+        .min(100);
+
+    let sort = get_sort_index(config, index)?;
+    let client = config.make_loadbalanced_read_client()?;
+    let meilisearch_index = client.get_index(sort.0).await?;
+
+    let mut filter_string = String::new();
+
+    // Convert offset and limit to page and hits_per_page
+    let hits_per_page = if limit == 0 { 1 } else { limit };
+
+    let page = offset / hits_per_page + 1;
+
+    let results = {
+        let mut query = meilisearch_index.search();
+        query
+            .with_page(page)
+            .with_hits_per_page(hits_per_page)
+            .with_query(info.query.as_deref().unwrap_or_default())
+            .with_sort(&sort.1);
+
+        if let Some(new_filters) = info.new_filters.as_deref() {
+            query.with_filter(new_filters);
+        } else {
+            let facets = if let Some(facets) = &info.facets {
+                Some(serde_json::from_str::<Vec<Vec<Value>>>(facets)?)
+            } else {
+                None
+            };
+
+            let filters: Cow<_> =
+                match (info.filters.as_deref(), info.version.as_deref()) {
+                    (Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
+                    (Some(f), None) => f.into(),
+                    (None, Some(v)) => v.into(),
+                    (None, None) => "".into(),
+                };
+
+            if let Some(facets) = facets {
+                // Search can now *optionally* have a third inner array: So Vec(AND)<Vec(OR)<Vec(AND)< _ >>>
+                // For every inner facet, we will check if it can be deserialized into a Vec<&str>, and do so.
+                // If not, we will assume it is a single facet and wrap it in a Vec.
+                let facets: Vec<Vec<Vec<String>>> = facets
+                    .into_iter()
+                    .map(|facets| {
+                        facets
+                            .into_iter()
+                            .map(|facet| {
+                                if facet.is_array() {
+                                    serde_json::from_value::<Vec<String>>(facet)
+                                        .unwrap_or_default()
+                                } else {
+                                    vec![
+                                        serde_json::from_value::<String>(facet)
+                                            .unwrap_or_default(),
+                                    ]
+                                }
+                            })
+                            .collect_vec()
+                    })
+                    .collect_vec();
+
+                filter_string.push('(');
+                for (index, facet_outer_list) in facets.iter().enumerate() {
+                    filter_string.push('(');
+
+                    for (facet_outer_index, facet_inner_list) in
+                        facet_outer_list.iter().enumerate()
+                    {
+                        filter_string.push('(');
+                        for (facet_inner_index, facet) in
+                            facet_inner_list.iter().enumerate()
+                        {
+                            filter_string.push_str(&facet.replace(':', " = "));
+                            if facet_inner_index != (facet_inner_list.len() - 1)
+                            {
+                                filter_string.push_str(" AND ")
+                            }
+                        }
+                        filter_string.push(')');
+
+                        if facet_outer_index != (facet_outer_list.len() - 1) {
+                            filter_string.push_str(" OR ")
+                        }
+                    }
+
+                    filter_string.push(')');
+
+                    if index != (facets.len() - 1) {
+                        filter_string.push_str(" AND ")
+                    }
+                }
+                filter_string.push(')');
+
+                if !filters.is_empty() {
+                    write!(filter_string, " AND ({filters})")?;
+                }
+            } else {
+                filter_string.push_str(&filters);
+            }
+
+            if !filter_string.is_empty() {
+                query.with_filter(&filter_string);
+            }
         }
-        SearchBackendKind::Elasticsearch => {
-            Box::new(backend::Elasticsearch::new(meta_namespace).unwrap())
-        }
-    }
+
+        query.execute::<ResultSearchProject>().await?
+    };
+
+    Ok(SearchResults {
+        hits: results.hits.into_iter().map(|r| r.result).collect(),
+        page: results.page.unwrap_or_default(),
+        hits_per_page: results.hits_per_page.unwrap_or_default(),
+        total_hits: results.total_hits.unwrap_or_default(),
+    })
 }

apps/labrinth/src/test/database.rs

@@ -2,9 +2,8 @@ use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::database::{MIGRATOR, ReadOnlyPgPool};
 use crate::env::ENV;
-use crate::search::{self, SearchBackend};
+use crate::search;
 use sqlx::postgres::PgPoolOptions;
-use std::sync::Arc;
 use std::time::Duration;
 use url::Url;
 
@@ -43,8 +42,8 @@ pub struct TemporaryDatabase {
     pub pool: PgPool,
     pub ro_pool: ReadOnlyPgPool,
     pub redis_pool: RedisPool,
+    pub search_config: crate::search::SearchConfig,
     pub database_name: String,
-    pub search_backend: Arc<dyn SearchBackend>,
 }
 
 impl TemporaryDatabase {
@@ -92,14 +91,15 @@ impl TemporaryDatabase {
         // Gets new Redis pool
         let redis_pool = RedisPool::new(temp_database_name.clone());
 
-        // Create search backend
-        let search_backend = search::backend(Some(temp_database_name.clone()));
+        // Create new meilisearch config
+        let search_config =
+            search::SearchConfig::new(Some(temp_database_name.clone()));
         Self {
             pool,
             ro_pool,
             database_name: temp_database_name,
             redis_pool,
-            search_backend: Arc::from(search_backend),
+            search_config,
         }
     }
 
@@ -193,9 +193,7 @@ impl TemporaryDatabase {
                         ro_pool: ReadOnlyPgPool::from(pool.clone()),
                         database_name: TEMPLATE_DATABASE_NAME.to_string(),
                         redis_pool: RedisPool::new(name.clone()),
-                        search_backend: Arc::from(search::backend(Some(
-                            name.clone(),
-                        ))),
+                        search_config: search::SearchConfig::new(Some(name)),
                     };
                     let setup_api =
                         TestEnvironment::<ApiV3>::build_setup_api(&db).await;

apps/labrinth/src/test/mod.rs

@@ -23,6 +23,7 @@ pub mod search;
 // If making a test, you should probably use environment::TestEnvironment::build() (which calls this)
 pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     println!("Setting up labrinth config");
+    dotenvy::dotenv().ok();
     env::init().expect("failed to initialize environment variables");
 
     let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
@@ -30,7 +31,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     let pool = db.pool.clone();
     let ro_pool = db.ro_pool.clone();
     let redis_pool = db.redis_pool.clone();
-    let search_backend = db.search_backend.clone();
+    let search_config = db.search_config.clone();
     let file_host: Arc<dyn file_hosting::FileHost + Send + Sync> =
         Arc::new(file_hosting::MockHost::new());
     let mut clickhouse = clickhouse::init_client().await.unwrap();
@@ -47,7 +48,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         pool.clone(),
         ro_pool.clone(),
         redis_pool.clone(),
-        search_backend.into(),
+        search_config,
         &mut clickhouse,
         file_host.clone(),
         stripe_client,

docker-compose.yml

@@ -37,167 +37,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-  elasticsearch-certs:
-    image: elasticsearch:9.3.0
-    container_name: labrinth-elasticsearch-certs
-    user: '0'
-    networks:
-      - elasticsearch-mesh
-    restart: 'no'
-    volumes:
-      - elasticsearch-certs:/usr/share/elasticsearch/config/certs
-    command: |
-      bash -c '
-        set -euo pipefail
-        if [ ! -s config/certs/ca/ca.crt ] || [ ! -s config/certs/elasticsearch0/elasticsearch0.crt ] || [ ! -s config/certs/elasticsearch1/elasticsearch1.crt ] || [ ! -s config/certs/elasticsearch2/elasticsearch2.crt ]; then
-          rm -rf config/certs/*
-          printf "%s\n" \
-            "instances:" \
-            "  - name: elasticsearch0" \
-            "    dns:" \
-            "      - elasticsearch0" \
-            "      - localhost" \
-            "    ip:" \
-            "      - 127.0.0.1" \
-            "  - name: elasticsearch1" \
-            "    dns:" \
-            "      - elasticsearch1" \
-            "      - localhost" \
-            "    ip:" \
-            "      - 127.0.0.1" \
-            "  - name: elasticsearch2" \
-            "    dns:" \
-            "      - elasticsearch2" \
-            "      - localhost" \
-            "    ip:" \
-            "      - 127.0.0.1" \
-            > config/certs/instances.yml
-          bin/elasticsearch-certutil ca --silent --pem --out config/certs/ca.zip
-          unzip config/certs/ca.zip -d config/certs
-          bin/elasticsearch-certutil cert --silent --pem --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key --out config/certs/certs.zip
-          unzip config/certs/certs.zip -d config/certs
-        fi
-        chown -R 1000:0 config/certs
-        find config/certs -type d -exec chmod 750 {} \;
-        find config/certs -type f -exec chmod 640 {} \;
-        echo "Set up certificates"
-      '
-  elasticsearch0:
-    image: elasticsearch:9.3.0
-    container_name: labrinth-elasticsearch0
-    networks:
-      - elasticsearch-mesh
-    restart: on-failure
-    depends_on:
-      elasticsearch-certs:
-        condition: service_completed_successfully
-    ports:
-      - '127.0.0.1:9200:9200'
-    volumes:
-      - elasticsearch0-data:/usr/share/elasticsearch/data
-      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
-    environment:
-      - logger.level=WARN
-      - node.name=elasticsearch0
-      - cluster.name=labrinth
-      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
-      - discovery.seed_hosts=elasticsearch1,elasticsearch2
-      - bootstrap.memory_lock=false
-      # auth
-      - xpack.security.enabled=true
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.security.transport.ssl.key=certs/elasticsearch0/elasticsearch0.key
-      - xpack.security.transport.ssl.certificate=certs/elasticsearch0/elasticsearch0.crt
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
-      - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=elastic
-    mem_limit: 1g
-    healthcheck:
-      test:
-        [
-          'CMD-SHELL',
-          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
-        ]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-  elasticsearch1:
-    image: elasticsearch:9.3.0
-    container_name: labrinth-elasticsearch1
-    networks:
-      - elasticsearch-mesh
-    restart: on-failure
-    depends_on:
-      elasticsearch-certs:
-        condition: service_completed_successfully
-    volumes:
-      - elasticsearch1-data:/usr/share/elasticsearch/data
-      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
-    environment:
-      - logger.level=WARN
-      - node.name=elasticsearch1
-      - cluster.name=labrinth
-      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
-      - discovery.seed_hosts=elasticsearch0,elasticsearch2
-      - bootstrap.memory_lock=false
-      # auth
-      - xpack.security.enabled=true
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.security.transport.ssl.key=certs/elasticsearch1/elasticsearch1.key
-      - xpack.security.transport.ssl.certificate=certs/elasticsearch1/elasticsearch1.crt
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
-      - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=elastic
-    mem_limit: 1g
-    healthcheck:
-      test:
-        [
-          'CMD-SHELL',
-          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
-        ]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-  elasticsearch2:
-    image: elasticsearch:9.3.0
-    container_name: labrinth-elasticsearch2
-    networks:
-      - elasticsearch-mesh
-    restart: on-failure
-    depends_on:
-      elasticsearch-certs:
-        condition: service_completed_successfully
-    volumes:
-      - elasticsearch2-data:/usr/share/elasticsearch/data
-      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
-    environment:
-      - logger.level=WARN
-      - node.name=elasticsearch2
-      - cluster.name=labrinth
-      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
-      - discovery.seed_hosts=elasticsearch0,elasticsearch1
-      - bootstrap.memory_lock=false
-      # auth
-      - xpack.security.enabled=true
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.security.transport.ssl.key=certs/elasticsearch2/elasticsearch2.key
-      - xpack.security.transport.ssl.certificate=certs/elasticsearch2/elasticsearch2.crt
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
-      - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=elastic
-    mem_limit: 1g
-    healthcheck:
-      test:
-        [
-          'CMD-SHELL',
-          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
-        ]
-      interval: 10s
-      timeout: 5s
-      retries: 10
   redis:
     image: redis:alpine
     container_name: labrinth-redis
@@ -250,6 +89,8 @@ services:
     ports:
       - '127.0.0.1:13000:3000'
     extra_hosts:
+      # Gotenberg must send a message on a webhook to our backend,
+      # so it must have access to our local network
       - 'host.docker.internal:host-gateway'
   labrinth:
     profiles:
@@ -268,12 +109,6 @@ services:
         condition: service_healthy
       meilisearch:
         condition: service_healthy
-      elasticsearch0:
-        condition: service_healthy
-      elasticsearch1:
-        condition: service_healthy
-      elasticsearch2:
-        condition: service_healthy
       redis:
         condition: service_healthy
       clickhouse:
@@ -301,9 +136,14 @@ services:
       timeout: 5s
       retries: 3
     volumes:
+      # Labrinth deposits version files here;
+      # Delphi reads them from here
       - /tmp/modrinth:/tmp/modrinth:ro,z
     extra_hosts:
+      # Delphi must send a message on a webhook to our backend,
+      # so it must have access to our local network
       - 'host.docker.internal:host-gateway'
+
   # Sharded Meilisearch
   meilisearch1:
     profiles:
@@ -326,6 +166,7 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
+
   nginx-meilisearch-lb:
     profiles:
       - sharded-meilisearch
@@ -345,15 +186,9 @@ services:
 networks:
   meilisearch-mesh:
     driver: bridge
-  elasticsearch-mesh:
-    driver: bridge
 volumes:
   meilisearch-data:
   meilisearch1-data:
-  elasticsearch0-data:
-  elasticsearch1-data:
-  elasticsearch2-data:
-  elasticsearch-certs:
   db-data:
   redis-data:
   labrinth-cdn-data:

packages/ui/src/components/servers/files/FileNavbar.vue

@@ -80,6 +80,17 @@
 				@update:model-value="$emit('update:searchQuery', $event)"
 			/>
 
+			<ButtonStyled v-if="showRefreshButton" type="outlined">
+				<button
+					type="button"
+					class="flex h-10 items-center gap-2 !border-[1px] !border-surface-5"
+					@click="$emit('refresh')"
+				>
+					<RefreshCwIcon aria-hidden="true" class="h-5 w-5" />
+					Refresh
+				</button>
+			</ButtonStyled>
+
 			<ButtonStyled type="outlined">
 				<OverflowMenu
 					:dropdown-id="`create-new-${baseId}`"
@@ -178,6 +189,7 @@ const props = defineProps<{
 	editingFilePath?: string
 	isEditingImage?: boolean
 	searchQuery: string
+	showRefreshButton?: boolean
 	baseId: string
 }>()
 
@@ -190,6 +202,7 @@ defineEmits<{
 	upload: []
 	uploadZip: []
 	unzipFromUrl: [cf: boolean]
+	refresh: []
 	save: []
 	saveAs: []
 	saveRestart: []

packages/ui/src/pages/hosting/manage/files.vue

@@ -30,6 +30,7 @@
 						:editing-file-path="editingFile?.path"
 						:is-editing-image="fileEditorRef?.isEditingImage"
 						:search-query="searchQuery"
+						:show-refresh-button="showRefreshButton"
 						:base-id="baseId"
 						@navigate="navigateToSegment"
 						@navigate-home="() => navigateToSegment(-1)"
@@ -39,6 +40,7 @@
 						@upload="initiateFileUpload"
 						@upload-zip="() => {}"
 						@unzip-from-url="showUnzipFromUrlModal"
+						@refresh="refreshList"
 						@save="() => fileEditorRef?.saveFileContent(true)"
 						@save-as="() => fileEditorRef?.saveFileContent(false)"
 						@save-restart="() => fileEditorRef?.saveAndRestart()"
@@ -303,6 +305,7 @@ import {
 
 defineProps<{
 	showDebugInfo?: boolean
+	showRefreshButton?: boolean
 }>()
 
 const notifications = injectNotificationManager()