moderation_notes everywhere

* Remove non-Typesense search backends, add default env vars

* shear

* remove some default keys

.github/workflows/labrinth-docker.yml

@@ -164,7 +164,7 @@ jobs:
         with:
           context: ./apps/labrinth/docker-stage
           file: ./apps/labrinth/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
           annotations: ${{ steps.docker-meta.outputs.annotations }}

Cargo.lock

@@ -3059,29 +3059,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "elasticsearch"
-version = "9.1.0-alpha.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12bb303aa6e1d28c0c86b6fbfe484fd0fd3f512629aeed1ac4f6b85f81d9834a"
-dependencies = [
- "base64 0.22.1",
- "bytes",
- "dyn-clone",
- "flate2",
- "lazy_static",
- "parking_lot",
- "percent-encoding",
- "reqwest 0.12.24",
- "rustc_version",
- "serde",
- "serde_json",
- "serde_with",
- "tokio",
- "url",
- "void",
-]
-
 [[package]]
 name = "elliptic-curve"
 version = "0.13.8"
@@ -5275,7 +5252,6 @@ dependencies = [
  "dotenv-build",
  "dotenvy",
  "either",
- "elasticsearch",
  "eyre",
  "futures",
  "futures-util",

Cargo.toml

@@ -77,7 +77,6 @@ dotenv-build = "0.1.1"
 dotenvy = "0.15.7"
 dunce = "1.0.5"
 either = "1.15.0"
-elasticsearch = "9.1.0-alpha.1"
 encoding_rs = "0.8.35"
 enumset = "1.1.10"
 eyre = "0.6.12"

apps/labrinth/.sqlx/query-1ca9b7c343ccdc4b0ac66c67d1b456440d1a4d687235c28d71bec7aea7bff6e7.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            INSERT INTO moderation_notes (user_id, organization_id, last_author, version, notes, user_rating)\n            SELECT\n              $1, $2, $3, 1, COALESCE($4::text, ''), COALESCE($5::integer, 0)\n            WHERE NOT EXISTS (\n              SELECT 1 FROM moderation_notes\n              WHERE\n                ($1::bigint IS NOT NULL AND user_id = $1)\n                OR ($2::bigint IS NOT NULL AND organization_id = $2)\n            )\n            ON CONFLICT DO NOTHING\n            RETURNING version\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8",
+        "Int8",
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "1ca9b7c343ccdc4b0ac66c67d1b456440d1a4d687235c28d71bec7aea7bff6e7"
+}

apps/labrinth/.sqlx/query-5894d30f04a5413a607f2417cd3b7a9710aeadd5f97e3c25521202e9e21e261e.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n\t\t\tSELECT user_id, organization_id, last_modified, created_at, last_author, version, notes, user_rating\n\t\t\tFROM moderation_notes\n\t\t\tWHERE organization_id = ANY($1)\n\t\t\t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "user_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "organization_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "last_modified",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "last_author",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "version",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 6,
+        "name": "notes",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "user_rating",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8Array"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "5894d30f04a5413a607f2417cd3b7a9710aeadd5f97e3c25521202e9e21e261e"
+}

apps/labrinth/.sqlx/query-5a86ade76259aafdd5b778e5b1909e1653c4442d9ace4105dd18aa5333c52aa2.json

@@ -0,0 +1,27 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            UPDATE moderation_notes\n            SET\n              last_modified = NOW(),\n              last_author = $1,\n              version = version + 1,\n              notes = COALESCE($2::text, notes),\n              user_rating = COALESCE($3::integer, user_rating)\n            WHERE (\n                ($4::bigint IS NOT NULL AND user_id = $4) OR\n                ($5::bigint IS NOT NULL AND organization_id = $5)\n              )\n              AND version = $6\n            RETURNING version\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text",
+        "Int4",
+        "Int8",
+        "Int8",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "5a86ade76259aafdd5b778e5b1909e1653c4442d9ace4105dd18aa5333c52aa2"
+}

apps/labrinth/.sqlx/query-ad49878323e942bdbaac87c351621e51871ec591bc0e7ff29119834e3ed8bf53.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n\t\t\tSELECT user_id, organization_id, last_modified, created_at, last_author, version, notes, user_rating\n\t\t\tFROM moderation_notes\n\t\t\tWHERE user_id = ANY($1)\n\t\t\t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "user_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "organization_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "last_modified",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "last_author",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "version",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 6,
+        "name": "notes",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "user_rating",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8Array"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "ad49878323e942bdbaac87c351621e51871ec591bc0e7ff29119834e3ed8bf53"
+}

apps/labrinth/Cargo.toml

@@ -45,7 +45,6 @@ deadpool-redis.workspace = true
 derive_more = { workspace = true, features = ["deref", "deref_mut"] }
 dotenvy = { workspace = true }
 either = { workspace = true }
-elasticsearch = { workspace = true, features = ["experimental-apis"] }
 eyre = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }

apps/labrinth/migrations/20260513120000_moderation_notes.sql

@@ -0,0 +1,21 @@
+CREATE TABLE moderation_notes (
+	user_id BIGINT NULL REFERENCES users(id) ON DELETE CASCADE,
+	organization_id BIGINT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+	last_modified TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+	created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+	last_author BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+	version INTEGER NOT NULL DEFAULT 1,
+	notes TEXT NOT NULL,
+	user_rating INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE UNIQUE INDEX moderation_notes_user_id_unique
+	ON moderation_notes(user_id)
+	WHERE user_id IS NOT NULL;
+
+CREATE UNIQUE INDEX moderation_notes_organization_id_unique
+	ON moderation_notes(organization_id)
+	WHERE organization_id IS NOT NULL;
+
+CREATE INDEX moderation_notes_user_id_idx ON moderation_notes(user_id);
+CREATE INDEX moderation_notes_organization_id_idx ON moderation_notes(organization_id);

apps/labrinth/src/database/models/mod.rs

@@ -13,6 +13,7 @@ pub mod legacy_loader_fields;
 pub mod loader_fields;
 pub mod moderation_external_item;
 pub mod moderation_lock_item;
+pub mod moderation_note_item;
 pub mod notification_item;
 pub mod notifications_deliveries_item;
 pub mod notifications_template_item;
@@ -56,6 +57,7 @@ pub use user_item::DBUser;
 pub use version_item::DBVersion;
 
 pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser};
+pub use moderation_note_item::DBModerationNote;
 
 #[derive(Error, Debug)]
 pub enum DatabaseError {

apps/labrinth/src/database/models/moderation_note_item.rs

@@ -0,0 +1,306 @@
+use std::collections::HashMap;
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+use crate::database::redis::RedisPool;
+
+use super::{DBOrganizationId, DBUserId, DatabaseError};
+
+const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users";
+const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str =
+    "moderation_notes_organizations";
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DBModerationNote {
+    pub user_id: Option<DBUserId>,
+    pub organization_id: Option<DBOrganizationId>,
+    pub last_modified: DateTime<Utc>,
+    pub created_at: DateTime<Utc>,
+    pub last_author: DBUserId,
+    pub version: i32,
+    pub notes: String,
+    pub user_rating: i32,
+}
+
+impl DBModerationNote {
+    pub async fn get_many_users<'a, E>(
+        user_ids: &[DBUserId],
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<HashMap<DBUserId, Self>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let ids = user_ids
+            .iter()
+            .map(|id| id.0.to_string())
+            .collect::<Vec<_>>();
+
+        let cached = {
+            let mut redis = redis.connect().await?;
+            redis
+                .get_many_deserialized_from_json::<Self>(
+                    MODERATION_NOTES_USERS_NAMESPACE,
+                    &ids,
+                )
+                .await?
+        };
+
+        let mut notes = HashMap::new();
+        let mut missing_ids = Vec::new();
+        for (id, cached_note) in user_ids.iter().copied().zip(cached) {
+            if let Some(note) = cached_note {
+                notes.insert(id, note);
+            } else {
+                missing_ids.push(id.0);
+            }
+        }
+
+        if missing_ids.is_empty() {
+            return Ok(notes);
+        }
+
+        let rows = sqlx::query!(
+            r#"
+			SELECT user_id, organization_id, last_modified, created_at, last_author, version, notes, user_rating
+			FROM moderation_notes
+			WHERE user_id = ANY($1)
+			"#,
+            &missing_ids,
+        )
+        .fetch_all(exec)
+        .await?;
+
+        let mut redis = redis.connect().await?;
+        for row in rows {
+            let note = Self {
+                user_id: row.user_id.map(DBUserId),
+                organization_id: row.organization_id.map(DBOrganizationId),
+                last_modified: row.last_modified,
+                created_at: row.created_at,
+                last_author: DBUserId(row.last_author),
+                version: row.version,
+                notes: row.notes,
+                user_rating: row.user_rating,
+            };
+
+            if let Some(user_id) = note.user_id {
+                redis
+                    .set_serialized_to_json(
+                        MODERATION_NOTES_USERS_NAMESPACE,
+                        user_id.0,
+                        &note,
+                        None,
+                    )
+                    .await?;
+                notes.insert(user_id, note);
+            }
+        }
+
+        Ok(notes)
+    }
+
+    pub async fn get_user<'a, E>(
+        user_id: DBUserId,
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<Option<Self>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        Ok(Self::get_many_users(&[user_id], exec, redis)
+            .await?
+            .remove(&user_id))
+    }
+
+    pub async fn get_many_organizations<'a, E>(
+        organization_ids: &[DBOrganizationId],
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<HashMap<DBOrganizationId, Self>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let ids = organization_ids
+            .iter()
+            .map(|id| id.0.to_string())
+            .collect::<Vec<_>>();
+
+        let cached = {
+            let mut redis = redis.connect().await?;
+            redis
+                .get_many_deserialized_from_json::<Self>(
+                    MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
+                    &ids,
+                )
+                .await?
+        };
+
+        let mut notes = HashMap::new();
+        let mut missing_ids = Vec::new();
+        for (id, cached_note) in organization_ids.iter().copied().zip(cached) {
+            if let Some(note) = cached_note {
+                notes.insert(id, note);
+            } else {
+                missing_ids.push(id.0);
+            }
+        }
+
+        if missing_ids.is_empty() {
+            return Ok(notes);
+        }
+
+        let rows = sqlx::query!(
+            r#"
+			SELECT user_id, organization_id, last_modified, created_at, last_author, version, notes, user_rating
+			FROM moderation_notes
+			WHERE organization_id = ANY($1)
+			"#,
+            &missing_ids,
+        )
+        .fetch_all(exec)
+        .await?;
+
+        let mut redis = redis.connect().await?;
+        for row in rows {
+            let note = Self {
+                user_id: row.user_id.map(DBUserId),
+                organization_id: row.organization_id.map(DBOrganizationId),
+                last_modified: row.last_modified,
+                created_at: row.created_at,
+                last_author: DBUserId(row.last_author),
+                version: row.version,
+                notes: row.notes,
+                user_rating: row.user_rating,
+            };
+
+            if let Some(organization_id) = note.organization_id {
+                redis
+                    .set_serialized_to_json(
+                        MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
+                        organization_id.0,
+                        &note,
+                        None,
+                    )
+                    .await?;
+                notes.insert(organization_id, note);
+            }
+        }
+
+        Ok(notes)
+    }
+
+    pub async fn get_organization<'a, E>(
+        organization_id: DBOrganizationId,
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<Option<Self>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        Ok(
+            Self::get_many_organizations(&[organization_id], exec, redis)
+                .await?
+                .remove(&organization_id),
+        )
+    }
+
+    pub async fn insert<'a, E>(
+        user_id: Option<DBUserId>,
+        organization_id: Option<DBOrganizationId>,
+        last_author: DBUserId,
+        notes: Option<&str>,
+        user_rating: Option<i32>,
+        exec: E,
+    ) -> Result<Option<i32>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let result = sqlx::query_scalar!(
+            r#"
+            INSERT INTO moderation_notes (user_id, organization_id, last_author, version, notes, user_rating)
+            SELECT
+              $1, $2, $3, 1, COALESCE($4::text, ''), COALESCE($5::integer, 0)
+            WHERE NOT EXISTS (
+              SELECT 1 FROM moderation_notes
+              WHERE
+                ($1::bigint IS NOT NULL AND user_id = $1)
+                OR ($2::bigint IS NOT NULL AND organization_id = $2)
+            )
+            ON CONFLICT DO NOTHING
+            RETURNING version
+            "#,
+            user_id.map(|x| x.0),
+            organization_id.map(|x| x.0),
+            last_author.0,
+            notes,
+            user_rating,
+        )
+        .fetch_optional(exec)
+        .await?;
+
+        Ok(result)
+    }
+
+    pub async fn update<'a, E>(
+        user_id: Option<DBUserId>,
+        organization_id: Option<DBOrganizationId>,
+        last_author: DBUserId,
+        expected_current_version: i32,
+        notes: Option<&str>,
+        user_rating: Option<i32>,
+        exec: E,
+    ) -> Result<Option<i32>, DatabaseError>
+    where
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let result = sqlx::query_scalar!(
+            r#"
+            UPDATE moderation_notes
+            SET
+              last_modified = NOW(),
+              last_author = $1,
+              version = version + 1,
+              notes = COALESCE($2::text, notes),
+              user_rating = COALESCE($3::integer, user_rating)
+            WHERE (
+                ($4::bigint IS NOT NULL AND user_id = $4) OR
+                ($5::bigint IS NOT NULL AND organization_id = $5)
+              )
+              AND version = $6
+            RETURNING version
+            "#,
+            last_author.0,
+            notes,
+            user_rating,
+            user_id.map(|x| x.0),
+            organization_id.map(|x| x.0),
+            expected_current_version
+        )
+        .fetch_optional(exec)
+        .await?;
+
+        Ok(result)
+    }
+
+    pub async fn clear_user_cache(
+        user_id: DBUserId,
+        redis: &RedisPool,
+    ) -> Result<(), DatabaseError> {
+        let mut redis = redis.connect().await?;
+        redis
+            .delete(MODERATION_NOTES_USERS_NAMESPACE, user_id.0)
+            .await
+    }
+
+    pub async fn clear_organization_cache(
+        organization_id: DBOrganizationId,
+        redis: &RedisPool,
+    ) -> Result<(), DatabaseError> {
+        let mut redis = redis.connect().await?;
+        redis
+            .delete(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0)
+            .await
+    }
+}

apps/labrinth/src/env.rs

@@ -120,42 +120,50 @@ impl FromStr for StringCsv {
 }
 
 vars! {
-    SENTRY_ENVIRONMENT: String;
-    SENTRY_TRACES_SAMPLE_RATE: f32;
-    SITE_URL: String;
-    CDN_URL: String;
-    LABRINTH_ADMIN_KEY: String;
-    LABRINTH_MEDAL_KEY: String;
-    LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
-    RATE_LIMIT_IGNORE_KEY: String;
-    DATABASE_URL: String;
-    REDIS_URL: String;
-    BIND_ADDR: String;
-    SELF_ADDR: String;
+    SENTRY_ENVIRONMENT: String = "development";
+    SENTRY_TRACES_SAMPLE_RATE: f32 = 0.1f32;
+    SITE_URL: String = "http://localhost:3000";
+    CDN_URL: String = "file:///tmp/modrinth";
+    LABRINTH_ADMIN_KEY: String = "";
+    LABRINTH_MEDAL_KEY: String = "";
+    LABRINTH_EXTERNAL_NOTIFICATION_KEY: String = "";
+    RATE_LIMIT_IGNORE_KEY: String = "";
+    DATABASE_URL: String = "postgresql://labrinth:labrinth@localhost/labrinth";
+    REDIS_URL: String = "redis://localhost";
+    BIND_ADDR: String = "";
+    SELF_ADDR: String = "";
 
-    LOCAL_INDEX_INTERVAL: u64;
-    VERSION_INDEX_INTERVAL: u64;
+    LOCAL_INDEX_INTERVAL: u64 = 3600u64;
+    VERSION_INDEX_INTERVAL: u64 = 1800u64;
 
-    WHITELISTED_MODPACK_DOMAINS: Json<Vec<String>>;
-    ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
-    ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
+    WHITELISTED_MODPACK_DOMAINS: Json<Vec<String>> = Json(vec![
+        "cdn.modrinth.com".into(),
+        "github.com".into(),
+        "raw.githubusercontent.com".into(),
+    ]);
+    ALLOWED_CALLBACK_URLS: Json<Vec<String>> = Json(vec![
+        "localhost".into(),
+        ".modrinth.com".into(),
+        "127.0.0.1".into(),
+        "[::1]".into(),
+    ]);
+    ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>> = Json(vec![
+        "http://127.0.0.1:3000".into(),
+        "http://localhost:3000".into(),
+        "https://modrinth.com".into(),
+        "https://www.modrinth.com".into(),
+        "*".into(),
+    ]);
 
     // search
     SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
-    MEILISEARCH_READ_ADDR: String;
-    MEILISEARCH_WRITE_ADDRS: StringCsv;
-    MEILISEARCH_KEY: String;
-    ELASTICSEARCH_URL: String;
-    ELASTICSEARCH_INDEX_PREFIX: String;
-    ELASTICSEARCH_USERNAME: String = "";
-    ELASTICSEARCH_PASSWORD: String = "";
     SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
     TYPESENSE_URL: String = "http://localhost:8108";
     TYPESENSE_API_KEY: String = "modrinth";
     TYPESENSE_INDEX_PREFIX: String = "labrinth";
 
     // storage
-    STORAGE_BACKEND: crate::file_hosting::FileHostKind;
+    STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
 
     // s3
     S3_PUBLIC_BUCKET_NAME: String = "";
@@ -173,98 +181,98 @@ vars! {
     S3_PRIVATE_SECRET: String = "";
 
     // local
-    MOCK_FILE_PATH: String = "";
+    MOCK_FILE_PATH: String = "/tmp/modrinth";
 
-    GITHUB_CLIENT_ID: String;
-    GITHUB_CLIENT_SECRET: String;
-    GITLAB_CLIENT_ID: String;
-    GITLAB_CLIENT_SECRET: String;
-    DISCORD_CLIENT_ID: String;
-    DISCORD_CLIENT_SECRET: String;
-    MICROSOFT_CLIENT_ID: String;
-    MICROSOFT_CLIENT_SECRET: String;
-    GOOGLE_CLIENT_ID: String;
-    GOOGLE_CLIENT_SECRET: String;
-    STEAM_API_KEY: String;
+    GITHUB_CLIENT_ID: String = "none";
+    GITHUB_CLIENT_SECRET: String = "none";
+    GITLAB_CLIENT_ID: String = "none";
+    GITLAB_CLIENT_SECRET: String = "none";
+    DISCORD_CLIENT_ID: String = "none";
+    DISCORD_CLIENT_SECRET: String = "none";
+    MICROSOFT_CLIENT_ID: String = "none";
+    MICROSOFT_CLIENT_SECRET: String = "none";
+    GOOGLE_CLIENT_ID: String = "none";
+    GOOGLE_CLIENT_SECRET: String = "none";
+    STEAM_API_KEY: String = "none";
 
-    TREMENDOUS_API_URL: String;
-    TREMENDOUS_API_KEY: String;
-    TREMENDOUS_PRIVATE_KEY: String;
+    TREMENDOUS_API_URL: String = "https://testflight.tremendous.com/api/v2/";
+    TREMENDOUS_API_KEY: String = "none";
+    TREMENDOUS_PRIVATE_KEY: String = "none";
 
-    PAYPAL_API_URL: String;
-    PAYPAL_WEBHOOK_ID: String;
-    PAYPAL_CLIENT_ID: String;
-    PAYPAL_CLIENT_SECRET: String;
-    PAYPAL_NVP_USERNAME: String;
-    PAYPAL_NVP_PASSWORD: String;
-    PAYPAL_NVP_SIGNATURE: String;
+    PAYPAL_API_URL: String = "https://api-m.sandbox.paypal.com/v1/";
+    PAYPAL_WEBHOOK_ID: String = "none";
+    PAYPAL_CLIENT_ID: String = "none";
+    PAYPAL_CLIENT_SECRET: String = "none";
+    PAYPAL_NVP_USERNAME: String = "none";
+    PAYPAL_NVP_PASSWORD: String = "none";
+    PAYPAL_NVP_SIGNATURE: String = "none";
 
     PAYPAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
     BREX_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
     TREMENDOUS_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
     MURAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
 
-    HCAPTCHA_SECRET: String;
+    HCAPTCHA_SECRET: String = "none";
 
-    SMTP_USERNAME: String;
-    SMTP_PASSWORD: String;
-    SMTP_HOST: String;
-    SMTP_PORT: u16;
-    SMTP_TLS: String;
-    SMTP_FROM_NAME: String;
-    SMTP_FROM_ADDRESS: String;
+    SMTP_USERNAME: String = "";
+    SMTP_PASSWORD: String = "";
+    SMTP_HOST: String = "localhost";
+    SMTP_PORT: u16 = 1025u16;
+    SMTP_TLS: String = "none";
+    SMTP_FROM_NAME: String = "Modrinth";
+    SMTP_FROM_ADDRESS: String = "no-reply@mail.modrinth.com";
 
-    SITE_VERIFY_EMAIL_PATH: String;
-    SITE_RESET_PASSWORD_PATH: String;
-    SITE_BILLING_PATH: String;
+    SITE_VERIFY_EMAIL_PATH: String = "auth/verify-email";
+    SITE_RESET_PASSWORD_PATH: String = "auth/reset-password";
+    SITE_BILLING_PATH: String = "none";
 
-    SENDY_URL: String;
-    SENDY_LIST_ID: String;
-    SENDY_API_KEY: String;
+    SENDY_URL: String = "none";
+    SENDY_LIST_ID: String = "none";
+    SENDY_API_KEY: String = "none";
 
-    CLICKHOUSE_REPLICATED: bool;
-    CLICKHOUSE_URL: String;
-    CLICKHOUSE_USER: String;
-    CLICKHOUSE_PASSWORD: String;
-    CLICKHOUSE_DATABASE: String;
+    CLICKHOUSE_REPLICATED: bool = false;
+    CLICKHOUSE_URL: String = "http://localhost:8123";
+    CLICKHOUSE_USER: String = "default";
+    CLICKHOUSE_PASSWORD: String = "default";
+    CLICKHOUSE_DATABASE: String = "staging_ariadne";
 
-    FLAME_ANVIL_URL: String;
+    FLAME_ANVIL_URL: String = "none";
 
-    GOTENBERG_URL: String;
-    GOTENBERG_CALLBACK_BASE: String;
-    GOTENBERG_TIMEOUT: u64;
+    GOTENBERG_URL: String = "http://localhost:13000";
+    GOTENBERG_CALLBACK_BASE: String = "http://host.docker.internal:8000/_internal/gotenberg";
+    GOTENBERG_TIMEOUT: u64 = 30000u64;
 
-    STRIPE_API_KEY: String;
-    STRIPE_WEBHOOK_SECRET: String;
+    STRIPE_API_KEY: String = "none";
+    STRIPE_WEBHOOK_SECRET: String = "none";
 
-    ADITUDE_API_KEY: String;
+    ADITUDE_API_KEY: String = "none";
 
-    PYRO_API_KEY: String;
+    PYRO_API_KEY: String = "none";
 
-    BREX_API_URL: String;
-    BREX_API_KEY: String;
+    BREX_API_URL: String = "https://platform.brexapis.com/v2/";
+    BREX_API_KEY: String = "none";
 
-    DELPHI_URL: String;
+    DELPHI_URL: String = "";
 
-    AVALARA_1099_API_URL: String;
-    AVALARA_1099_API_KEY: String;
-    AVALARA_1099_API_TEAM_ID: String;
-    AVALARA_1099_COMPANY_ID: String;
+    AVALARA_1099_API_URL: String = "https://www.track1099.com/api";
+    AVALARA_1099_API_KEY: String = "none";
+    AVALARA_1099_API_TEAM_ID: String = "none";
+    AVALARA_1099_COMPANY_ID: String = "207337084";
 
-    ANROK_API_URL: String;
-    ANROK_API_KEY: String;
+    ANROK_API_URL: String = "";
+    ANROK_API_KEY: String = "";
 
-    PAYOUT_ALERT_SLACK_WEBHOOK: String;
+    PAYOUT_ALERT_SLACK_WEBHOOK: String = "none";
     CLOUDFLARE_INTEGRATION: bool = false;
 
-    ARCHON_URL: String;
+    ARCHON_URL: String = "";
 
-    MURALPAY_API_URL: String;
-    MURALPAY_API_KEY: String;
-    MURALPAY_TRANSFER_API_KEY: String;
+    MURALPAY_API_URL: String = "https://api-staging.muralpay.com";
+    MURALPAY_API_KEY: String = "none";
+    MURALPAY_TRANSFER_API_KEY: String = "none";
     MURALPAY_SOURCE_ACCOUNT_ID: muralpay::AccountId = muralpay::AccountId(uuid::Uuid::nil());
 
-    DEFAULT_AFFILIATE_REVENUE_SPLIT: Decimal;
+    DEFAULT_AFFILIATE_REVENUE_SPLIT: Decimal = Decimal::new(1, 1);
 
     DATABASE_ACQUIRE_TIMEOUT_MS: u64 = 30000u64;
     DATABASE_MIN_CONNECTIONS: u32 = 0u32;
@@ -286,7 +294,7 @@ vars! {
     MODERATION_SLACK_WEBHOOK: String = "";
     DELPHI_SLACK_WEBHOOK: String = "";
 
-    TREMENDOUS_CAMPAIGN_ID: String = "";
+    TREMENDOUS_CAMPAIGN_ID: String = "none";
 
     // server pinging
     SERVER_PING_MAX_CONCURRENT: usize = 16usize;

apps/labrinth/src/models/mod.rs

@@ -8,6 +8,7 @@ pub use v3::billing;
 pub use v3::collections;
 pub use v3::ids;
 pub use v3::images;
+pub use v3::moderation_notes;
 pub use v3::notifications;
 pub use v3::oauth_clients;
 pub use v3::organizations;

apps/labrinth/src/models/v3/mod.rs

@@ -4,6 +4,7 @@ pub mod billing;
 pub mod collections;
 pub mod ids;
 pub mod images;
+pub mod moderation_notes;
 pub mod notifications;
 pub mod oauth_clients;
 pub mod organizations;

apps/labrinth/src/models/v3/moderation_notes.rs

@@ -0,0 +1,68 @@
+use actix_web::{HttpRequest, http::header::IF_MATCH};
+use ariadne::ids::UserId;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+use crate::routes::ApiError;
+
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+pub struct ModerationNote {
+    pub notes: String,
+    pub last_modified: DateTime<Utc>,
+    pub created_at: DateTime<Utc>,
+    pub last_author: UserId,
+    pub user_rating: i32,
+    pub version: i32,
+}
+
+impl From<crate::database::models::DBModerationNote> for ModerationNote {
+    fn from(note: crate::database::models::DBModerationNote) -> Self {
+        Self {
+            notes: note.notes,
+            last_modified: note.last_modified,
+            created_at: note.created_at,
+            last_author: note.last_author.into(),
+            user_rating: note.user_rating,
+            version: note.version,
+        }
+    }
+}
+
+#[derive(Deserialize)]
+pub struct PatchModerationNote {
+    pub notes: Option<String>,
+    pub user_rating: Option<i32>,
+}
+
+impl PatchModerationNote {
+    pub fn validate_not_empty(&self) -> Result<(), ApiError> {
+        if self.notes.is_none() && self.user_rating.is_none() {
+            return Err(ApiError::InvalidInput(
+                "must specify `notes` or `user_rating`".to_string(),
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+pub fn parse_if_match_header(
+    req: &HttpRequest,
+) -> Result<Option<i32>, ApiError> {
+    let Some(value) = req.headers().get(IF_MATCH) else {
+        return Ok(None);
+    };
+
+    let value = value.to_str().map_err(|_| {
+        ApiError::InvalidInput(
+            "`if-match` header must be a valid integer".to_string(),
+        )
+    })?;
+
+    Some(value.parse::<i32>().map_err(|_| {
+        ApiError::InvalidInput(
+            "`if-match` header must be a valid integer".to_string(),
+        )
+    }))
+    .transpose()
+}

apps/labrinth/src/models/v3/organizations.rs

@@ -1,3 +1,4 @@
+use super::moderation_notes::ModerationNote;
 use super::teams::TeamMember;
 use crate::models::ids::{OrganizationId, TeamId};
 use serde::{Deserialize, Serialize};
@@ -23,6 +24,8 @@ pub struct Organization {
 
     /// A list of the members of the organization
     pub members: Vec<TeamMember>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub moderation_notes: Option<Option<ModerationNote>>,
 }
 
 impl Organization {
@@ -39,6 +42,7 @@ impl Organization {
             members: team_members,
             icon_url: data.icon_url,
             color: data.color,
+            moderation_notes: None,
         }
     }
 }

apps/labrinth/src/models/v3/users.rs

@@ -1,3 +1,4 @@
+use super::moderation_notes::ModerationNote;
 use crate::{auth::AuthProvider, bitflags_serde_impl};
 use ariadne::ids::UserId;
 pub use ariadne::users::UserStatus;
@@ -64,6 +65,8 @@ pub struct User {
     pub payout_data: Option<UserPayoutData>,
     pub stripe_customer_id: Option<String>,
     pub allow_friend_requests: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub moderation_notes: Option<Option<ModerationNote>>,
 
     // DEPRECATED. Always returns None
     pub github_id: Option<u64>,
@@ -98,6 +101,7 @@ impl From<DBUser> for User {
             github_id: None,
             stripe_customer_id: None,
             allow_friend_requests: None,
+            moderation_notes: None,
         }
     }
 }
@@ -150,6 +154,7 @@ impl User {
             }),
             stripe_customer_id: db_user.stripe_customer_id,
             allow_friend_requests: Some(db_user.allow_friend_requests),
+            moderation_notes: None,
         }
     }
 }

apps/labrinth/src/routes/mod.rs

@@ -141,6 +141,10 @@ pub enum ApiError {
     NotFound,
     #[error("Conflict: {0}")]
     Conflict(String),
+    #[error("precondition required: {0}")]
+    PreconditionRequired(String),
+    #[error("precondition failed: {0}")]
+    PreconditionFailed(String),
     #[error("External tax compliance API error")]
     TaxComplianceApi,
     #[error(transparent)]
@@ -194,6 +198,8 @@ impl ApiError {
                 Self::Reroute(..) => "reroute_error",
                 Self::NotFound => "not_found",
                 Self::Conflict(..) => "conflict",
+                Self::PreconditionRequired(..) => "precondition_required",
+                Self::PreconditionFailed(..) => "precondition_failed",
                 Self::TaxComplianceApi => "tax_compliance_api_error",
                 Self::Zip(..) => "zip_error",
                 Self::Io(..) => "io_error",
@@ -256,6 +262,8 @@ impl actix_web::ResponseError for ApiError {
             Self::Reroute(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::NotFound => StatusCode::NOT_FOUND,
             Self::Conflict(..) => StatusCode::CONFLICT,
+            Self::PreconditionRequired(..) => StatusCode::PRECONDITION_REQUIRED,
+            Self::PreconditionFailed(..) => StatusCode::PRECONDITION_FAILED,
             Self::TaxComplianceApi => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Zip(..) => StatusCode::BAD_REQUEST,
             Self::Io(..) => StatusCode::BAD_REQUEST,

apps/labrinth/src/routes/v2/users.rs

@@ -78,14 +78,18 @@ pub struct UserIds {
 )]
 #[get("/users")]
 pub async fn users_get(
+    req: HttpRequest,
     web::Query(ids): web::Query<UserIds>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     let response = v3::users::users_get(
+        req,
         web::Query(v3::users::UserIds { ids: ids.ids }),
         pool,
         redis,
+        session_queue,
     )
     .await
     .or_else(v2_reroute::flatten_404_error)?;

apps/labrinth/src/routes/v3/organizations.rs

@@ -7,7 +7,7 @@ use crate::auth::{filter_visible_projects, get_user_from_headers};
 use crate::database::PgPool;
 use crate::database::models::team_item::DBTeamMember;
 use crate::database::models::{
-    DBOrganization, generate_organization_id, team_item,
+    DBModerationNote, DBOrganization, generate_organization_id, team_item,
 };
 use crate::database::redis::RedisPool;
 use crate::file_hosting::{FileHost, FileHostPublicity};
@@ -34,6 +34,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
         web::scope("organization")
             .route("", web::post().to(organization_create))
             .route("{id}/projects", web::get().to(organization_projects_get))
+            .route("{id}/notes", web::patch().to(organization_notes_edit))
             .route("{id}", web::get().to(organization_get))
             .route("{id}", web::patch().to(organizations_edit))
             .route("{id}", web::delete().to(organization_delete))
@@ -283,13 +284,97 @@ pub async fn organization_get(
             })
             .collect();
 
-        let organization =
+        let mut organization =
             models::organizations::Organization::from(data, team_members);
+        if current_user.as_ref().is_some_and(|x| x.role.is_mod()) {
+            let note = DBModerationNote::get_organization(
+                organization.id.into(),
+                &**pool,
+                &redis,
+            )
+            .await?;
+            organization.moderation_notes = Some(note.map(Into::into));
+        }
         return Ok(HttpResponse::Ok().json(organization));
     }
     Err(ApiError::NotFound)
 }
 
+pub async fn organization_notes_edit(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    new_note: web::Json<crate::models::moderation_notes::PatchModerationNote>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    let user = get_user_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::SESSION_ACCESS,
+    )
+    .await?
+    .1;
+
+    if !user.role.is_mod() {
+        return Err(ApiError::CustomAuthentication(
+            "you do not have permission to edit moderation notes".to_string(),
+        ));
+    }
+
+    new_note.validate_not_empty()?;
+    let expected_version =
+        crate::models::moderation_notes::parse_if_match_header(&req)?;
+
+    let organization =
+        DBOrganization::get(&info.into_inner().0, &**pool, &redis)
+            .await?
+            .ok_or(ApiError::NotFound)?;
+
+    let mut transaction = pool.begin().await?;
+    if let Some(expected) = expected_version {
+        let updated = DBModerationNote::update(
+            None,
+            Some(organization.id),
+            user.id.into(),
+            expected,
+            new_note.notes.as_deref(),
+            new_note.user_rating,
+            &mut transaction,
+        )
+        .await?;
+
+        if updated.is_none() {
+            return Err(ApiError::PreconditionFailed(
+                "moderation note version does not match".to_string(),
+            ));
+        }
+    } else {
+        let updated = DBModerationNote::insert(
+            None,
+            Some(organization.id),
+            user.id.into(),
+            new_note.notes.as_deref(),
+            new_note.user_rating,
+            &mut transaction,
+        )
+        .await?;
+
+        if updated.is_none() {
+            return Err(ApiError::PreconditionRequired(
+                "moderation note version does not match".to_string(),
+            ));
+        }
+    };
+
+    transaction.commit().await?;
+    DBModerationNote::clear_organization_cache(organization.id, &redis).await?;
+
+    Ok(HttpResponse::NoContent().finish())
+}
+
 #[derive(Deserialize)]
 pub struct OrganizationIds {
     pub ids: String,
@@ -331,6 +416,17 @@ pub async fn organizations_get(
     .map(|x| x.1)
     .ok();
     let user_id = current_user.as_ref().map(|x| x.id.into());
+    let include_notes = current_user.as_ref().is_some_and(|x| x.role.is_mod());
+    let notes = if include_notes {
+        DBModerationNote::get_many_organizations(
+            &organizations_data.iter().map(|x| x.id).collect::<Vec<_>>(),
+            &**pool,
+            &redis,
+        )
+        .await?
+    } else {
+        HashMap::new()
+    };
 
     let mut organizations = vec![];
 
@@ -375,8 +471,13 @@ pub async fn organizations_get(
             })
             .collect();
 
-        let organization =
+        let data_id = data.id;
+        let mut organization =
             models::organizations::Organization::from(data, team_members);
+        if include_notes {
+            organization.moderation_notes =
+                Some(notes.get(&data_id).cloned().map(Into::into));
+        }
         organizations.push(organization);
     }
 

apps/labrinth/src/routes/v3/users.rs

@@ -5,10 +5,14 @@ use crate::database::PgPool;
 use crate::util::error::Context;
 use crate::{
     auth::{
-        checks::is_visible_organization, filter_visible_collections,
-        filter_visible_projects, get_user_from_headers,
+        check_is_moderator_from_headers, checks::is_visible_organization,
+        filter_visible_collections, filter_visible_projects,
+        get_user_from_headers,
+    },
+    database::{
+        models::{DBModerationNote, DBUser},
+        redis::RedisPool,
     },
-    database::{models::DBUser, redis::RedisPool},
     file_hosting::{FileHost, FileHostPublicity},
     models::{
         notifications::Notification,
@@ -35,6 +39,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::scope("user")
             .route("{user_id}/projects", web::get().to(projects_list))
+            .route("{id}/notes", web::patch().to(user_notes_edit))
             .route("{id}", web::get().to(user_get))
             .route("{user_id}/collections", web::get().to(collections_list))
             .route("{user_id}/organizations", web::get().to(orgs_list))
@@ -167,6 +172,12 @@ pub async fn user_auth_get(
         user.payout_data = None;
     }
 
+    if user.role.is_mod() {
+        let note =
+            DBModerationNote::get_user(user.id.into(), &**pool, &redis).await?;
+        user.moderation_notes = Some(note.map(Into::into));
+    }
+
     Ok(HttpResponse::Ok().json(user))
 }
 
@@ -176,16 +187,49 @@ pub struct UserIds {
 }
 
 pub async fn users_get(
+    req: HttpRequest,
     web::Query(ids): web::Query<UserIds>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     let user_ids = serde_json::from_str::<Vec<String>>(&ids.ids)?;
 
     let users_data = DBUser::get_many(&user_ids, &**pool, &redis).await?;
 
-    let users: Vec<crate::models::users::User> =
-        users_data.into_iter().map(From::from).collect();
+    let auth_user = get_user_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::SESSION_ACCESS,
+    )
+    .await
+    .map(|x| x.1)
+    .ok();
+
+    let notes = if auth_user.as_ref().is_some_and(|x| x.role.is_mod()) {
+        DBModerationNote::get_many_users(
+            &users_data.iter().map(|x| x.id).collect::<Vec<_>>(),
+            &**pool,
+            &redis,
+        )
+        .await?
+    } else {
+        HashMap::new()
+    };
+
+    let users: Vec<crate::models::users::User> = users_data
+        .into_iter()
+        .map(|data| {
+            let mut user = crate::models::users::User::from(data.clone());
+            if auth_user.as_ref().is_some_and(|x| x.role.is_mod()) {
+                user.moderation_notes =
+                    Some(notes.get(&data.id).cloned().map(Into::into));
+            }
+            user
+        })
+        .collect();
 
     Ok(HttpResponse::Ok().json(users))
 }
@@ -211,12 +255,21 @@ pub async fn user_get(
         .map(|x| x.1)
         .ok();
 
-        let response: crate::models::users::User =
-            if auth_user.is_some_and(|x| x.role.is_admin()) {
-                crate::models::users::User::from_full(data)
-            } else {
-                data.into()
-            };
+        let is_admin = auth_user.as_ref().is_some_and(|x| x.role.is_admin());
+        let is_mod = auth_user.as_ref().is_some_and(|x| x.role.is_mod());
+        let user_id = data.id;
+
+        let mut response: crate::models::users::User = if is_admin {
+            crate::models::users::User::from_full(data)
+        } else {
+            data.into()
+        };
+
+        if is_mod {
+            let note =
+                DBModerationNote::get_user(user_id, &**pool, &redis).await?;
+            response.moderation_notes = Some(note.map(Into::into));
+        }
 
         Ok(HttpResponse::Ok().json(response))
     } else {
@@ -224,6 +277,73 @@ pub async fn user_get(
     }
 }
 
+pub async fn user_notes_edit(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    new_note: web::Json<crate::models::moderation_notes::PatchModerationNote>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    let user = check_is_moderator_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::SESSION_ACCESS,
+    )
+    .await?;
+
+    new_note.validate_not_empty()?;
+    let expected_version =
+        crate::models::moderation_notes::parse_if_match_header(&req)?;
+
+    let user_data = DBUser::get(&info.into_inner().0, &**pool, &redis)
+        .await?
+        .ok_or(ApiError::NotFound)?;
+
+    let mut transaction = pool.begin().await?;
+    if let Some(expected) = expected_version {
+        let updated = DBModerationNote::update(
+            Some(user_data.id),
+            None,
+            user.id.into(),
+            expected,
+            new_note.notes.as_deref(),
+            new_note.user_rating,
+            &mut transaction,
+        )
+        .await?;
+
+        if updated.is_none() {
+            return Err(ApiError::PreconditionFailed(
+                "moderation note version does not match".to_string(),
+            ));
+        }
+    } else {
+        let updated = DBModerationNote::insert(
+            Some(user_data.id),
+            None,
+            user.id.into(),
+            new_note.notes.as_deref(),
+            new_note.user_rating,
+            &mut transaction,
+        )
+        .await?;
+
+        if updated.is_none() {
+            return Err(ApiError::PreconditionRequired(
+                "moderation note version does not match".to_string(),
+            ));
+        }
+    };
+
+    transaction.commit().await?;
+    DBModerationNote::clear_user_cache(user_data.id, &redis).await?;
+
+    Ok(HttpResponse::NoContent().finish())
+}
+
 pub async fn collections_list(
     req: HttpRequest,
     info: web::Path<(String,)>,

apps/labrinth/src/search/backend/meilisearch/indexing.rs

@@ -1,701 +0,0 @@
-use std::sync::LazyLock;
-use std::time::Duration;
-
-use crate::database::PgPool;
-use crate::database::redis::RedisPool;
-use crate::env::ENV;
-use crate::search::backend::meilisearch::MeilisearchConfig;
-use crate::search::indexing::index_local;
-use crate::search::{SearchField, UploadSearchProject};
-use crate::util::error::Context;
-use ariadne::ids::base62_impl::to_base62;
-use eyre::{Result, eyre};
-use futures::StreamExt;
-use futures::stream::FuturesOrdered;
-use meilisearch_sdk::client::{Client, SwapIndexes};
-use meilisearch_sdk::indexes::Index;
-use meilisearch_sdk::settings::{PaginationSetting, Settings};
-use meilisearch_sdk::task_info::TaskInfo;
-use tracing::{Instrument, error, info, info_span, instrument};
-
-// // The chunk size for adding projects to the indexing database. If the request size
-// // is too large (>10MiB) then the request fails with an error.  This chunk size
-// // assumes a max average size of 4KiB per project to avoid this cap.
-//
-// Set this to 50k for better observability
-const MEILISEARCH_CHUNK_SIZE: usize = 50000; // 10_000_000
-
-fn search_operation_timeout() -> std::time::Duration {
-    std::time::Duration::from_millis(ENV.SEARCH_OPERATION_TIMEOUT)
-}
-
-pub async fn remove_documents(
-    ids: &[crate::models::ids::VersionId],
-    config: &MeilisearchConfig,
-) -> Result<()> {
-    let mut indexes = get_indexes_for_indexing(config, false, false)
-        .await
-        .wrap_err("failed to get current indexes")?;
-    let indexes_next = get_indexes_for_indexing(config, true, false)
-        .await
-        .wrap_err("failed to get next indexes")?;
-
-    for list in &mut indexes {
-        for alt_list in &indexes_next {
-            list.extend(alt_list.iter().cloned());
-        }
-    }
-
-    let client = config
-        .make_batch_client()
-        .wrap_err("failed to create batch client")?;
-    let client = &client;
-
-    let ids_base62 = ids.iter().map(|x| to_base62(x.0)).collect::<Vec<_>>();
-    let mut deletion_tasks = FuturesOrdered::new();
-
-    client.across_all(indexes, |index_list, client| {
-        for index in index_list {
-            let owned_client = client.clone();
-            let ids_base62_ref = &ids_base62;
-            deletion_tasks.push_back(async move {
-                index
-                    .delete_documents(ids_base62_ref)
-                    .await
-                    .wrap_err_with(|| {
-                        eyre!("failed to request to delete documents {ids_base62_ref:?}")
-                    })?
-                    .wait_for_completion(
-                        &owned_client,
-                        None,
-                        Some(Duration::from_secs(15)),
-                    )
-                    .await
-                    .wrap_err_with(|| {
-                        eyre!("failed to delete documents {ids_base62_ref:?}")
-                    })
-            });
-        }
-    });
-
-    while let Some(result) = deletion_tasks.next().await {
-        result?;
-    }
-
-    Ok(())
-}
-
-pub async fn index_projects(
-    ro_pool: PgPool,
-    redis: RedisPool,
-    config: &MeilisearchConfig,
-) -> Result<()> {
-    info!("Indexing projects.");
-
-    info!("Ensuring current indexes exists");
-    // First, ensure current index exists (so no error happens- current index should be worst-case empty, not missing)
-    get_indexes_for_indexing(config, false, false)
-        .await
-        .wrap_err("failed to get indexes for indexing")?;
-
-    info!("Deleting surplus indexes");
-    // Then, delete the next index if it still exists
-    let indices = get_indexes_for_indexing(config, true, false)
-        .await
-        .wrap_err("failed to get next indexes to delete")?;
-    for client_indices in indices {
-        for index in client_indices {
-            index.delete().await.wrap_err("failed to delete an index")?;
-        }
-    }
-
-    info!("Recreating next index");
-    // Recreate the next index for indexing
-    let indices = get_indexes_for_indexing(config, true, true)
-        .await
-        .wrap_internal_err("failed to recreate next index")?;
-
-    let all_loader_fields =
-        crate::database::models::loader_fields::LoaderField::get_fields_all(
-            &ro_pool, &redis,
-        )
-        .await
-        .wrap_internal_err("failed to get all loader fields")?
-        .into_iter()
-        .map(|x| x.field)
-        .collect::<Vec<_>>();
-
-    info!("Gathering local projects");
-
-    let mut cursor = 0;
-    let mut idx = 0;
-    let mut total = 0;
-
-    loop {
-        info!("Gathering index data chunk {idx}");
-        idx += 1;
-
-        let (uploads, next_cursor) =
-            index_local(&ro_pool, &redis, cursor, 10000).await?;
-        total += uploads.len();
-
-        if uploads.is_empty() {
-            info!(
-                "No more projects to index, indexed {total} projects after {idx} chunks"
-            );
-            break;
-        }
-
-        cursor = next_cursor;
-
-        add_projects_batch_client(
-            &indices,
-            uploads,
-            all_loader_fields.clone(),
-            config,
-        )
-        .await?;
-    }
-
-    info!("Swapping indexes");
-
-    // Swap the index
-    swap_index(config, "projects").await?;
-    swap_index(config, "projects_filtered").await?;
-
-    info!("Deleting old indexes");
-
-    // Delete the now-old index
-    for index_list in indices {
-        for index in index_list {
-            index.delete().await?;
-        }
-    }
-
-    info!("Done adding projects.");
-    Ok(())
-}
-
-pub async fn swap_index(
-    config: &MeilisearchConfig,
-    index_name: &str,
-) -> Result<()> {
-    let client = config.make_batch_client()?;
-    let index_name_next = config.get_index_name(index_name, true);
-    let index_name = config.get_index_name(index_name, false);
-    let swap_indices = SwapIndexes {
-        indexes: (index_name_next, index_name),
-        rename: None,
-    };
-
-    let swap_indices_ref = &swap_indices;
-
-    // is it "indexes" or "indices"? who knows! roll a die!
-    client
-        .with_all_clients("swap_indexes", |client| async move {
-            let task = client
-                .swap_indexes([swap_indices_ref])
-                .await
-                .wrap_err("failed to swap indices")?;
-
-            monitor_task(
-                client,
-                task,
-                Duration::from_secs(60 * 10), // 10 minutes
-                Some(Duration::from_secs(1)),
-            )
-            .await?;
-            Ok(())
-        })
-        .await?;
-
-    Ok(())
-}
-
-#[instrument(skip(config))]
-pub async fn get_indexes_for_indexing(
-    config: &MeilisearchConfig,
-    next: bool, // Get the 'next' one
-    update_settings: bool,
-) -> Result<Vec<Vec<Index>>> {
-    let client = config.make_batch_client()?;
-    let project_name = config.get_index_name("projects", next);
-    let project_filtered_name =
-        config.get_index_name("projects_filtered", next);
-
-    let project_name_ref = &project_name;
-    let project_filtered_name_ref = &project_filtered_name;
-
-    let results = client
-        .with_all_clients("get_indexes_for_indexing", |client| async move {
-            let projects_index = create_or_update_index(
-                client,
-                project_name_ref,
-                Some(&[
-                    "words",
-                    "typo",
-                    "proximity",
-                    "attribute",
-                    "exactness",
-                    "sort",
-                ]),
-                update_settings,
-            )
-            .await?;
-            let projects_filtered_index = create_or_update_index(
-                client,
-                project_filtered_name_ref,
-                Some(&[
-                    "sort",
-                    "words",
-                    "typo",
-                    "proximity",
-                    "attribute",
-                    "exactness",
-                ]),
-                update_settings,
-            )
-            .await?;
-
-            Ok(vec![projects_index, projects_filtered_index])
-        })
-        .await?;
-
-    Ok(results)
-}
-
-#[instrument(skip_all, fields(name))]
-async fn create_or_update_index(
-    client: &Client,
-    name: &str,
-    custom_rules: Option<&'static [&'static str]>,
-    update_settings: bool,
-) -> Result<Index, meilisearch_sdk::errors::Error> {
-    info!("Updating/creating index");
-
-    match client.get_index(name).await {
-        Ok(index) => {
-            info!("Updating index settings.");
-
-            let mut settings = default_settings();
-
-            if let Some(custom_rules) = custom_rules {
-                settings = settings.with_ranking_rules(custom_rules);
-            }
-
-            if update_settings {
-                info!("Updating index settings");
-                index
-                    .set_settings(&settings)
-                    .await
-                    .inspect_err(|e| {
-                        error!("Error setting index settings: {e:?}")
-                    })?
-                    .wait_for_completion(
-                        client,
-                        None,
-                        Some(search_operation_timeout()),
-                    )
-                    .await
-                    .inspect_err(|e| {
-                        error!(
-                            "Error setting index settings while waiting: {e:?}"
-                        )
-                    })?;
-            }
-            info!("Done performing index settings set.");
-
-            Ok(index)
-        }
-        _ => {
-            info!("Creating index.");
-
-            // Only create index and set settings if the index doesn't already exist
-            let task = client.create_index(name, Some("version_id")).await?;
-            let task = task
-                .wait_for_completion(
-                    client,
-                    None,
-                    Some(search_operation_timeout()),
-                )
-                .await
-                .inspect_err(|e| {
-                    error!("Error creating index while waiting: {e:?}")
-                })?;
-            let index = task
-                .try_make_index(client)
-                .map_err(|x| x.unwrap_failure())?;
-
-            let mut settings = default_settings();
-
-            if let Some(custom_rules) = custom_rules {
-                settings = settings.with_ranking_rules(custom_rules);
-            }
-
-            if update_settings {
-                index
-                    .set_settings(&settings)
-                    .await
-                    .inspect_err(|e| {
-                        error!("Error setting index settings: {e:?}")
-                    })?
-                    .wait_for_completion(
-                        client,
-                        None,
-                        Some(search_operation_timeout()),
-                    )
-                    .await
-                    .inspect_err(|e| {
-                        error!(
-                            "Error setting index settings while waiting: {e:?}"
-                        )
-                    })?;
-            }
-
-            Ok(index)
-        }
-    }
-}
-
-#[instrument(skip_all, fields(%index.uid, mods.len = mods.len()))]
-async fn add_to_index(
-    client: &Client,
-    index: &Index,
-    mods: &[UploadSearchProject],
-) -> Result<()> {
-    for chunk in mods.chunks(MEILISEARCH_CHUNK_SIZE) {
-        info!(
-            "Adding chunk of {} versions starting with version id {}",
-            chunk.len(),
-            chunk[0].version_id
-        );
-
-        let now = std::time::Instant::now();
-
-        let task = index
-            .add_or_replace(chunk, Some("version_id"))
-            .await
-            .inspect_err(|e| error!("Error adding chunk to index: {e:?}"))?;
-
-        monitor_task(
-            client,
-            task,
-            Duration::from_secs(60 * 5), // Timeout after 10 minutes
-            Some(Duration::from_secs(1)), // Poll once every second
-        )
-        .await?;
-
-        info!(
-            "Added chunk of {} projects to index in {:.2} seconds",
-            chunk.len(),
-            now.elapsed().as_secs_f64()
-        );
-    }
-
-    Ok(())
-}
-
-async fn monitor_task(
-    client: &Client,
-    task: TaskInfo,
-    timeout: Duration,
-    poll: Option<Duration>,
-) -> Result<()> {
-    let now = std::time::Instant::now();
-
-    let id = task.get_task_uid();
-    let mut interval = tokio::time::interval(Duration::from_secs(30));
-    interval.reset();
-
-    let wait = task.wait_for_completion(client, poll, Some(timeout));
-
-    tokio::select! {
-        biased;
-
-        result = wait => {
-            info!("Task {id} completed in {:.2} seconds: {result:?}", now.elapsed().as_secs_f64());
-            result?;
-        }
-
-        _ = interval.tick() => {
-            struct Id(u32);
-
-            impl AsRef<u32> for Id {
-                fn as_ref(&self) -> &u32 {
-                    &self.0
-                }
-            }
-
-            // it takes an AsRef<u32> but u32 itself doesn't impl it lol
-            if let Ok(task) = client.get_task(Id(id)).await {
-                if task.is_pending() {
-                    info!("Task {id} is still pending after {:.2} seconds", now.elapsed().as_secs_f64());
-                }
-            } else {
-                error!("Error getting task {id}");
-            }
-        }
-    };
-
-    Ok(())
-}
-
-#[instrument(skip_all, fields(index.uid = %index.uid))]
-async fn update_and_add_to_index(
-    client: &Client,
-    index: &Index,
-    projects: &[UploadSearchProject],
-    _additional_fields: &[String],
-) -> Result<()> {
-    // TODO: Uncomment this- hardcoding loader_fields is a band-aid fix, and will be fixed soon
-    // let mut new_filterable_attributes: Vec<String> = index.get_filterable_attributes().await?;
-    // let mut new_displayed_attributes = index.get_displayed_attributes().await?;
-
-    // // Check if any 'additional_fields' are not already in the index
-    // // Only add if they are not already in the index
-    // let new_fields = additional_fields
-    //     .iter()
-    //     .filter(|x| !new_filterable_attributes.contains(x))
-    //     .collect::<Vec<_>>();
-    // if !new_fields.is_empty() {
-    //     info!("Adding new fields to index: {:?}", new_fields);
-    //     new_filterable_attributes.extend(new_fields.iter().map(|s: &&String| s.to_string()));
-    //     new_displayed_attributes.extend(new_fields.iter().map(|s| s.to_string()));
-
-    //     // Adds new fields to the index
-    //     let filterable_task = index
-    //         .set_filterable_attributes(new_filterable_attributes)
-    //         .await?;
-    //     let displayable_task = index
-    //         .set_displayed_attributes(new_displayed_attributes)
-    //         .await?;
-
-    //     // Allow a long timeout for adding new attributes- it only needs to happen the once
-    //     filterable_task
-    //         .wait_for_completion(client, None, Some(search_operation_timeout() * 100))
-    //         .await?;
-    //     displayable_task
-    //         .wait_for_completion(client, None, Some(search_operation_timeout() * 100))
-    //         .await?;
-    // }
-
-    info!("Adding to index.");
-
-    add_to_index(client, index, projects).await?;
-
-    Ok(())
-}
-
-pub async fn add_projects_batch_client(
-    indices: &[Vec<Index>],
-    projects: Vec<UploadSearchProject>,
-    additional_fields: Vec<String>,
-    config: &MeilisearchConfig,
-) -> Result<()> {
-    let client = config.make_batch_client()?;
-
-    let index_references = indices
-        .iter()
-        .map(|x| x.iter().collect())
-        .collect::<Vec<Vec<&Index>>>();
-
-    let mut tasks = FuturesOrdered::new();
-
-    let mut id = 0;
-
-    client.across_all(index_references, |index_list, client| {
-        let span = info_span!("add_projects_batch", client.idx = id);
-        id += 1;
-
-        for index in index_list {
-            let owned_client = client.clone();
-            let projects_ref = &projects;
-            let additional_fields_ref = &additional_fields;
-            tasks.push_back(
-                async move {
-                    update_and_add_to_index(
-                        &owned_client,
-                        index,
-                        projects_ref,
-                        additional_fields_ref,
-                    )
-                    .await
-                }
-                .instrument(span.clone()),
-            );
-        }
-    });
-
-    while let Some(result) = tasks.next().await {
-        result?;
-    }
-
-    Ok(())
-}
-
-fn default_settings() -> Settings {
-    Settings::new()
-        .with_distinct_attribute(Some("project_id"))
-        .with_displayed_attributes(DEFAULT_DISPLAYED_ATTRIBUTES)
-        .with_searchable_attributes(DEFAULT_SEARCHABLE_ATTRIBUTES)
-        .with_sortable_attributes(DEFAULT_SORTABLE_ATTRIBUTES)
-        .with_filterable_attributes(&*MEILI_FILTERABLE_ATTRIBUTES)
-        .with_pagination(PaginationSetting {
-            max_total_hits: 2147483647,
-        })
-}
-
-pub struct MeilisearchFieldSpec {
-    pub path: &'static str,
-    pub filterable: bool,
-}
-
-impl SearchField {
-    pub const fn meilisearch_spec(self) -> MeilisearchFieldSpec {
-        match self {
-            SearchField::Categories => MeilisearchFieldSpec {
-                path: "categories",
-                filterable: true,
-            },
-            SearchField::Name => MeilisearchFieldSpec {
-                path: "name",
-                filterable: true,
-            },
-            SearchField::Author => MeilisearchFieldSpec {
-                path: "author",
-                filterable: true,
-            },
-            SearchField::License => MeilisearchFieldSpec {
-                path: "license",
-                filterable: true,
-            },
-            SearchField::ProjectTypes => MeilisearchFieldSpec {
-                path: "project_types",
-                filterable: true,
-            },
-            SearchField::ProjectId => MeilisearchFieldSpec {
-                path: "project_id",
-                filterable: true,
-            },
-            SearchField::OpenSource => MeilisearchFieldSpec {
-                path: "open_source",
-                filterable: true,
-            },
-            SearchField::Environment => MeilisearchFieldSpec {
-                path: "environment",
-                filterable: true,
-            },
-            SearchField::GameVersions => MeilisearchFieldSpec {
-                path: "game_versions",
-                filterable: true,
-            },
-            SearchField::ClientSide => MeilisearchFieldSpec {
-                path: "client_side",
-                filterable: true,
-            },
-            SearchField::ServerSide => MeilisearchFieldSpec {
-                path: "server_side",
-                filterable: true,
-            },
-            SearchField::MinecraftServerRegion => MeilisearchFieldSpec {
-                path: "minecraft_server.region",
-                filterable: true,
-            },
-            SearchField::MinecraftServerLanguages => MeilisearchFieldSpec {
-                path: "minecraft_server.languages",
-                filterable: true,
-            },
-            SearchField::MinecraftJavaServerContentKind => {
-                MeilisearchFieldSpec {
-                    path: "minecraft_java_server.content.kind",
-                    filterable: true,
-                }
-            }
-            SearchField::MinecraftJavaServerContentSupportedGameVersions => {
-                MeilisearchFieldSpec {
-                    path: "minecraft_java_server.content.supported_game_versions",
-                    filterable: true,
-                }
-            }
-            SearchField::MinecraftJavaServerPingData => MeilisearchFieldSpec {
-                path: "minecraft_java_server.ping.data",
-                filterable: true,
-            },
-        }
-    }
-}
-
-static MEILI_FILTERABLE_ATTRIBUTES: LazyLock<Vec<&'static str>> =
-    LazyLock::new(|| {
-        use strum::IntoEnumIterator;
-
-        SearchField::iter()
-            .filter_map(|field| {
-                let spec = field.meilisearch_spec();
-                spec.filterable.then_some(spec.path)
-            })
-            .collect()
-    });
-
-const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
-    "project_id",
-    "version_id",
-    "project_types",
-    "slug",
-    "author",
-    "name",
-    "summary",
-    "categories",
-    "display_categories",
-    "downloads",
-    "follows",
-    "icon_url",
-    "date_created",
-    "date_modified",
-    "latest_version",
-    "license",
-    "gallery",
-    "featured_gallery",
-    "color",
-    // Note: loader fields are not here, but are added on as they are needed (so they can be dynamically added depending on which exist).
-    // TODO: remove these- as they should be automatically populated. This is a band-aid fix.
-    "environment",
-    "game_versions",
-    "mrpack_loaders",
-    // V2 legacy fields for logical consistency
-    "client_side",
-    "server_side",
-    // Non-searchable fields for filling out the Project model.
-    "license_url",
-    "monetization_status",
-    "team_id",
-    "thread_id",
-    "versions",
-    "date_published",
-    "date_queued",
-    "status",
-    "requested_status",
-    "games",
-    "organization_id",
-    "links",
-    "gallery_items",
-    "loaders", // search uses loaders as categories- this is purely for the Project model.
-    "project_loader_fields",
-    "minecraft_mod",
-    "minecraft_server",
-    "minecraft_java_server",
-    "minecraft_bedrock_server",
-];
-
-const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] =
-    &["name", "summary", "author", "slug"];
-
-const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] = &[
-    "downloads",
-    "follows",
-    "date_created",
-    "date_modified",
-    "version_published_timestamp",
-    "minecraft_java_server.verified_plays_2w",
-    "minecraft_java_server.ping.data.players_online",
-];

apps/labrinth/src/search/backend/meilisearch/mod.rs

@@ -1,489 +0,0 @@
-use crate::database::PgPool;
-use crate::database::redis::RedisPool;
-use crate::env::ENV;
-use crate::models::ids::VersionId;
-use crate::routes::ApiError;
-use crate::search::backend::{
-    SearchIndex, SearchIndexName, combined_search_filters, parse_search_index,
-    parse_search_request,
-};
-use crate::search::{
-    ResultSearchProject, SearchBackend, SearchRequest, SearchResults,
-    TasksCancelFilter,
-};
-use crate::util::error::Context;
-use async_trait::async_trait;
-use eyre::Result;
-use futures::TryStreamExt;
-use futures::stream::FuturesOrdered;
-use itertools::Itertools;
-use meilisearch_sdk::client::Client;
-use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
-use serde::Serialize;
-use serde_json::Value;
-use std::collections::HashMap;
-use std::fmt::Write;
-use std::time::Duration;
-use tracing::{Instrument, info_span};
-
-pub mod indexing;
-
-#[derive(Debug, Clone)]
-pub struct MeilisearchReadClient {
-    pub client: Client,
-}
-
-impl std::ops::Deref for MeilisearchReadClient {
-    type Target = Client;
-
-    fn deref(&self) -> &Self::Target {
-        &self.client
-    }
-}
-
-pub struct BatchClient {
-    pub clients: Vec<Client>,
-}
-
-impl BatchClient {
-    pub fn new(clients: Vec<Client>) -> Self {
-        Self { clients }
-    }
-
-    pub async fn with_all_clients<'a, T, G, Fut>(
-        &'a self,
-        task_name: &str,
-        generator: G,
-    ) -> Result<Vec<T>>
-    where
-        G: Fn(&'a Client) -> Fut,
-        Fut: Future<Output = Result<T>> + 'a,
-    {
-        let mut tasks = FuturesOrdered::new();
-        for (idx, client) in self.clients.iter().enumerate() {
-            tasks.push_back(generator(client).instrument(info_span!(
-                "client_task",
-                task.name = task_name,
-                client.idx = idx,
-            )));
-        }
-
-        let results = tasks.try_collect::<Vec<T>>().await?;
-        Ok(results)
-    }
-
-    pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
-    where
-        F: FnMut(T, &Client) -> R,
-    {
-        assert_eq!(
-            data.len(),
-            self.clients.len(),
-            "mismatch between data len and meilisearch client count"
-        );
-        self.clients
-            .iter()
-            .zip(data)
-            .map(|(client, item)| predicate(item, client))
-            .collect()
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct MeilisearchConfig {
-    pub addresses: Vec<String>,
-    pub read_lb_address: String,
-    pub key: String,
-    pub meta_namespace: String,
-}
-
-impl MeilisearchConfig {
-    pub fn new(meta_namespace: Option<String>) -> Self {
-        Self {
-            addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
-            key: ENV.MEILISEARCH_KEY.clone(),
-            meta_namespace: meta_namespace.unwrap_or_default(),
-            read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
-        }
-    }
-
-    pub fn make_loadbalanced_read_client(
-        &self,
-    ) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
-        Ok(MeilisearchReadClient {
-            client: Client::new(&self.read_lb_address, Some(&self.key))?,
-        })
-    }
-
-    pub fn make_batch_client(
-        &self,
-    ) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
-        Ok(BatchClient::new(
-            self.addresses
-                .iter()
-                .map(|address| {
-                    Client::new(address.as_str(), Some(self.key.as_str()))
-                })
-                .collect::<Result<Vec<_>, _>>()?,
-        ))
-    }
-
-    pub fn get_index_name(&self, index: &str, next: bool) -> String {
-        let alt = if next { "_alt" } else { "" };
-        format!("{}_{}_{}", self.meta_namespace, index, alt)
-    }
-}
-
-pub struct Meilisearch {
-    pub config: MeilisearchConfig,
-}
-
-impl Meilisearch {
-    pub fn new(config: MeilisearchConfig) -> Self {
-        Self { config }
-    }
-
-    fn get_sort_index(
-        &self,
-        index: &str,
-        new_filters: Option<&str>,
-    ) -> Result<(String, &'static [&'static str]), ApiError> {
-        let sort = parse_search_index(index, new_filters)?;
-        let index_name = match sort.index_name {
-            SearchIndexName::Projects => {
-                self.config.get_index_name("projects", false)
-            }
-            SearchIndexName::ProjectsFiltered => {
-                self.config.get_index_name("projects_filtered", false)
-            }
-        };
-
-        Ok(match sort.index {
-            SearchIndex::Relevance => (
-                index_name,
-                &["downloads:desc", "version_published_timestamp:desc"],
-            ),
-            SearchIndex::Downloads => (
-                index_name,
-                &["downloads:desc", "version_published_timestamp:desc"],
-            ),
-            SearchIndex::Follows => (
-                index_name,
-                &["follows:desc", "version_published_timestamp:desc"],
-            ),
-            SearchIndex::Updated => (
-                index_name,
-                &["date_modified:desc", "version_published_timestamp:desc"],
-            ),
-            SearchIndex::Newest => (
-                index_name,
-                &["date_created:desc", "version_published_timestamp:desc"],
-            ),
-            SearchIndex::MinecraftJavaServerVerifiedPlays2w => (
-                index_name,
-                &[
-                    "minecraft_java_server.verified_plays_2w:desc",
-                    "minecraft_java_server.ping.data.players_online:desc",
-                    "version_published_timestamp:desc",
-                ],
-            ),
-            SearchIndex::MinecraftJavaServerPlayersOnline => (
-                index_name,
-                &[
-                    "minecraft_java_server.ping.data.players_online:desc",
-                    "version_published_timestamp:desc",
-                ],
-            ),
-        })
-    }
-}
-
-#[async_trait]
-impl SearchBackend for Meilisearch {
-    async fn search_for_project_raw(
-        &self,
-        info: &SearchRequest,
-    ) -> Result<SearchResults, ApiError> {
-        let parsed = parse_search_request(info)?;
-
-        let (index_name, sort_name) =
-            self.get_sort_index(parsed.index, info.new_filters.as_deref())?;
-        let client = self
-            .config
-            .make_loadbalanced_read_client()
-            .wrap_internal_err("failed to make load-balanced read client")?;
-        let meilisearch_index = client
-            .get_index(index_name)
-            .await
-            .wrap_internal_err("failed to get index")?;
-
-        let mut filter_string = String::new();
-
-        let results = {
-            let mut query = meilisearch_index.search();
-            query
-                .with_page(parsed.page)
-                .with_hits_per_page(parsed.hits_per_page)
-                .with_query(parsed.query)
-                .with_sort(sort_name);
-
-            if let Some(new_filters) = info.new_filters.as_deref() {
-                query.with_filter(new_filters);
-            } else {
-                let facets = if let Some(facets) = &info.facets {
-                    let facets =
-                        serde_json::from_str::<Vec<Vec<Value>>>(facets)
-                            .wrap_request_err("failed to parse facets")?;
-                    Some(facets)
-                } else {
-                    None
-                };
-
-                let filters =
-                    combined_search_filters(info).unwrap_or_else(|| "".into());
-
-                if let Some(facets) = facets {
-                    let facets: Vec<Vec<Vec<String>>> =
-                        facets
-                            .into_iter()
-                            .map(|facets| {
-                                facets
-								.into_iter()
-								.map(|facet| {
-									if facet.is_array() {
-										serde_json::from_value::<Vec<String>>(facet)
-											.unwrap_or_default()
-									} else {
-										vec![
-											serde_json::from_value::<String>(facet)
-												.unwrap_or_default(),
-										]
-									}
-								})
-								.collect_vec()
-                            })
-                            .collect_vec();
-
-                    filter_string.push('(');
-                    for (index, facet_outer_list) in facets.iter().enumerate() {
-                        filter_string.push('(');
-
-                        for (facet_outer_index, facet_inner_list) in
-                            facet_outer_list.iter().enumerate()
-                        {
-                            filter_string.push('(');
-                            for (facet_inner_index, facet) in
-                                facet_inner_list.iter().enumerate()
-                            {
-                                filter_string
-                                    .push_str(&facet.replace(':', " = "));
-                                if facet_inner_index
-                                    != (facet_inner_list.len() - 1)
-                                {
-                                    filter_string.push_str(" AND ")
-                                }
-                            }
-                            filter_string.push(')');
-
-                            if facet_outer_index != (facet_outer_list.len() - 1)
-                            {
-                                filter_string.push_str(" OR ")
-                            }
-                        }
-
-                        filter_string.push(')');
-
-                        if index != (facets.len() - 1) {
-                            filter_string.push_str(" AND ")
-                        }
-                    }
-                    filter_string.push(')');
-
-                    if !filters.is_empty() {
-                        write!(filter_string, " AND ({filters})")
-                            .expect("write should not fail");
-                    }
-                } else {
-                    filter_string.push_str(&filters);
-                }
-
-                if !filter_string.is_empty() {
-                    query.with_filter(&filter_string);
-                }
-            }
-
-            if info.show_metadata {
-                query.with_show_ranking_score(true);
-                query.with_show_ranking_score_details(true);
-                query.execute().await?
-            } else {
-                query.execute::<ResultSearchProject>().await?
-            }
-        };
-
-        if info.show_metadata {
-            let hits = results
-                .hits
-                .into_iter()
-                .map(|hit| {
-                    let metadata = serde_json::to_value(&hit)
-                        .ok()
-                        .and_then(|value| value.as_object().cloned())
-                        .map(|mut value| {
-                            value.remove("_formatted");
-                            value.remove("_matchesPosition");
-                            value.remove("_federation");
-                            let result = value.remove("result");
-                            let metadata = Value::Object(value);
-                            (result, metadata)
-                        });
-
-                    let (result, metadata) =
-                        metadata.unwrap_or((None, Value::Null));
-                    let mut result = result
-                        .and_then(|value| {
-                            serde_json::from_value::<ResultSearchProject>(value)
-                                .ok()
-                        })
-                        .unwrap_or(hit.result);
-
-                    if !metadata.is_null() {
-                        result.search_metadata = Some(metadata);
-                    }
-
-                    result
-                })
-                .collect();
-
-            Ok(SearchResults {
-                hits,
-                page: results.page.unwrap_or_default(),
-                hits_per_page: results.hits_per_page.unwrap_or_default(),
-                total_hits: results.total_hits.unwrap_or_default(),
-            })
-        } else {
-            Ok(SearchResults {
-                hits: results.hits.into_iter().map(|r| r.result).collect(),
-                page: results.page.unwrap_or_default(),
-                hits_per_page: results.hits_per_page.unwrap_or_default(),
-                total_hits: results.total_hits.unwrap_or_default(),
-            })
-        }
-    }
-
-    async fn index_projects(
-        &self,
-        ro_pool: PgPool,
-        redis: RedisPool,
-    ) -> eyre::Result<()> {
-        indexing::index_projects(ro_pool, redis, &self.config).await?;
-        Ok(())
-    }
-
-    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()> {
-        indexing::remove_documents(ids, &self.config).await?;
-        Ok(())
-    }
-
-    async fn tasks(&self) -> eyre::Result<Value> {
-        let client = self
-            .config
-            .make_batch_client()
-            .wrap_internal_err("failed to make batch client")?;
-        let tasks = client
-            .with_all_clients("get_tasks", async |client| {
-                let tasks = client.get_tasks().await?;
-                Ok(tasks.results)
-            })
-            .await
-            .wrap_internal_err("failed to get tasks")?;
-
-        #[derive(Serialize)]
-        struct MeiliTask<Time> {
-            uid: u32,
-            status: &'static str,
-            duration: Option<Duration>,
-            enqueued_at: Option<Time>,
-        }
-
-        #[derive(Serialize)]
-        struct TaskList<Time> {
-            by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
-        }
-
-        let response = tasks
-            .into_iter()
-            .enumerate()
-            .map(|(idx, instance_tasks)| {
-                let tasks = instance_tasks
-                    .into_iter()
-                    .filter_map(|task| {
-                        Some(match task {
-                            Task::Enqueued { content } => MeiliTask {
-                                uid: content.uid,
-                                status: "enqueued",
-                                duration: None,
-                                enqueued_at: Some(content.enqueued_at),
-                            },
-                            Task::Processing { content } => MeiliTask {
-                                uid: content.uid,
-                                status: "processing",
-                                duration: None,
-                                enqueued_at: Some(content.enqueued_at),
-                            },
-                            Task::Failed { content } => MeiliTask {
-                                uid: content.task.uid,
-                                status: "failed",
-                                duration: Some(content.task.duration),
-                                enqueued_at: Some(content.task.enqueued_at),
-                            },
-                            Task::Succeeded { .. } => return None,
-                        })
-                    })
-                    .collect();
-
-                (idx.to_string(), tasks)
-            })
-            .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
-
-        let response = serde_json::to_value(TaskList {
-            by_instance: response,
-        })
-        .wrap_internal_err("failed to serialize tasks response")?;
-        Ok(response)
-    }
-
-    async fn tasks_cancel(
-        &self,
-        filter: &TasksCancelFilter,
-    ) -> eyre::Result<()> {
-        let client = self
-            .config
-            .make_batch_client()
-            .wrap_internal_err("failed to make batch client")?;
-        let all_results = client
-            .with_all_clients("cancel_tasks", async |client| {
-                let mut q = TasksCancelQuery::new(client);
-                match filter {
-                    TasksCancelFilter::All => {}
-                    TasksCancelFilter::Indexes { indexes } => {
-                        q.with_index_uids(indexes.iter().map(|s| s.as_str()));
-                    }
-                    TasksCancelFilter::AllEnqueued => {
-                        q.with_statuses(["enqueued"]);
-                    }
-                };
-
-                let result = client.cancel_tasks_with(&q).await;
-                Ok(result)
-            })
-            .await
-            .wrap_internal_err("failed to cancel tasks")?;
-
-        for r in all_results {
-            r.wrap_internal_err("failed to cancel tasks")?;
-        }
-
-        Ok(())
-    }
-}

apps/labrinth/src/search/backend/mod.rs

@@ -1,12 +1,8 @@
 mod common;
-pub mod elasticsearch;
-pub mod meilisearch;
 pub mod typesense;
 
 pub use common::{
     ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
     combined_search_filters, parse_search_index, parse_search_request,
 };
-pub use elasticsearch::Elasticsearch;
-pub use meilisearch::{Meilisearch, MeilisearchConfig};
 pub use typesense::{Typesense, TypesenseConfig};

apps/labrinth/src/search/mod.rs

@@ -52,8 +52,6 @@ pub struct SearchRequest {
     #[serde(default)]
     pub show_metadata: bool,
     #[serde(default)]
-    pub elasticsearch_config: backend::elasticsearch::RequestConfig,
-    #[serde(default)]
     pub typesense_config: backend::typesense::RequestConfig,
 
     pub new_filters: Option<String>,
@@ -71,8 +69,6 @@ impl From<SearchQuery> for SearchRequest {
             index: query.index,
             limit: query.limit,
             show_metadata: false,
-            elasticsearch_config:
-                backend::elasticsearch::RequestConfig::default(),
             typesense_config: backend::typesense::RequestConfig::default(),
             new_filters: query.new_filters,
             facets: query.facets,
@@ -178,8 +174,6 @@ pub enum TasksCancelFilter {
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum SearchBackendKind {
-    Meilisearch,
-    Elasticsearch,
     Typesense,
 }
 
@@ -212,8 +206,6 @@ impl FromStr for SearchBackendKind {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
-            "meilisearch" => SearchBackendKind::Meilisearch,
-            "elasticsearch" => SearchBackendKind::Elasticsearch,
             "typesense" => SearchBackendKind::Typesense,
             _ => return Err(InvalidSearchBackendKind),
         })
@@ -351,13 +343,6 @@ impl From<UploadSearchProject> for ResultSearchProject {
 
 pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
     match ENV.SEARCH_BACKEND {
-        SearchBackendKind::Meilisearch => {
-            let config = backend::MeilisearchConfig::new(meta_namespace);
-            Box::new(backend::Meilisearch::new(config))
-        }
-        SearchBackendKind::Elasticsearch => {
-            Box::new(backend::Elasticsearch::new(meta_namespace).unwrap())
-        }
         SearchBackendKind::Typesense => {
             let config = backend::TypesenseConfig::new(meta_namespace);
             Box::new(backend::Typesense::new(config))

apps/labrinth/tests/moderation_notes.rs

@@ -0,0 +1,283 @@
+use actix_http::StatusCode;
+use actix_web::test;
+use common::{
+    api_common::{Api, AppendsOptionalPat},
+    database::{MOD_USER_PAT, USER_USER_ID, USER_USER_PAT},
+    environment::with_test_environment_all,
+};
+use serde_json::{Value, json};
+
+pub mod common;
+
+#[actix_rt::test]
+pub async fn moderation_notes_users() {
+    with_test_environment_all(None, |test_env| async move {
+        let api = test_env.api;
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/user/{USER_USER_ID}"))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert!(body.get("moderation_notes").unwrap().is_null());
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/user/{USER_USER_ID}"))
+                    .append_pat(USER_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert!(body.get("moderation_notes").is_none());
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .set_json(
+                        json!({ "notes": "first note", "user_rating": 1 }),
+                    )
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::NO_CONTENT); // OK without If-Match for the first patch
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .append_header(("If-Match", "0"))
+                    .set_json(json!({}))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::BAD_REQUEST);
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(USER_USER_PAT)
+                    .append_header(("If-Match", "0"))
+                    .set_json(json!({ "notes": "first note" }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::UNAUTHORIZED);
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .set_json(json!({
+                        "notes": "first note",
+                        "user_rating": 2,
+                    }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::PRECONDITION_REQUIRED); // Needs If-Match moving forward
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/user/{USER_USER_ID}"))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert_eq!(body["moderation_notes"]["notes"], "first note");
+        assert_eq!(body["moderation_notes"]["user_rating"], 1);
+        assert_eq!(body["moderation_notes"]["version"], 1);
+        assert_eq!(body["moderation_notes"]["last_author"], "2");
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .append_header(("If-Match", "0"))
+                    .set_json(json!({ "notes": "stale note" }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::PRECONDITION_FAILED);
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/user/{USER_USER_ID}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .append_header(("If-Match", "1"))
+                    .set_json(json!({ "user_rating": 4 }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::NO_CONTENT);
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/user/{USER_USER_ID}"))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert_eq!(body["moderation_notes"]["notes"], "first note");
+        assert_eq!(body["moderation_notes"]["user_rating"], 4);
+        assert_eq!(body["moderation_notes"]["version"], 2);
+
+        let user_ids = serde_json::to_string(&vec![USER_USER_ID]).unwrap();
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!(
+                        "/v3/users?ids={}",
+                        urlencoding::encode(&user_ids)
+                    ))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert_eq!(body[0]["moderation_notes"]["version"], 2);
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!(
+                        "/v3/users?ids={}",
+                        urlencoding::encode(&user_ids)
+                    ))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert!(body[0].get("moderation_notes").is_none());
+    })
+    .await;
+}
+
+#[actix_rt::test]
+pub async fn moderation_notes_organizations() {
+    with_test_environment_all(None, |test_env| async move {
+        let api = test_env.api;
+        let organization_id =
+            test_env.dummy.organization_zeta.organization_id.clone();
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/organization/{organization_id}"))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert!(body.get("moderation_notes").unwrap().is_null());
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/organization/{organization_id}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .append_header(("If-Match", "0"))
+                    .set_json(json!({
+                        "notes": "org note",
+                        "user_rating": -1,
+                    }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::PRECONDITION_FAILED); // Shouldn't have If-Match for the first patch
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/organization/{organization_id}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .set_json(json!({
+                        "notes": "org note",
+                        "user_rating": -1,
+                    }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::NO_CONTENT);
+
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!("/v3/organization/{organization_id}"))
+                    .append_pat(USER_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert!(body.get("moderation_notes").is_none());
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/organization/{organization_id}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .append_header(("If-Match", "1"))
+                    .set_json(json!({ "notes": "updated org note" }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::NO_CONTENT);
+
+        let resp = api
+            .call(
+                test::TestRequest::patch()
+                    .uri(&format!("/v3/organization/{organization_id}/notes"))
+                    .append_pat(MOD_USER_PAT)
+                    .set_json(json!({
+                        "notes": "new note",
+                    }))
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::PRECONDITION_REQUIRED); // Needs If-Match moving forward
+
+        let ids =
+            serde_json::to_string(&vec![organization_id.as_str()]).unwrap();
+        let resp = api
+            .call(
+                test::TestRequest::get()
+                    .uri(&format!(
+                        "/v3/organizations?ids={}",
+                        urlencoding::encode(&ids)
+                    ))
+                    .append_pat(MOD_USER_PAT)
+                    .to_request(),
+            )
+            .await;
+        assert_status!(&resp, StatusCode::OK);
+        let body: Value = test::read_body_json(resp).await;
+        assert_eq!(body[0]["moderation_notes"]["notes"], "updated org note");
+        assert_eq!(body[0]["moderation_notes"]["user_rating"], -1);
+        assert_eq!(body[0]["moderation_notes"]["version"], 2);
+    })
+    .await;
+}