fix search indexing not showing fields

Cargo.lock

@@ -640,6 +640,19 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "async-minecraft-ping"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "668b459c14dd8d9ef21e296af3f2a3651ff7dc3536e092fb0b09e528daaa6d89"
+dependencies = [
+ "async-trait",
+ "serde",
+ "serde_json",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
 [[package]]
 name = "async-process"
 version = "2.5.0"
@@ -2167,6 +2180,16 @@ dependencies = [
  "darling_macro 0.21.3",
 ]
 
+[[package]]
+name = "darling"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
+dependencies = [
+ "darling_core 0.23.0",
+ "darling_macro 0.23.0",
+]
+
 [[package]]
 name = "darling_core"
 version = "0.20.11"
@@ -2195,6 +2218,19 @@ dependencies = [
  "syn 2.0.106",
 ]
 
+[[package]]
+name = "darling_core"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
+dependencies = [
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.106",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.11"
@@ -2217,6 +2253,17 @@ dependencies = [
  "syn 2.0.106",
 ]
 
+[[package]]
+name = "darling_macro"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
+dependencies = [
+ "darling_core 0.23.0",
+ "quote",
+ "syn 2.0.106",
+]
+
 [[package]]
 name = "dashmap"
 version = "6.1.0"
@@ -4730,6 +4777,7 @@ dependencies = [
  "arc-swap",
  "argon2",
  "ariadne",
+ "async-minecraft-ping",
  "async-stripe",
  "async-trait",
  "base64 0.22.1",
@@ -4811,6 +4859,16 @@ dependencies = [
  "zxcvbn",
 ]
 
+[[package]]
+name = "labrinth-derive"
+version = "0.0.0"
+dependencies = [
+ "darling 0.23.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.106",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "packages/app-lib",
   "packages/ariadne",
   "packages/daedalus",
+  "packages/labrinth-derive",
   "packages/modrinth-log",
   "packages/modrinth-maxmind",
   "packages/modrinth-util",
@@ -32,6 +33,7 @@ arc-swap = "1.7.1"
 argon2 = { version = "0.5.3", features = ["std"] }
 ariadne = { path = "packages/ariadne" }
 async-compression = { version = "0.4.32", default-features = false }
+async-minecraft-ping = { version = "0.8.0" }
 async-recursion = "1.1.1"
 async-stripe = { version = "0.41.0", default-features = false, features = [
   "runtime-tokio-hyper-rustls",
@@ -58,6 +60,7 @@ color-eyre = "0.6.5"
 color-thief = "0.2.2"
 const_format = "0.2.34"
 daedalus = { path = "packages/daedalus" }
+darling = { version = "0.23" }
 dashmap = "6.1.0"
 data-url = "0.3.2"
 deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "db5fb00b036ecc8fe5f18853c559b745ffe47bde", version = "0.22.1" }
@@ -121,9 +124,11 @@ paste = "1.0.15"
 path-util = { path = "packages/path-util" }
 phf = { version = "0.13.1", features = ["macros"] }
 png = "0.18.0"
+proc-macro2 = { version = "1.0" }
 prometheus = "0.14.0"
 quartz_nbt = "0.2.9"
 quick-xml = "0.38.3"
+quote = { version = "1.0" }
 rand = "=0.8.5"  # Locked on 0.8 until argon2 and p256 update to 0.9
 rand_chacha = "=0.3.1"  # Locked on 0.3 until we can update rand to 0.9
 redis = "0.32.7"
@@ -166,6 +171,7 @@ spdx = "0.12.0"
 sqlx = { version = "0.8.6", default-features = false }
 sqlx-tracing = { path = "packages/sqlx-tracing" }
 strum = "0.27.2"
+syn = { version = "2.0" }
 sysinfo = { version = "0.37.2", default-features = false }
 tar = "0.4.44"
 tauri = "2.8.5"

apps/labrinth/.env.docker-compose

@@ -157,3 +157,5 @@ MURALPAY_TRANSFER_API_KEY=none
 MURALPAY_SOURCE_ACCOUNT_ID=00000000-0000-0000-0000-000000000000
 
 DEFAULT_AFFILIATE_REVENUE_SPLIT=0.1
+SERVER_PING_TIMEOUT=10000
+SERVER_PING_RETRIES=3

apps/labrinth/.env.local

@@ -167,3 +167,6 @@ MURALPAY_TRANSFER_API_KEY=none
 MURALPAY_SOURCE_ACCOUNT_ID=00000000-0000-0000-0000-000000000000
 
 DEFAULT_AFFILIATE_REVENUE_SPLIT=0.1
+SERVER_PING_TIMEOUT=10000
+SERVER_PING_RETRIES=3
+SERVER_PING_MIN_INTERVAL_SEC=1800

apps/labrinth/.sqlx/query-07dff8c7711178f10bf85271287704deb72636f4cbd6927108aadfd6e0d63f1c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO versions (\n                id, mod_id, author_id, name, version_number,\n                changelog, date_published, downloads,\n                version_type, featured, status, ordering\n            )\n            VALUES (\n                $1, $2, $3, $4, $5,\n                $6, $7, $8,\n                $9, $10, $11, $12\n            )\n            ",
+  "query": "\n            INSERT INTO versions (\n                id, mod_id, author_id, name, version_number,\n                changelog, date_published, downloads,\n                version_type, featured, status, ordering,\n                components\n            )\n            VALUES (\n                $1, $2, $3, $4, $5,\n                $6, $7, $8,\n                $9, $10, $11, $12,\n                $13\n            )\n            ",
   "describe": {
     "columns": [],
     "parameters": {
@@ -16,10 +16,11 @@
         "Varchar",
         "Bool",
         "Varchar",
-        "Int4"
+        "Int4",
+        "Jsonb"
       ]
     },
     "nullable": []
   },
-  "hash": "a4745a3dc87c3a858819b208b0c3a010dc297425883113565d934b8a834014ce"
+  "hash": "07dff8c7711178f10bf85271287704deb72636f4cbd6927108aadfd6e0d63f1c"
 }

apps/labrinth/.sqlx/query-46f309eb085e487bf868d4fee5170eef9d1af5ff41b097b3552de86d3940e01e.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        UPDATE mods\n        SET components = $1\n        WHERE id = $2\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "46f309eb085e487bf868d4fee5170eef9d1af5ff41b097b3552de86d3940e01e"
+}

apps/labrinth/.sqlx/query-591bf29b883b66b111496f8c07ebd2619cd3bcf92f637ca99876c14bd44c85f6.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT id, components AS \"components: Json<exp::ProjectSerial>\"\n            FROM mods\n            WHERE components ? 'minecraft_java_server'\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "components: Json<exp::ProjectSerial>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "591bf29b883b66b111496f8c07ebd2619cd3bcf92f637ca99876c14bd44c85f6"
+}

apps/labrinth/.sqlx/query-59b6eea93ce248d2b1eaf14fe8970bba8ccd2df0995a21bdb34ae3214cef6377.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                    SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n                    m.icon_url icon_url, m.raw_icon_url raw_icon_url, m.description description, m.published published,\n                    m.approved approved, m.queued, m.status status, m.requested_status requested_status,\n                    m.license_url license_url,\n                    m.team_id team_id, m.organization_id organization_id, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,\n                    m.webhook_sent, m.color,\n                    t.id thread_id, m.monetization_status monetization_status,\n                    m.side_types_migration_review_status side_types_migration_review_status,\n                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories\n                    FROM mods m\n                    INNER JOIN threads t ON t.mod_id = m.id\n                    LEFT JOIN mods_categories mc ON mc.joining_mod_id = m.id\n                    LEFT JOIN categories c ON mc.joining_category_id = c.id\n                    WHERE m.id = ANY($1) OR m.slug = ANY($2)\n                    GROUP BY t.id, m.id;\n                    ",
+  "query": "\n                    SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n                    m.icon_url icon_url, m.raw_icon_url raw_icon_url, m.description description, m.published published,\n                    m.approved approved, m.queued, m.status status, m.requested_status requested_status,\n                    m.license_url license_url,\n                    m.team_id team_id, m.organization_id organization_id, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,\n                    m.webhook_sent, m.color,\n                    t.id thread_id, m.monetization_status monetization_status,\n                    m.side_types_migration_review_status side_types_migration_review_status,\n                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n                    m.components AS \"components: sqlx::types::Json<exp::ProjectSerial>\"\n\n                    FROM mods m\n                    INNER JOIN threads t ON t.mod_id = m.id\n                    LEFT JOIN mods_categories mc ON mc.joining_mod_id = m.id\n                    LEFT JOIN categories c ON mc.joining_category_id = c.id\n\n                    WHERE m.id = ANY($1) OR m.slug = ANY($2)\n                    GROUP BY t.id, m.id\n                    ",
   "describe": {
     "columns": [
       {
@@ -137,6 +137,11 @@
         "ordinal": 26,
         "name": "additional_categories",
         "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 27,
+        "name": "components: sqlx::types::Json<exp::ProjectSerial>",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -172,8 +177,9 @@
       false,
       false,
       null,
-      null
+      null,
+      false
     ]
   },
-  "hash": "7a6d6a91e6bd27f7be34b8cc7955a66c4175ebd1c55e437f187f61efca681c62"
+  "hash": "59b6eea93ce248d2b1eaf14fe8970bba8ccd2df0995a21bdb34ae3214cef6377"
 }

apps/labrinth/.sqlx/query-760df5118dcd1f9d28db991caabb3c1ce6fb8ac08829345445b027e5a46c4860.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                    SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,\n                    v.changelog changelog, v.date_published date_published, v.downloads downloads,\n                    v.version_type version_type, v.featured featured, v.status status, v.requested_status requested_status, v.ordering ordering\n                    FROM versions v\n                    WHERE v.id = ANY($1);\n                    ",
+  "query": "\n                    SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,\n                    v.changelog changelog, v.date_published date_published, v.downloads downloads,\n                    v.version_type version_type, v.featured featured, v.status status, v.requested_status requested_status, v.ordering ordering,\n                    v.components AS \"components: sqlx::types::Json<exp::VersionSerial>\"\n                    FROM versions v\n                    WHERE v.id = ANY($1);\n                    ",
   "describe": {
     "columns": [
       {
@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "ordering",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 13,
+        "name": "components: sqlx::types::Json<exp::VersionSerial>",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -87,8 +92,9 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
-  "hash": "32f4aa1ab67fbdcd7187fbae475876bf3d3225ca7b4994440a67cbd6a7b610f6"
+  "hash": "760df5118dcd1f9d28db991caabb3c1ce6fb8ac08829345445b027e5a46c4860"
 }

apps/labrinth/.sqlx/query-b562b547271c94a1ea3db93c7b75ff7d7d79642345c93002dd774d7b30ac3d66.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(\n            SELECT 1 FROM mods WHERE slug = $1 OR text_id_lower = $1\n        )",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "b562b547271c94a1ea3db93c7b75ff7d7d79642345c93002dd774d7b30ac3d66"
+}

apps/labrinth/.sqlx/query-b760951120c1885530f3fb8208a5874fc472ec160a673abeeee10c778b9cf23f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n        FROM mods m\n        WHERE m.status = ANY($1) AND m.id > $3\n        GROUP BY m.id\n        ORDER BY m.id ASC\n        LIMIT $2;\n        ",
+  "query": "\n        SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color,\n        m.components AS \"components: sqlx::types::Json<exp::ProjectSerial>\"\n        FROM mods m\n        WHERE m.status = ANY($1) AND m.id > $3\n        GROUP BY m.id\n        ORDER BY m.id ASC\n        LIMIT $2;\n        ",
   "describe": {
     "columns": [
       {
@@ -62,6 +62,11 @@
         "ordinal": 11,
         "name": "color",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 12,
+        "name": "components: sqlx::types::Json<exp::ProjectSerial>",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -83,8 +88,9 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
-  "hash": "702a2826d5857dc51b1a7a79c9043ae8987441bb5e89c9ea62d347e47899e3c2"
+  "hash": "b760951120c1885530f3fb8208a5874fc472ec160a673abeeee10c778b9cf23f"
 }

apps/labrinth/.sqlx/query-cf0ce4ce54edc7533332f0bfab27d977a9613f8aa22669c0f8fe7bab2d5d6192.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO mods (\n                id, team_id, name, summary, description,\n                published, downloads, icon_url, raw_icon_url, status, requested_status,\n                license_url, license,\n                slug, color, monetization_status, organization_id,\n                side_types_migration_review_status\n            )\n            VALUES (\n                $1, $2, $3, $4, $5, $6,\n                $7, $8, $9, $10, $11,\n                $12, $13,\n                LOWER($14), $15, $16, $17,\n                $18\n            )\n            ",
+  "query": "\n            INSERT INTO mods (\n                id, team_id, name, summary, description,\n                published, downloads, icon_url, raw_icon_url, status, requested_status,\n                license_url, license,\n                slug, color, monetization_status, organization_id,\n                side_types_migration_review_status,\n                components\n            )\n            VALUES (\n                $1, $2, $3, $4, $5, $6,\n                $7, $8, $9, $10, $11,\n                $12, $13,\n                LOWER($14), $15, $16, $17,\n                $18,\n                $19\n            )\n            ",
   "describe": {
     "columns": [],
     "parameters": {
@@ -22,10 +22,11 @@
         "Int4",
         "Varchar",
         "Int8",
-        "Varchar"
+        "Varchar",
+        "Jsonb"
       ]
     },
     "nullable": []
   },
-  "hash": "ee74bbff42dd29ab5a23d5811ea18e62ac199fe5e68275bf1bc7c71ace630702"
+  "hash": "cf0ce4ce54edc7533332f0bfab27d977a9613f8aa22669c0f8fe7bab2d5d6192"
 }

apps/labrinth/.sqlx/query-fc99699ff89c3245c7288718ff15d16058b00bf5df286923c0c6b1db2dbbde4f.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT\n                v.id AS \"version_id: DBVersionId\",\n                m.id AS \"project_id: DBProjectId\",\n                m.name AS \"project_name!\",\n                COALESCE(m.icon_url, '') AS \"project_icon!\"\n            FROM versions v\n            INNER JOIN mods m ON m.id = v.mod_id\n            WHERE v.id = ANY($1)\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version_id: DBVersionId",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "project_id: DBProjectId",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "project_name!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "project_icon!",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8Array"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "fc99699ff89c3245c7288718ff15d16058b00bf5df286923c0c6b1db2dbbde4f"
+}

apps/labrinth/Cargo.toml

@@ -21,6 +21,7 @@ actix-ws = { workspace = true }
 arc-swap = { workspace = true }
 argon2 = { workspace = true }
 ariadne = { workspace = true }
+async-minecraft-ping = { workspace = true }
 async-stripe = { workspace = true, features = [
   "billing",
   "checkout",

apps/labrinth/src/background_task.rs

@@ -11,7 +11,8 @@ use crate::search::indexing::index_projects;
 use crate::util::anrok;
 use crate::{database, search};
 use clap::ValueEnum;
-use tracing::{error, info, warn};
+use eyre::WrapErr;
+use tracing::info;
 
 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
 #[clap(rename_all = "kebab_case")]
@@ -25,6 +26,7 @@ pub enum BackgroundTask {
     IndexSubscriptions,
     Migrations,
     Mail,
+    PingMinecraftJavaServers,
 }
 
 impl BackgroundTask {
@@ -40,7 +42,7 @@ impl BackgroundTask {
         anrok_client: anrok::Client,
         email_queue: EmailQueue,
         mural_client: muralpay::Client,
-    ) {
+    ) -> eyre::Result<()> {
         use BackgroundTask::*;
         match self {
             Migrations => run_migrations().await,
@@ -62,7 +64,7 @@ impl BackgroundTask {
                 )
                 .await;
 
-                update_bank_balances(pool).await;
+                update_bank_balances(pool).await
             }
             IndexSubscriptions => {
                 index_subscriptions(
@@ -71,71 +73,66 @@ impl BackgroundTask {
                     stripe_client,
                     anrok_client,
                 )
-                .await
+                .await;
+                Ok(())
             }
-            Mail => {
-                run_email(email_queue).await;
+            Mail => run_email(email_queue).await,
+            PingMinecraftJavaServers => {
+                ping_minecraft_java_servers(pool, redis_pool, clickhouse).await
             }
         }
     }
 }
 
-pub async fn run_email(email_queue: EmailQueue) {
+pub async fn run_email(email_queue: EmailQueue) -> eyre::Result<()> {
     // Only index for 5 emails at a time, to reduce transaction length,
     // for a total of 100 emails.
     for _ in 0..20 {
         let then = std::time::Instant::now();
 
-        match email_queue.index(5).await {
-            Ok(true) => {
-                info!(
-                    "Indexed email queue in {}ms",
-                    then.elapsed().as_millis()
-                );
-            }
-            Ok(false) => {
-                info!("No more emails to index");
-                break;
-            }
-            Err(error) => {
-                error!(%error, "Failed to index email queue");
-            }
+        let indexed = email_queue
+            .index(5)
+            .await
+            .wrap_err("failed to index email queue")?;
+        if indexed {
+            info!("Indexed email queue in {}ms", then.elapsed().as_millis());
+        } else {
+            info!("No more emails to index");
+            break;
         }
     }
+
+    Ok(())
 }
 
-pub async fn update_bank_balances(pool: PgPool) {
+pub async fn update_bank_balances(pool: PgPool) -> eyre::Result<()> {
     let payouts_queue = PayoutsQueue::new();
 
-    match insert_bank_balances_and_webhook(&payouts_queue, &pool).await {
-        Ok(_) => info!("Bank balances updated successfully"),
-        Err(error) => error!(%error, "Bank balances update failed"),
-    }
+    insert_bank_balances_and_webhook(&payouts_queue, &pool)
+        .await
+        .wrap_err("failed to update bank balances")?;
+    info!("Bank balances updated successfully");
+    Ok(())
 }
 
-pub async fn run_migrations() {
-    database::check_for_migrations()
-        .await
-        .expect("An error occurred while running migrations.");
+pub async fn run_migrations() -> eyre::Result<()> {
+    database::check_for_migrations().await?;
+    Ok(())
 }
 
 pub async fn index_search(
     ro_pool: PgPool,
     redis_pool: RedisPool,
     search_config: search::SearchConfig,
-) {
+) -> eyre::Result<()> {
     info!("Indexing local database");
-    let result = index_projects(ro_pool, redis_pool, &search_config).await;
-    if let Err(e) = result {
-        warn!("Local project indexing failed: {:?}", e);
-    }
-    info!("Done indexing local database");
+    index_projects(ro_pool, redis_pool, &search_config).await
 }
 
-pub async fn release_scheduled(pool: PgPool) {
+pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {
     info!("Releasing scheduled versions/projects!");
 
-    let projects_results = sqlx::query!(
+    sqlx::query!(
         "
         UPDATE mods
         SET status = requested_status
@@ -143,14 +140,11 @@ pub async fn release_scheduled(pool: PgPool) {
         ",
         crate::models::projects::ProjectStatus::Scheduled.as_str(),
     )
-        .execute(&pool)
-        .await;
+    .execute(&pool)
+    .await
+    .wrap_err("failed syncing scheduled releases for projects")?;
 
-    if let Err(e) = projects_results {
-        warn!("Syncing scheduled releases for projects failed: {:?}", e);
-    }
-
-    let versions_results = sqlx::query!(
+    sqlx::query!(
         "
         UPDATE versions
         SET status = requested_status
@@ -158,78 +152,96 @@ pub async fn release_scheduled(pool: PgPool) {
         ",
         crate::models::projects::VersionStatus::Scheduled.as_str(),
     )
-        .execute(&pool)
-        .await;
-
-    if let Err(e) = versions_results {
-        warn!("Syncing scheduled releases for versions failed: {:?}", e);
-    }
+    .execute(&pool)
+    .await
+    .wrap_err("failed syncing scheduled releases for versions")?;
 
     info!("Finished releasing scheduled versions/projects");
+    Ok(())
 }
 
-pub async fn update_versions(pool: PgPool, redis_pool: RedisPool) {
+pub async fn update_versions(
+    pool: PgPool,
+    redis_pool: RedisPool,
+) -> eyre::Result<()> {
     info!("Indexing game versions list from Mojang");
-    let result = version_updater::update_versions(&pool, &redis_pool).await;
-    if let Err(e) = result {
-        warn!("Version update failed: {}", e);
-    }
+    version_updater::update_versions(&pool, &redis_pool)
+        .await
+        .wrap_err("failed to update game versions")?;
     info!("Done indexing game versions");
+    Ok(())
 }
 
 pub async fn payouts(
     pool: PgPool,
     clickhouse: clickhouse::Client,
     redis_pool: RedisPool,
-) {
+) -> eyre::Result<()> {
     info!("Started running payouts");
-    let result = process_payout(&pool, &clickhouse).await;
-    if let Err(e) = result {
-        warn!("Payouts run failed: {e:#?}");
-    }
+    process_payout(&pool, &clickhouse)
+        .await
+        .wrap_err("payout processing failed")?;
 
-    let result = index_payouts_notifications(&pool, &redis_pool).await;
-    if let Err(e) = result {
-        warn!("Payouts notifications indexing failed: {e:#?}");
-    }
+    index_payouts_notifications(&pool, &redis_pool)
+        .await
+        .wrap_err("payout notifications indexing failed")?;
 
-    let result = process_affiliate_payouts(&pool).await;
-    if let Err(e) = result {
-        warn!("Affiliate payouts run failed: {e:#?}");
-    }
+    process_affiliate_payouts(&pool)
+        .await
+        .wrap_err("affiliate payouts processing failed")?;
 
-    let result = remove_payouts_for_refunded_charges(&pool).await;
-    if let Err(e) = result {
-        warn!("Removing affiliate payouts for refunded charges failed: {e:#?}");
-    }
+    remove_payouts_for_refunded_charges(&pool)
+        .await
+        .wrap_err("removing payouts for refunded charges failed")?;
 
     info!("Done running payouts");
+    Ok(())
 }
 
-pub async fn sync_payout_statuses(pool: PgPool, mural: muralpay::Client) {
+pub async fn sync_payout_statuses(
+    pool: PgPool,
+    mural: muralpay::Client,
+) -> eyre::Result<()> {
     // Mural sets a max limit of 100 for search payouts endpoint
     const LIMIT: u32 = 100;
 
     info!("Started syncing payout statuses");
 
-    let result = crate::queue::payouts::mural::sync_pending_payouts_from_mural(
+    crate::queue::payouts::mural::sync_pending_payouts_from_mural(
         &pool, &mural, LIMIT,
     )
-    .await;
-    if let Err(e) = result {
-        warn!("Failed to sync pending payouts from Mural: {e:?}");
-    }
+    .await
+    .wrap_err("failed to sync pending payouts from Mural")?;
 
-    let result =
-        crate::queue::payouts::mural::sync_failed_mural_payouts_to_labrinth(
-            &pool, &mural, LIMIT,
-        )
-        .await;
-    if let Err(e) = result {
-        warn!("Failed to sync failed Mural payouts to Labrinth: {e:?}");
-    }
+    crate::queue::payouts::mural::sync_failed_mural_payouts_to_labrinth(
+        &pool, &mural, LIMIT,
+    )
+    .await
+    .wrap_err("failed to sync failed Mural payouts to Labrinth")?;
 
     info!("Done syncing payout statuses");
+    Ok(())
+}
+
+pub async fn ping_minecraft_java_servers(
+    pool: PgPool,
+    redis_pool: RedisPool,
+    clickhouse: clickhouse::Client,
+) -> eyre::Result<()> {
+    info!("Started pinging Minecraft Java servers");
+
+    let server_ping_queue = crate::queue::server_ping::ServerPingQueue::new(
+        pool, redis_pool, clickhouse,
+    );
+
+    server_ping_queue
+        .ping_minecraft_java_servers()
+        .await
+        .wrap_err("failed to ping Minecraft Java servers")?;
+    info!("Successfully pinged Minecraft Java servers");
+
+    info!("Done pinging Minecraft Java servers");
+    Ok(())
 }
 
 mod version_updater {

apps/labrinth/src/clickhouse/mod.rs

@@ -6,6 +6,7 @@ mod fetch;
 pub use fetch::*;
 
 use crate::env::ENV;
+use crate::queue::server_ping;
 
 pub async fn init_client() -> clickhouse::error::Result<clickhouse::Client> {
     init_client_with_database(&ENV.CLICKHOUSE_DATABASE).await
@@ -14,6 +15,9 @@ pub async fn init_client() -> clickhouse::error::Result<clickhouse::Client> {
 pub async fn init_client_with_database(
     database: &str,
 ) -> clickhouse::error::Result<clickhouse::Client> {
+    const MINECRAFT_JAVA_SERVER_PINGS: &str = server_ping::CLICKHOUSE_TABLE;
+    const MINECRAFT_JAVA_SERVER_PLAYS: &str = "minecraft_java_server_plays";
+
     let client = {
         let https_connector = HttpsConnectorBuilder::new()
             .with_native_roots()?
@@ -160,5 +164,49 @@ pub async fn init_client_with_database(
         .execute()
         .await?;
 
+    client
+        .query(&format!(
+            "
+            CREATE TABLE IF NOT EXISTS {database}.{MINECRAFT_JAVA_SERVER_PINGS} {cluster_line}
+            (
+                recorded DateTime64(4),
+                project_id UInt64,
+                address String,
+                port UInt16,
+                online Bool,
+                latency_ms Nullable(UInt32),
+                description Nullable(String),
+                version_name Nullable(String),
+                version_protocol Nullable(UInt32),
+                players_online Nullable(UInt32),
+                players_max Nullable(UInt32)
+            )
+            ENGINE = {engine}
+            {ttl}
+            PRIMARY KEY (project_id, recorded)
+            SETTINGS index_granularity = 8192
+            "
+        ))
+        .execute()
+        .await?;
+
+    client
+        .query(&format!(
+            "
+            CREATE TABLE IF NOT EXISTS {database}.{MINECRAFT_JAVA_SERVER_PLAYS} {cluster_line}
+            (
+                recorded DateTime64(4),
+                user_id UInt64,
+                project_id UInt64
+            )
+            ENGINE = {engine}
+            {ttl}
+            PRIMARY KEY (project_id, recorded)
+            SETTINGS index_granularity = 8192
+            "
+        ))
+        .execute()
+        .await?;
+
     Ok(client.with_database(database))
 }

apps/labrinth/src/database/models/mod.rs

@@ -58,6 +58,8 @@ pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser};
 
 #[derive(Error, Debug)]
 pub enum DatabaseError {
+    #[error(transparent)]
+    Internal(#[from] eyre::Report),
     #[error("Error while interacting with the database: {0}")]
     Database(#[from] sqlx::Error),
     #[error("Error while trying to generate random ID")]

apps/labrinth/src/database/models/project_item.rs

@@ -6,9 +6,13 @@ use super::{DBUser, ids::*};
 use crate::database::models::DatabaseError;
 use crate::database::redis::RedisPool;
 use crate::database::{PgTransaction, models};
+use crate::models::exp;
+use crate::models::ids::ProjectId;
 use crate::models::projects::{
     MonetizationStatus, ProjectStatus, SideTypesMigrationReviewStatus,
 };
+use crate::routes::ApiError;
+use crate::util::error::Context;
 use ariadne::ids::base62_impl::parse_base62;
 use chrono::{DateTime, Utc};
 use dashmap::{DashMap, DashSet};
@@ -176,6 +180,7 @@ pub struct ProjectBuilder {
     pub gallery_items: Vec<DBGalleryItem>,
     pub color: Option<u32>,
     pub monetization_status: MonetizationStatus,
+    pub components: exp::ProjectSerial,
 }
 
 impl ProjectBuilder {
@@ -215,6 +220,7 @@ impl ProjectBuilder {
             side_types_migration_review_status:
                 SideTypesMigrationReviewStatus::Reviewed,
             loaders: vec![],
+            components: self.components,
         };
         project_struct.insert(&mut *transaction).await?;
 
@@ -294,6 +300,7 @@ pub struct DBProject {
     pub monetization_status: MonetizationStatus,
     pub side_types_migration_review_status: SideTypesMigrationReviewStatus,
     pub loaders: Vec<String>,
+    pub components: exp::ProjectSerial,
 }
 
 impl DBProject {
@@ -308,14 +315,16 @@ impl DBProject {
                 published, downloads, icon_url, raw_icon_url, status, requested_status,
                 license_url, license,
                 slug, color, monetization_status, organization_id,
-                side_types_migration_review_status
+                side_types_migration_review_status,
+                components
             )
             VALUES (
                 $1, $2, $3, $4, $5, $6,
                 $7, $8, $9, $10, $11,
                 $12, $13,
                 LOWER($14), $15, $16, $17,
-                $18
+                $18,
+                $19
             )
             ",
             self.id as DBProjectId,
@@ -335,7 +344,8 @@ impl DBProject {
             self.color.map(|x| x as i32),
             self.monetization_status.as_str(),
             self.organization_id.map(|x| x.0 as i64),
-            self.side_types_migration_review_status.as_str()
+            self.side_types_migration_review_status.as_str(),
+            serde_json::to_value(&self.components).expect("serialization shouldn't fail"),
         )
         .execute(&mut *transaction)
         .await?;
@@ -347,12 +357,15 @@ impl DBProject {
         id: DBProjectId,
         transaction: &mut PgTransaction<'_>,
         redis: &RedisPool,
-    ) -> Result<Option<()>, DatabaseError> {
-        let project = Self::get_id(id, &mut *transaction, redis).await?;
+    ) -> Result<Option<()>, ApiError> {
+        let project = Self::get_id(id, &mut *transaction, redis)
+            .await
+            .wrap_internal_err("failed to fetch project by ID")?;
 
         if let Some(project) = project {
             DBProject::clear_cache(id, project.inner.slug, Some(true), redis)
-                .await?;
+                .await
+                .wrap_internal_err("failed to clear project cache")?;
 
             sqlx::query!(
                 "
@@ -362,7 +375,8 @@ impl DBProject {
                 id as DBProjectId
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete project followers")?;
 
             sqlx::query!(
                 "
@@ -372,7 +386,8 @@ impl DBProject {
                 id as DBProjectId
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete project gallery items")?;
 
             sqlx::query!(
                 "
@@ -382,7 +397,10 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err(
+                "failed to delete duplicate project followers",
+            )?;
 
             sqlx::query!(
                 "
@@ -393,7 +411,8 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to clear report project references")?;
 
             sqlx::query!(
                 "
@@ -403,7 +422,8 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete project categories")?;
 
             sqlx::query!(
                 "
@@ -413,11 +433,13 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete project links")?;
 
             for version in project.versions {
                 super::DBVersion::remove_full(version, redis, transaction)
-                    .await?;
+                    .await
+                    .wrap_internal_err("failed to remove project version")?;
             }
 
             sqlx::query!(
@@ -427,7 +449,8 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete dependency references")?;
 
             sqlx::query!(
                 "
@@ -438,7 +461,8 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to clear payout project references")?;
 
             sqlx::query!(
                 "
@@ -448,10 +472,12 @@ impl DBProject {
                 id as DBProjectId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete project row")?;
 
             models::DBTeamMember::clear_cache(project.inner.team_id, redis)
-                .await?;
+                .await
+                .wrap_internal_err("failed to clear team member cache")?;
 
             let affected_user_ids = sqlx::query!(
                 "
@@ -464,9 +490,12 @@ impl DBProject {
             .fetch(&mut *transaction)
             .map_ok(|x| DBUserId(x.user_id))
             .try_collect::<Vec<_>>()
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete team members")?;
 
-            DBUser::clear_project_cache(&affected_user_ids, redis).await?;
+            DBUser::clear_project_cache(&affected_user_ids, redis)
+                .await
+                .wrap_internal_err("failed to clear user project cache")?;
 
             sqlx::query!(
                 "
@@ -476,7 +505,8 @@ impl DBProject {
                 project.inner.team_id as DBTeamId,
             )
             .execute(&mut *transaction)
-            .await?;
+            .await
+            .wrap_internal_err("failed to delete team")?;
 
             Ok(Some(()))
         } else {
@@ -488,7 +518,7 @@ impl DBProject {
         string: &str,
         executor: E,
         redis: &RedisPool,
-    ) -> Result<Option<ProjectQueryResult>, DatabaseError>
+    ) -> Result<Option<ProjectQueryResult>, ApiError>
     where
         E: crate::database::Acquire<'a, Database = sqlx::Postgres>,
     {
@@ -501,7 +531,7 @@ impl DBProject {
         id: DBProjectId,
         executor: E,
         redis: &RedisPool,
-    ) -> Result<Option<ProjectQueryResult>, DatabaseError>
+    ) -> Result<Option<ProjectQueryResult>, ApiError>
     where
         E: crate::database::Acquire<'a, Database = sqlx::Postgres>,
     {
@@ -518,7 +548,7 @@ impl DBProject {
         project_ids: &[DBProjectId],
         exec: E,
         redis: &RedisPool,
-    ) -> Result<Vec<ProjectQueryResult>, DatabaseError>
+    ) -> Result<Vec<ProjectQueryResult>, ApiError>
     where
         E: crate::database::Acquire<'a, Database = sqlx::Postgres>,
     {
@@ -537,7 +567,7 @@ impl DBProject {
         project_strings: &[T],
         exec: E,
         redis: &RedisPool,
-    ) -> Result<Vec<ProjectQueryResult>, DatabaseError>
+    ) -> Result<Vec<ProjectQueryResult>, ApiError>
     where
         E: crate::database::Acquire<'a, Database = sqlx::Postgres>,
     {
@@ -547,7 +577,9 @@ impl DBProject {
             false,
             project_strings,
             |ids| async move {
-                let mut exec = exec.acquire().await?;
+                let mut exec = exec
+                    .acquire()
+                    .await?;
                 let project_ids_parsed: Vec<i64> = ids
                     .iter()
                     .filter_map(|x| parse_base62(&x.to_string()).ok())
@@ -668,7 +700,8 @@ impl DBProject {
                             });
                         async move { Ok(acc) }
                     }
-                    ).await?;
+                    )
+                    .await?;
 
                 let links: DashMap<DBProjectId, Vec<LinkUrl>> = sqlx::query!(
                     "
@@ -692,7 +725,8 @@ impl DBProject {
                             });
                         async move { Ok(acc) }
                     }
-                    ).await?;
+                    )
+                    .await?;
 
                 #[derive(Default)]
                 struct VersionLoaderData {
@@ -743,7 +777,9 @@ impl DBProject {
                         (project_id, version_loader_data)
 
                     }
-                    ).try_collect().await?;
+                    )
+                    .try_collect()
+                    .await?;
 
                 let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
                     "
@@ -766,8 +802,8 @@ impl DBProject {
                     .try_collect()
                     .await?;
 
-                let projects = sqlx::query!(
-                    "
+                let project_rows = sqlx::query!(
+                    r#"
                     SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,
                     m.icon_url icon_url, m.raw_icon_url raw_icon_url, m.description description, m.published published,
                     m.approved approved, m.queued, m.status status, m.requested_status requested_status,
@@ -777,24 +813,48 @@ impl DBProject {
                     t.id thread_id, m.monetization_status monetization_status,
                     m.side_types_migration_review_status side_types_migration_review_status,
                     ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,
-                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories
+                    ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,
+                    m.components AS "components: sqlx::types::Json<exp::ProjectSerial>"
+
                     FROM mods m
                     INNER JOIN threads t ON t.mod_id = m.id
                     LEFT JOIN mods_categories mc ON mc.joining_mod_id = m.id
                     LEFT JOIN categories c ON mc.joining_category_id = c.id
+
                     WHERE m.id = ANY($1) OR m.slug = ANY($2)
-                    GROUP BY t.id, m.id;
-                    ",
+                    GROUP BY t.id, m.id
+                    "#,
                     &project_ids_parsed,
                     &slugs,
                 )
-                    .fetch(&mut exec)
-                    .try_fold(DashMap::new(), |acc, m| {
+                .fetch_all(&mut exec)
+                .await?;
+
+                let project_components = project_rows
+                    .iter()
+                    .map(|row| {
+                        (
+                            ProjectId::from(DBProjectId(row.id)),
+                            &row.components.0,
+                        )
+                    })
+                    .collect::<Vec<_>>();
+                let project_query_context = exp::project::fetch_query_context(
+                    &project_components,
+                    &mut exec,
+                    redis,
+                )
+                .await
+                .wrap_err("failed to fetch project query context")?;
+
+                let projects = project_rows
+                    .into_iter()
+                    .try_fold(DashMap::new(), |acc, m| -> Result<_, DatabaseError> {
                         let id = m.id;
                         let project_id = DBProjectId(id);
                         let VersionLoaderData {
                             loaders,
-                            project_types,
+                            mut project_types,
                             games,
                             loader_loader_field_ids,
                         } = loaders_ptypes_games.remove(&project_id).map(|x|x.1).unwrap_or_default();
@@ -809,6 +869,17 @@ impl DBProject {
                             .filter(|x| loader_loader_field_ids.contains(&x.id))
                             .collect::<Vec<_>>();
 
+                        let components_serial = m.components.0;
+                        exp::compat::correct_project_types(
+                            &components_serial,
+                            &mut project_types,
+                        );
+                        let components = components_serial.clone().into_query(
+                            ProjectId::from(project_id),
+                            &project_query_context,
+                        )
+                        .wrap_err("failed to populate query components")?;
+
                         let project = ProjectQueryResult {
                             inner: DBProject {
                                 id: DBProjectId(id),
@@ -845,6 +916,7 @@ impl DBProject {
                                     &m.side_types_migration_review_status,
                                 ),
                                 loaders,
+                                components: components_serial,
                             },
                             categories: m.categories.unwrap_or_default(),
                             additional_categories: m.additional_categories.unwrap_or_default(),
@@ -858,16 +930,19 @@ impl DBProject {
                             urls,
                             aggregate_version_fields: VersionField::from_query_json(version_fields, &loader_fields, &loader_field_enum_values, true),
                             thread_id: DBThreadId(m.thread_id),
+                            components,
                         };
 
                         acc.insert(m.id, (m.slug, project));
-                        async move { Ok(acc) }
+                        Ok(acc)
                     })
-                    .await?;
+                    ?;
 
                 Ok(projects)
             },
-        ).await?;
+        )
+        .await
+        .wrap_internal_err("failed to fetch cached projects")?;
 
         Ok(val)
     }
@@ -983,4 +1058,6 @@ pub struct ProjectQueryResult {
     pub gallery_items: Vec<DBGalleryItem>,
     pub thread_id: DBThreadId,
     pub aggregate_version_fields: Vec<VersionField>,
+    #[serde(flatten)]
+    pub components: exp::ProjectQuery,
 }

apps/labrinth/src/database/models/version_item.rs

@@ -6,6 +6,7 @@ use crate::database::models::loader_fields::{
     QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
 };
 use crate::database::redis::RedisPool;
+use crate::models::exp;
 use crate::models::projects::{FileType, VersionStatus};
 use crate::routes::internal::delphi::DelphiRunParameters;
 use chrono::{DateTime, Utc};
@@ -37,6 +38,7 @@ pub struct VersionBuilder {
     pub status: VersionStatus,
     pub requested_status: Option<VersionStatus>,
     pub ordering: Option<i32>,
+    pub components: exp::VersionSerial,
 }
 
 #[derive(Clone)]
@@ -206,6 +208,7 @@ impl VersionBuilder {
             status: self.status,
             requested_status: self.requested_status,
             ordering: self.ordering,
+            components: self.components,
         };
 
         version.insert(transaction).await?;
@@ -285,7 +288,7 @@ impl DBLoaderVersion {
     }
 }
 
-#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
+#[derive(Clone, Deserialize, Serialize)]
 pub struct DBVersion {
     pub id: DBVersionId,
     pub project_id: DBProjectId,
@@ -300,8 +303,17 @@ pub struct DBVersion {
     pub status: VersionStatus,
     pub requested_status: Option<VersionStatus>,
     pub ordering: Option<i32>,
+    pub components: exp::VersionSerial,
 }
 
+impl PartialEq for DBVersion {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+impl Eq for DBVersion {}
+
 impl DBVersion {
     pub async fn insert(
         &self,
@@ -312,12 +324,14 @@ impl DBVersion {
             INSERT INTO versions (
                 id, mod_id, author_id, name, version_number,
                 changelog, date_published, downloads,
-                version_type, featured, status, ordering
+                version_type, featured, status, ordering,
+                components
             )
             VALUES (
                 $1, $2, $3, $4, $5,
                 $6, $7, $8,
-                $9, $10, $11, $12
+                $9, $10, $11, $12,
+                $13
             )
             ",
             self.id as DBVersionId,
@@ -331,7 +345,9 @@ impl DBVersion {
             &self.version_type,
             self.featured,
             self.status.as_str(),
-            self.ordering
+            self.ordering,
+            serde_json::to_value(&self.components)
+                .expect("serialization shouldn't fail"),
         )
         .execute(&mut *transaction)
         .await?;
@@ -719,13 +735,14 @@ impl DBVersion {
                     ).await?;
 
                 let res = sqlx::query!(
-                    "
+                    r#"
                     SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,
                     v.changelog changelog, v.date_published date_published, v.downloads downloads,
-                    v.version_type version_type, v.featured featured, v.status status, v.requested_status requested_status, v.ordering ordering
+                    v.version_type version_type, v.featured featured, v.status status, v.requested_status requested_status, v.ordering ordering,
+                    v.components AS "components: sqlx::types::Json<exp::VersionSerial>"
                     FROM versions v
                     WHERE v.id = ANY($1);
-                    ",
+                    "#,
                     &version_ids
                 )
                     .fetch(&mut exec)
@@ -746,6 +763,7 @@ impl DBVersion {
                             .filter(|x| loader_loader_field_ids.contains(&x.id))
                             .collect::<Vec<_>>();
 
+                        let components_serial = v.components.0;
                         let query_version = VersionQueryResult {
                             inner: DBVersion {
                                 id: DBVersionId(v.id),
@@ -762,6 +780,7 @@ impl DBVersion {
                                 requested_status: v.requested_status
                                     .map(|x| VersionStatus::from_string(&x)),
                                 ordering: v.ordering,
+                                components: components_serial,
                             },
                             files: {
                                 let mut files = files.into_iter().map(|x| {
@@ -804,6 +823,8 @@ impl DBVersion {
                             project_types,
                             games,
                             dependencies,
+                            // TODO populate
+                            components: exp::VersionQuery::default(),
                         };
 
                         acc.insert(v.id, query_version);
@@ -937,7 +958,7 @@ impl DBVersion {
     }
 }
 
-#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
+#[derive(Clone, Deserialize, Serialize)]
 pub struct VersionQueryResult {
     pub inner: DBVersion,
 
@@ -947,6 +968,8 @@ pub struct VersionQueryResult {
     pub project_types: Vec<String>,
     pub games: Vec<String>,
     pub dependencies: Vec<DependencyQueryResult>,
+    #[serde(flatten)]
+    pub components: exp::VersionQuery,
 }
 
 #[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
@@ -993,6 +1016,14 @@ impl std::cmp::PartialOrd for VersionQueryResult {
     }
 }
 
+impl std::cmp::PartialEq for VersionQueryResult {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner == other.inner
+    }
+}
+
+impl std::cmp::Eq for VersionQueryResult {}
+
 impl std::cmp::Ord for DBVersion {
     fn cmp(&self, other: &Self) -> Ordering {
         let ordering_order = match (self.ordering, other.ordering) {
@@ -1061,6 +1092,7 @@ mod tests {
             featured: false,
             status: VersionStatus::Listed,
             requested_status: None,
+            components: exp::VersionSerial::default(),
         }
     }
 }

apps/labrinth/src/env.rs

@@ -277,4 +277,10 @@ vars! {
     DELPHI_SLACK_WEBHOOK: String = "";
 
     TREMENDOUS_CAMPAIGN_ID: String = "";
+
+    // server pinging
+    SERVER_PING_MAX_CONCURRENT: usize = 16usize;
+    SERVER_PING_RETRIES: usize = 3usize;
+    SERVER_PING_MIN_INTERVAL_SEC: u64 = 30u64 * 60;
+    SERVER_PING_TIMEOUT_MS: u64 = 3u64 * 1000;
 }

apps/labrinth/src/lib.rs

@@ -120,12 +120,15 @@ pub fn app_setup(
             let redis_pool_ref = redis_pool_ref.clone();
             let search_config_ref = search_config_ref.clone();
             async move {
-                background_task::index_search(
+                if let Err(e) = background_task::index_search(
                     pool_ref,
                     redis_pool_ref,
                     search_config_ref,
                 )
-                .await;
+                .await
+                {
+                    warn!("Local project indexing failed: {e:#}");
+                }
             }
         });
 
@@ -135,7 +138,11 @@ pub fn app_setup(
         scheduler.run(Duration::from_secs(60 * 5), move || {
             let pool_ref = pool_ref.clone();
             async move {
-                background_task::release_scheduled(pool_ref).await;
+                if let Err(e) =
+                    background_task::release_scheduled(pool_ref).await
+                {
+                    warn!("Syncing scheduled releases failed: {e:#}");
+                }
             }
         });
 
@@ -147,7 +154,9 @@ pub fn app_setup(
             let pool_ref = pool_ref.clone();
             let redis = redis_pool_ref.clone();
             async move {
-                update_versions(pool_ref, redis).await;
+                if let Err(e) = update_versions(pool_ref, redis).await {
+                    warn!("Version update failed: {e:#}");
+                }
             }
         });
 
@@ -159,7 +168,12 @@ pub fn app_setup(
             let client_ref = client_ref.clone();
             let redis_ref = redis_pool_ref.clone();
             async move {
-                background_task::payouts(pool_ref, client_ref, redis_ref).await;
+                if let Err(e) =
+                    background_task::payouts(pool_ref, client_ref, redis_ref)
+                        .await
+                {
+                    warn!("Payout task failed: {e:#}");
+                }
             }
         });
 

apps/labrinth/src/main.rs

@@ -178,7 +178,8 @@ async fn app() -> std::io::Result<()> {
             email_queue,
             muralpay,
         )
-        .await;
+        .await
+        .map_err(std::io::Error::other)?;
         return Ok(());
     }
 

apps/labrinth/src/models/exp/base.rs

@@ -0,0 +1,33 @@
+use serde::{Deserialize, Serialize};
+use validator::Validate;
+
+use crate::models::{ids::OrganizationId, projects::ProjectStatus};
+
+#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+pub struct Project {
+    /// Human-readable friendly name of the project.
+    #[validate(
+        length(min = 3, max = 64),
+        custom(function = "crate::util::validate::validate_name")
+    )]
+    pub name: String,
+    /// Slug of the project, used in vanity URLs.
+    #[validate(
+        length(min = 3, max = 64),
+        regex(path = *crate::util::validate::RE_URL_SAFE)
+    )]
+    pub slug: String,
+    /// Short description of the project.
+    #[validate(length(min = 3, max = 255))]
+    pub summary: String,
+    /// A long description of the project, in markdown.
+    #[validate(length(max = 65536))]
+    pub description: String,
+    /// What status the user would like the project to be in after review.
+    pub requested_status: ProjectStatus,
+    /// What organization the project belongs to.
+    pub organization_id: Option<OrganizationId>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+pub struct Version {}

apps/labrinth/src/models/exp/compat.rs

@@ -0,0 +1,20 @@
+//! Compatibility utilities for V3 API.
+
+use crate::models::exp::ProjectSerial;
+
+const MINECRAFT_JAVA_SERVER: &str = "minecraft_java_server";
+
+/// Adjusts V3 project types based on a project's components.
+///
+/// The experimental API does not have a concept of project types; instead, a
+/// project's "type" is implicit based on what components it has.
+/// To reflect this in the V3 API, we manually add `project_types` values
+/// for compatibility with stuff like searching.
+pub fn correct_project_types(
+    components: &ProjectSerial,
+    project_types: &mut Vec<String>,
+) {
+    if components.minecraft_server.is_some() {
+        project_types.push(MINECRAFT_JAVA_SERVER.into());
+    }
+}

apps/labrinth/src/models/exp/component.rs

@@ -0,0 +1,335 @@
+macro_rules! define {
+    () => {};
+    (
+        $(#[$meta:meta])*
+        $vis:vis struct $name:ident {
+            $(
+                #[base(
+                    $($field_base_meta:meta),*
+                )]
+                #[edit(
+                    $($field_edit_meta:meta),*
+                )]
+                #[create($create:ident)]
+                $(#[$field_meta:meta])*
+                $field_vis:vis $field:ident: $field_ty:ty
+            ),* $(,)?
+        }
+
+        $($rest:tt)*
+    ) => { paste::paste! {
+        $(#[$meta])*
+        $vis struct $name {
+            $(
+                $(#[$field_meta])*
+                $(#[$field_base_meta])*
+                $field_vis $field: $field_ty,
+            )*
+        }
+
+        $(#[$meta])*
+        $vis struct [< $name Edit >] {
+            $(
+                $(#[$field_meta])*
+                $(#[$field_edit_meta])*
+                $field_vis $field: Option<$field_ty>,
+            )*
+        }
+
+        impl $crate::models::exp::component::Component for $name {
+            type EntityId = $crate::models::ids::ProjectId;
+            type Query = $name;
+            type Edit = [< $name Edit >];
+        }
+
+        impl $crate::models::exp::component::ComponentQuery for $name {
+            type Component = $name;
+            type Context = $crate::models::exp::project::ProjectQueryContext;
+            type Requirements = $crate::models::exp::project::ProjectQueryRequirements;
+
+            fn collect_requirements(
+                _serial: &Self::Component,
+                _entity_id: <Self::Component as Component>::EntityId,
+                _requirements: &mut Self::Requirements,
+            ) {}
+
+            fn populate(
+                serial: Self::Component,
+                _entity_id: <Self::Component as Component>::EntityId,
+                _context: &Self::Context,
+            ) -> Result<Self> {
+                Ok(serial)
+            }
+        }
+
+        impl $crate::models::exp::component::ComponentEdit for [< $name Edit >] {
+            type Component = $name;
+
+            fn create(self) -> Result<Self::Component> {
+                Ok($name {
+                    $(
+                        $field: $crate::models::exp::component::unwrap_edit::$create(
+                            self.$field,
+                            stringify!($field),
+                        )?,
+                    )*
+                })
+            }
+
+            async fn apply_to(
+                self,
+                #[allow(unused_variables)]
+                component: &mut Self::Component,
+            ) -> Result<()> {
+                $(
+                    if let Some(f) = self.$field {
+                        component.$field = f;
+                    }
+                )*
+                Ok(())
+            }
+        }
+
+        $crate::models::exp::component::define!($($rest)*);
+    }};
+}
+
+pub mod unwrap_edit {
+    use eyre::{Result, eyre};
+
+    pub fn required<T>(field: Option<T>, field_name: &str) -> Result<T> {
+        field.ok_or_else(|| eyre!("missing field `{field_name}`"))
+    }
+
+    pub fn optional<T>(
+        field: Option<Option<T>>,
+        field_name: &str,
+    ) -> Result<Option<T>> {
+        match field {
+            // present value
+            Some(Some(t)) => Ok(Some(t)),
+            // value is omitted from json -> no value
+            None => Ok(None),
+            // value is in json but is null -> empty
+            Some(None) => Err(eyre!("missing field `{field_name}`")),
+        }
+    }
+
+    pub fn default<T: Default>(
+        field: Option<T>,
+        _field_name: &str,
+    ) -> Result<T> {
+        Ok(field.unwrap_or_default())
+    }
+}
+
+macro_rules! relations {
+    ($vis:vis static $name:ident: $component_kind:ty = $expr:block) => {
+        $vis static $name: std::sync::LazyLock<Vec<$crate::models::exp::component::ComponentRelation<$component_kind>>> = std::sync::LazyLock::new(|| {
+            #[allow(unused_imports)]
+            use $crate::models::exp::component::{ComponentKindExt, ComponentKindArrayExt};
+
+            Vec::<$crate::models::exp::component::ComponentRelation<$component_kind>>::from($expr)
+        });
+    };
+}
+
+pub(crate) use define;
+use eyre::Result;
+pub(crate) use relations;
+
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use std::{collections::HashSet, hash::Hash};
+use thiserror::Error;
+
+pub trait ComponentKind:
+    Clone + Send + Sync + PartialEq + Eq + Hash + 'static
+{
+}
+
+/// Data attached to an entity (like a project or a version), comparable to a
+/// component in the [ECS paradigm](https://en.wikipedia.org/wiki/Entity_component_system).
+///
+/// The struct that implements this trait is the *serial form* of the
+/// component, as stored in the database. When it is queried or edited, the
+/// schema may take a different form - see [`Component::Query`],
+/// [`Component::Edit`].
+pub trait Component: Sized + Serialize + DeserializeOwned {
+    /// Type of ID that entities which have this type of component use to
+    /// identify themselves.
+    ///
+    /// - For project components, this is [`ProjectId`].
+    ///
+    /// [`ProjectId`]: crate::models::ids::ProjectId
+    type EntityId: Clone + Copy + Eq + Hash + Send + Sync;
+
+    /// Schema of the data returned when querying a component of this type from
+    /// the backend.
+    ///
+    /// See [`ComponentQuery`].
+    type Query: ComponentQuery<Component = Self>;
+
+    /// Schema of a modification that can be applied to an existing component of
+    /// this type.
+    ///
+    /// See [`ComponentEdit`].
+    type Edit: ComponentEdit<Component = Self>;
+}
+
+/// Schema of the data returned when querying a component of type
+/// [`Self::Component`] from the backend.
+///
+/// The [`Component`] stores persistent, serialized data; but when we
+/// request a project, we also request its components, and we may want to
+/// request extra data alongside the serialized form. For example, if our
+/// component stores a project ID to another project, we may want to return
+/// that project's name, icon, etc. alongside the ID. [`Component::Query`]
+/// provides a way to populate this extra data.
+pub trait ComponentQuery: Sized {
+    /// Type of serial component this [`ComponentQuery`] is queried from.
+    type Component: Component<Query = Self>;
+
+    /// Type of the whole set of information that a query requests from the
+    /// database.
+    ///
+    /// - For project components, this is [`ProjectQueryRequirements`].
+    ///
+    /// [`ProjectQueryRequirements`]: crate::models::exp::project::ProjectQueryRequirements
+    type Requirements;
+
+    /// Type of context provided during [`ComponentQuery::populate`].
+    ///
+    /// - For project components, this is [`ProjectQueryContext`].
+    ///
+    /// [`ProjectQueryContext`]: crate::models::exp::project::ProjectQueryContext
+    type Context;
+
+    /// What information does this query type require from the database to
+    /// populate itself (excluding the [`ComponentQuery::Component`])?
+    ///
+    /// For example, if the [`ComponentQuery::Component`] has a projecet ID,
+    /// this will add the project ID to `requirements`. This will require the
+    /// fetcher to also fetch this project ID, which will be available in the
+    /// [`ComponentQuery::Context`] during [`ComponentQuery::populate`].
+    fn collect_requirements(
+        serial: &Self::Component,
+        entity_id: <Self::Component as Component>::EntityId,
+        requirements: &mut Self::Requirements,
+    );
+
+    /// Creates the final component with all queried data, using the serialized
+    /// form of the component ([`ComponentQuery::Component`]) and any additional
+    /// info requested in [`ComponentQuery::collect_requirements`]
+    ///
+    /// # Errors
+    ///
+    /// Errors if some required data in the `context` is missing, indicating a
+    /// logic bug.
+    fn populate(
+        serial: Self::Component,
+        entity_id: <Self::Component as Component>::EntityId,
+        context: &Self::Context,
+    ) -> Result<Self>;
+}
+
+/// Schema of a modification to an existing component, or potentially creation
+/// of a component.
+///
+/// The [`Component`] stores persistent, serialized data; but when we want to
+/// edit only specific fields of an existing component, we have to be able to
+/// exclude fields which are not edited by wrapping the field in an [`Option`].
+/// This trait provides a schema for doing this.
+pub trait ComponentEdit: Sized {
+    /// Type of serial component this [`ComponentQuery`] is a modification for.
+    type Component: Component<Edit = Self>;
+
+    /// Attempts to create a [`ComponentEdit::Component`] if this edit has all
+    /// of the appropriate fields set.
+    ///
+    /// # Errors
+    ///
+    /// Errors if a required field is missing.
+    fn create(self) -> Result<Self::Component>;
+
+    /// Applies this edit to an existing component.
+    ///
+    /// Errors if an edit could not be applied.
+    // note: this is `async` because in the future this might issue db/sqlx queries
+    #[expect(async_fn_in_trait, reason = "internal trait")]
+    async fn apply_to(self, component: &mut Self::Component) -> Result<()>;
+}
+
+#[derive(Debug, Clone)]
+pub enum ComponentRelation<K> {
+    /// If one of these components is present, then it can only be present with
+    /// other components from this set.
+    Only(HashSet<K>),
+    /// If component `0` is present, then `1` must also be present.
+    Requires(K, K),
+}
+
+pub trait ComponentKindExt<K> {
+    fn requires(self, other: K) -> ComponentRelation<K>;
+}
+
+impl<K> ComponentKindExt<K> for K {
+    fn requires(self, other: K) -> ComponentRelation<K> {
+        ComponentRelation::Requires(self, other)
+    }
+}
+
+pub trait ComponentKindArrayExt<K> {
+    fn only(self) -> ComponentRelation<K>;
+}
+
+impl<K: ComponentKind, const N: usize> ComponentKindArrayExt<K> for [K; N] {
+    fn only(self) -> ComponentRelation<K> {
+        ComponentRelation::Only(self.iter().cloned().collect())
+    }
+}
+
+#[derive(Debug, Clone, Error, Serialize, Deserialize)]
+pub enum ComponentRelationError<K: ComponentKind> {
+    #[error("no components")]
+    NoComponents,
+    #[error("component `{target:?}` is missing")]
+    Missing { target: K },
+    #[error(
+        "only components {only:?} can be together, found extra components {extra:?}"
+    )]
+    Only { only: HashSet<K>, extra: HashSet<K> },
+    #[error("component `{target:?}` requires `{requires:?}`")]
+    Requires { target: K, requires: K },
+}
+
+pub fn kinds_valid<K: ComponentKind>(
+    kinds: &HashSet<K>,
+    relations: &[ComponentRelation<K>],
+) -> Result<(), ComponentRelationError<K>> {
+    for relation in relations {
+        match relation {
+            ComponentRelation::Only(set) => {
+                if kinds.iter().any(|k| set.contains(k)) {
+                    let extra: HashSet<_> =
+                        kinds.difference(set).cloned().collect();
+                    if !extra.is_empty() {
+                        return Err(ComponentRelationError::Only {
+                            only: set.clone(),
+                            extra,
+                        });
+                    }
+                }
+            }
+            ComponentRelation::Requires(a, b) => {
+                if kinds.contains(a) && !kinds.contains(b) {
+                    return Err(ComponentRelationError::Requires {
+                        target: a.clone(),
+                        requires: b.clone(),
+                    });
+                }
+            }
+        }
+    }
+
+    Ok(())
+}

apps/labrinth/src/models/exp/minecraft.rs

@@ -0,0 +1,388 @@
+use std::time::Duration;
+
+use chrono::{DateTime, Utc};
+use eyre::Result;
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+use validator::Validate;
+
+use crate::{
+    models::{
+        exp::{
+            ProjectComponentKind,
+            component::{self, Component, ComponentEdit, ComponentQuery},
+            project::{
+                ProjectComponent, ProjectQueryContext, ProjectQueryRequirements,
+            },
+        },
+        ids::{ProjectId, VersionId},
+    },
+    util::error::Context,
+};
+
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    Serialize,
+    Deserialize,
+    utoipa::ToSchema,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum Language {
+    En,
+    Es,
+    Pt,
+    Fr,
+    De,
+    It,
+    Nl,
+    Ru,
+    Uk,
+    Pl,
+    Cs,
+    Sk,
+    Hu,
+    Ro,
+    Bg,
+    Hr,
+    Sr,
+    El,
+    Tr,
+    Ar,
+    He,
+    Hi,
+    Bn,
+    Ur,
+    Zh,
+    Ja,
+    Ko,
+    Th,
+    Vi,
+    Id,
+    Ms,
+    Tl,
+    Sv,
+    No,
+    Da,
+    Fi,
+    Lt,
+    Lv,
+    Et,
+}
+
+component::define! {
+    #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+    pub struct ModProject {}
+
+    /// Listing for a Minecraft server.
+    #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+    pub struct ServerProject {
+        #[base(serde(default))]
+        #[edit(serde(
+            default,
+            skip_serializing_if = "Option::is_none",
+            with = "serde_with::rust::double_option"
+        ))]
+        #[create(optional)]
+        /// Maximum number of players allowed on the server.
+        pub max_players: Option<u32>,
+        #[base(serde(default))]
+        #[edit(serde(
+            default,
+            skip_serializing_if = "Option::is_none",
+            with = "serde_with::rust::double_option"
+        ))]
+        #[create(optional)]
+        /// Country which this server is hosted in.
+        #[validate(length(min = 2, max = 2))]
+        pub country: Option<String>,
+        #[base(serde(default))]
+        #[edit(serde(default))]
+        #[create(default)]
+        /// Languages which the owners of this server prefer.
+        pub languages: Vec<Language>,
+        #[base(serde(default))]
+        #[edit(serde(
+            default,
+            skip_serializing_if = "Option::is_none",
+            with = "serde_with::rust::double_option"
+        ))]
+        #[create(optional)]
+        /// Which version of the listing this server is currently using.
+        pub active_version: Option<VersionId>,
+    }
+
+    /// Version of a Minecraft Java server listing.
+    #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+    pub struct JavaServerVersion {}
+
+    /// Listing for a Minecraft Bedrock server.
+    #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+    pub struct BedrockServerProject {
+        #[base()]
+        #[edit(serde(default))]
+        #[create(required)]
+        /// Address (IP or domain name) of the Bedrock server, excluding port.
+        #[validate(length(max = 255))]
+        pub address: String,
+        #[base()]
+        #[edit(serde(default))]
+        #[create(required)]
+        /// Port which the server runs on.
+        pub port: u16,
+    }
+}
+
+impl ProjectComponent for ModProject {
+    fn kind() -> ProjectComponentKind {
+        ProjectComponentKind::MinecraftMod
+    }
+}
+
+impl ProjectComponent for ServerProject {
+    fn kind() -> ProjectComponentKind {
+        ProjectComponentKind::MinecraftServer
+    }
+}
+
+impl ProjectComponent for JavaServerProject {
+    fn kind() -> ProjectComponentKind {
+        ProjectComponentKind::MinecraftJavaServer
+    }
+}
+
+impl ProjectComponent for BedrockServerProject {
+    fn kind() -> ProjectComponentKind {
+        ProjectComponentKind::MinecraftBedrockServer
+    }
+}
+
+/// Listing for a Minecraft Java server.
+#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+pub struct JavaServerProject {
+    /// Address (IP or domain name) of the Java server, excluding port.
+    #[validate(length(max = 255))]
+    pub address: String,
+    /// Port which the server runs on.
+    pub port: u16,
+    /// What game content this server is using.
+    pub content: ServerContent,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+pub struct JavaServerProjectEdit {
+    #[validate(length(max = 255))]
+    #[serde(default)]
+    pub address: Option<String>,
+    #[serde(default)]
+    pub port: Option<u16>,
+    #[serde(default)]
+    pub content: Option<ServerContent>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+pub struct JavaServerProjectQuery {
+    pub address: String,
+    pub port: u16,
+    pub content: ServerContentQuery,
+    pub ping: Option<JavaServerPing>,
+}
+
+impl Component for JavaServerProject {
+    type EntityId = ProjectId;
+    type Query = JavaServerProjectQuery;
+    type Edit = JavaServerProjectEdit;
+}
+
+impl ComponentQuery for JavaServerProjectQuery {
+    type Component = JavaServerProject;
+    type Requirements = ProjectQueryRequirements;
+    type Context = ProjectQueryContext;
+
+    fn collect_requirements(
+        serial: &Self::Component,
+        project_id: ProjectId,
+        requirements: &mut ProjectQueryRequirements,
+    ) {
+        match serial.content {
+            ServerContent::Vanilla { .. } => {}
+            ServerContent::Modpack { version_id } => {
+                requirements.partial_versions.insert(version_id);
+            }
+        }
+        requirements.minecraft_java_server_pings.insert(project_id);
+    }
+
+    fn populate(
+        serial: Self::Component,
+        project_id: ProjectId,
+        context: &ProjectQueryContext,
+    ) -> Result<Self> {
+        Ok(Self {
+            address: serial.address,
+            port: serial.port,
+            content: match serial.content {
+                ServerContent::Vanilla {
+                    supported_game_versions,
+                    recommended_game_version,
+                } => ServerContentQuery::Vanilla {
+                    supported_game_versions,
+                    recommended_game_version,
+                },
+                ServerContent::Modpack { version_id } => {
+                    match context.partial_versions.get(&version_id) {
+                        Some(version) => ServerContentQuery::Modpack {
+                            version_id,
+                            project_id: version.project_id,
+                            project_name: version.project_name.clone(),
+                            project_icon: version.project_icon.clone(),
+                        },
+                        None => {
+                            // TODO: should be upgraded to an error,
+                            // but it's too easy to fall into this illegal state right now
+                            warn!("no modpack info for version {version_id:?}");
+                            ServerContentQuery::Modpack {
+                                version_id,
+                                project_id: ProjectId(0),
+                                project_name: String::new(),
+                                project_icon: String::new(),
+                            }
+                        }
+                    }
+                }
+            },
+            ping: context
+                .minecraft_java_server_pings
+                .get(&project_id)
+                .cloned(),
+        })
+    }
+}
+
+impl ComponentEdit for JavaServerProjectEdit {
+    type Component = JavaServerProject;
+
+    fn create(self) -> Result<Self::Component> {
+        Ok(JavaServerProject {
+            address: self.address.wrap_err("missing `address`")?,
+            port: self.port.wrap_err("missing `port`")?,
+            content: self.content.unwrap_or_default(),
+        })
+    }
+
+    async fn apply_to(self, component: &mut Self::Component) -> Result<()> {
+        if let Some(address) = self.address {
+            component.address = address;
+        }
+        if let Some(port) = self.port {
+            component.port = port;
+        }
+        if let Some(content) = self.content {
+            component.content = content;
+        }
+        Ok(())
+    }
+}
+
+/// What game content a [`JavaServerProject`] is using.
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum ServerContent {
+    /// Server runs modded content with a modpack found on the Modrinth platform.
+    Modpack {
+        /// Version ID of the modpack which the server runs.
+        ///
+        /// This version may or may not belong to the server project, since
+        /// server projects may also be treated as modpacks.
+        version_id: VersionId,
+    },
+    /// Server is a vanilla Minecraft server.
+    Vanilla {
+        /// List of supported Minecraft Java client versions which can join this
+        /// server.
+        supported_game_versions: Vec<String>,
+        /// Recommended Minecraft Java client version to use to join this server.
+        recommended_game_version: Option<String>,
+    },
+}
+
+/// What game content a [`JavaServerProject`] is using.
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum ServerContentQuery {
+    /// Server runs modded content with a modpack found on the Modrinth platform.
+    Modpack {
+        version_id: VersionId,
+        project_id: ProjectId,
+        project_name: String,
+        project_icon: String,
+    },
+    /// Server is a vanilla Minecraft server.
+    Vanilla {
+        /// List of supported Minecraft Java client versions which can join this
+        /// server.
+        supported_game_versions: Vec<String>,
+        /// Recommended Minecraft Java client version to use to join this server.
+        recommended_game_version: Option<String>,
+    },
+}
+
+impl Default for ServerContent {
+    fn default() -> Self {
+        ServerContent::Vanilla {
+            supported_game_versions: Vec::new(),
+            recommended_game_version: None,
+        }
+    }
+}
+
+/// Recorded ping attempt that Labrinth made to a Minecraft Java server project.
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+pub struct JavaServerPing {
+    /// When the ping was performed.
+    pub when: DateTime<Utc>,
+    /// Address of the server at the time of the ping.
+    pub address: String,
+    /// Port of the server at the time of the ping.
+    pub port: u16,
+    /// If the ping was successful, info on the ping response.
+    pub data: Option<JavaServerPingData>,
+}
+
+/// Ping response data for a Minecraft Java server.
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+pub struct JavaServerPingData {
+    /// How long it took for the Labrinth worker to ping the server.
+    ///
+    /// Note: this is explicitly *not* a client-side ping time, so this should
+    /// not be used to display to a client how much latency they have to a
+    /// specific server. This is purely for internal metrics.
+    pub latency: Duration,
+    /// Reported version name of the server.
+    pub version_name: String,
+    /// Reported version protocol number of the server.
+    pub version_protocol: u32,
+    /// Description/MOTD of the server as shown in the server list.
+    pub description: String,
+    /// Number of players online at the time.
+    pub players_online: u32,
+    /// Maximum number of players allowed on the server.
+    pub players_max: u32,
+}
+
+component::relations! {
+    pub(super) static PROJECT_COMPONENT_RELATIONS: ProjectComponentKind = {
+        use ProjectComponentKind::*;
+
+        [
+            [MinecraftMod].only(),
+            [MinecraftServer, MinecraftJavaServer, MinecraftBedrockServer].only(),
+            MinecraftJavaServer.requires(MinecraftServer),
+            MinecraftBedrockServer.requires(MinecraftServer),
+        ]
+    }
+}

apps/labrinth/src/models/exp/mod.rs

@@ -0,0 +1,29 @@
+//! Highly experimental and unstable API endpoint models.
+//!
+//! These are used for testing new API patterns and exploring future endpoints,
+//! which may or may not make it into an official release.
+//!
+//! # Projects and versions
+//!
+//! Projects and versions work in an ECS-like architecture, where each project
+//! is an entity (project ID), and components can be attached to that project to
+//! determine the project's type, like a Minecraft mod, data pack, etc. Project
+//! components *may* store extra data (like a server listing which stores the
+//! server address), but typically, the version will store this data in *version
+//! components*.
+
+pub mod base;
+pub mod compat;
+pub mod component;
+pub mod minecraft;
+pub mod project;
+pub mod version;
+
+pub use project::{
+    PROJECT_COMPONENT_RELATIONS, ProjectComponentKind, ProjectEdit,
+    ProjectQuery, ProjectSerial,
+};
+pub use version::{
+    VersionComponentKind, VersionCreate, VersionEdit, VersionQuery,
+    VersionSerial,
+};

apps/labrinth/src/models/exp/project.rs

@@ -0,0 +1,269 @@
+use eyre::Result;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use validator::Validate;
+
+use crate::{
+    database::{
+        models::{DBProjectId, DBVersionId},
+        redis::RedisPool,
+    },
+    models::{
+        exp::{
+            component::{
+                self, Component, ComponentEdit, ComponentKind, ComponentQuery,
+            },
+            minecraft,
+        },
+        ids::{ProjectId, VersionId},
+    },
+    queue::server_ping,
+    util::error::Context,
+};
+
+pub trait ProjectComponent: Component<EntityId = ProjectId> {
+    fn kind() -> ProjectComponentKind;
+}
+
+macro_rules! define_project_components {
+    (
+        $(($field_name:ident, $variant_name:ident): $ty:ty),* $(,)?
+    ) => {
+        // kinds
+
+        #[expect(dead_code, reason = "static check so $ty implements `ProjectComponent`")]
+        const _: () = {
+            fn assert_implements_component<T: ProjectComponent>() {}
+
+            fn assert_components_implement_trait() {
+                $(assert_implements_component::<$ty>();)*
+            }
+        };
+
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+        pub enum ProjectComponentKind {
+            $($variant_name,)*
+        }
+
+        impl ComponentKind for ProjectComponentKind {}
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize, Validate)]
+        pub struct ProjectSerial {
+            $(
+                #[validate(nested)]
+                pub $field_name: Option<$ty>,
+            )*
+        }
+
+        impl ProjectSerial {
+            #[must_use]
+            pub fn component_kinds(&self) -> HashSet<ProjectComponentKind> {
+                let mut kinds = HashSet::new();
+                $(
+                    if self.$field_name.is_some() {
+                        kinds.insert(ProjectComponentKind::$variant_name);
+                    }
+                )*
+                kinds
+            }
+
+            pub fn collect_query_requirements(
+                &self,
+                project_id: ProjectId,
+                requirements: &mut ProjectQueryRequirements,
+            ) {
+                $(
+                    if let Some(component) = &self.$field_name {
+                        <<$ty as Component>::Query as ComponentQuery>::collect_requirements(
+                            component,
+                            project_id,
+                            requirements
+                        );
+                    }
+                )*
+            }
+
+            pub fn into_query(
+                self,
+                project_id: ProjectId,
+                context: &ProjectQueryContext,
+            ) -> Result<ProjectQuery> {
+                Ok(ProjectQuery {
+                    $(
+                        $field_name: match self.$field_name {
+                            Some(serial) => {
+                                <$ty as Component>::Query::populate(
+                                    serial,
+                                    project_id,
+                                    context,
+                                )
+                                .map(Some)
+                                .wrap_err(concat!("failed to populate `", stringify!($ty), "`"))?
+                            }
+                            None => None,
+                        },
+                    )*
+                })
+            }
+        }
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)]
+        pub struct ProjectQuery {
+            $(
+                pub $field_name: Option<Query<$ty>>,
+            )*
+        }
+
+        #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+        pub struct ProjectEdit {
+            $(
+                #[validate(nested)]
+                #[serde(default)]
+                pub $field_name: Option<Edit<$ty>>,
+            )*
+        }
+
+        impl ProjectEdit {
+            #[must_use]
+            pub fn component_kinds(&self) -> HashSet<ProjectComponentKind> {
+                let mut kinds = HashSet::new();
+                $(
+                    if self.$field_name.is_some() {
+                        kinds.insert(ProjectComponentKind::$variant_name);
+                    }
+                )*
+                kinds
+            }
+
+            pub fn create(self) -> Result<ProjectSerial> {
+                Ok(ProjectSerial {
+                    $(
+                        $field_name: self
+                            .$field_name
+                            .map(<<$ty as Component>::Edit as ComponentEdit>::create)
+                            .transpose()?,
+                    )*
+                })
+            }
+        }
+    };
+}
+
+// needed because the `utoipa::ToSchema` macro is broken
+// when you have a `::` in the type path
+type Edit<T> = <T as Component>::Edit;
+type Query<T> = <T as Component>::Query;
+
+define_project_components![
+    (minecraft_mod, MinecraftMod): minecraft::ModProject,
+    (minecraft_server, MinecraftServer): minecraft::ServerProject,
+    (minecraft_java_server, MinecraftJavaServer): minecraft::JavaServerProject,
+    (minecraft_bedrock_server, MinecraftBedrockServer): minecraft::BedrockServerProject,
+];
+
+component::relations! {
+    pub static PROJECT_COMPONENT_RELATIONS: ProjectComponentKind = {
+        minecraft::PROJECT_COMPONENT_RELATIONS.clone()
+    }
+}
+
+// query logic
+
+#[derive(Default)]
+pub struct ProjectQueryRequirements {
+    pub partial_versions: HashSet<VersionId>,
+    pub minecraft_java_server_pings: HashSet<ProjectId>,
+}
+
+pub struct ProjectQueryContext {
+    pub partial_versions: HashMap<VersionId, PartialVersion>,
+    pub minecraft_java_server_pings:
+        HashMap<ProjectId, minecraft::JavaServerPing>,
+}
+
+#[derive(Clone, Debug)]
+pub struct PartialVersion {
+    pub project_id: ProjectId,
+    pub project_name: String,
+    pub project_icon: String,
+}
+
+pub async fn fetch_query_context(
+    projects: &[(ProjectId, &ProjectSerial)],
+    db: impl crate::database::Executor<'_, Database = sqlx::Postgres>,
+    redis: &RedisPool,
+) -> Result<ProjectQueryContext> {
+    let mut requirements = ProjectQueryRequirements::default();
+    for (project_id, project) in projects {
+        project.collect_query_requirements(*project_id, &mut requirements);
+    }
+    let ProjectQueryRequirements {
+        partial_versions,
+        minecraft_java_server_pings,
+    } = requirements;
+
+    let partial_versions = if partial_versions.is_empty() {
+        HashMap::new()
+    } else {
+        sqlx::query!(
+            r#"
+            SELECT
+                v.id AS "version_id: DBVersionId",
+                m.id AS "project_id: DBProjectId",
+                m.name AS "project_name!",
+                COALESCE(m.icon_url, '') AS "project_icon!"
+            FROM versions v
+            INNER JOIN mods m ON m.id = v.mod_id
+            WHERE v.id = ANY($1)
+            "#,
+            &partial_versions
+                .iter()
+                .map(|id| DBVersionId::from(*id).0)
+                .collect::<Vec<_>>(),
+        )
+        .fetch_all(db)
+        .await
+        .wrap_err("failed to fetch partial versions")?
+        .into_iter()
+        .map(|row| {
+            (
+                VersionId::from(row.version_id),
+                PartialVersion {
+                    project_id: ProjectId::from(row.project_id),
+                    project_name: row.project_name,
+                    project_icon: row.project_icon,
+                },
+            )
+        })
+        .collect::<HashMap<_, _>>()
+    };
+
+    let minecraft_java_server_pings =
+        minecraft_java_server_pings.into_iter().collect::<Vec<_>>();
+    let minecraft_java_server_pings = if minecraft_java_server_pings.is_empty()
+    {
+        HashMap::new()
+    } else {
+        let mut redis = redis.connect().await?;
+        redis
+            .get_many_deserialized_from_json::<minecraft::JavaServerPing>(
+                server_ping::REDIS_NAMESPACE,
+                &minecraft_java_server_pings
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>(),
+            )
+            .await?
+            .into_iter()
+            .enumerate()
+            .filter_map(|(idx, ping)| {
+                ping.map(|ping| (minecraft_java_server_pings[idx], ping))
+            })
+            .collect::<HashMap<_, _>>()
+    };
+
+    Ok(ProjectQueryContext {
+        partial_versions,
+        minecraft_java_server_pings,
+    })
+}

apps/labrinth/src/models/exp/version.rs

@@ -0,0 +1,98 @@
+use crate::models::exp::{
+    base,
+    component::{Component, ComponentKind},
+    minecraft,
+};
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use validator::Validate;
+
+macro_rules! define_version_components {
+    (
+        $(($field_name:ident, $variant_name:ident): $ty:ty),* $(,)?
+    ) => {
+        // kinds
+
+        #[expect(dead_code, reason = "static check so $ty implements `Component`")]
+        const _: () = {
+            fn assert_implements_component<T: Component>() {}
+
+            fn assert_components_implement_trait() {
+                $(assert_implements_component::<$ty>();)*
+            }
+        };
+
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+        pub enum VersionComponentKind {
+            $($variant_name,)*
+        }
+
+        impl ComponentKind for VersionComponentKind {}
+
+        // structs
+
+        #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+        pub struct Version {
+            #[validate(nested)]
+            pub base: base::Version,
+            $(
+                #[serde(skip_serializing_if = "Option::is_none")]
+                #[validate(nested)]
+                pub $field_name: Option<$ty>,
+            )*
+        }
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize)]
+        pub struct VersionSerial {
+            $(
+                pub $field_name: Option<$ty>,
+            )*
+        }
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+        pub struct VersionCreate {
+            #[validate(nested)]
+            pub base: Option<base::Project>,
+            $(
+                #[validate(nested)]
+                pub $field_name: Option<$ty>,
+            )*
+        }
+
+        impl VersionCreate {
+            #[must_use]
+            pub fn component_kinds(&self) -> HashSet<VersionComponentKind> {
+                let mut kinds = HashSet::new();
+                $(if self.$field_name.is_some() {
+                    kinds.insert(VersionComponentKind::$variant_name);
+                })*
+                kinds
+            }
+        }
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)]
+        pub struct VersionQuery {
+            $(
+                pub $field_name: Option<Query<$ty>>,
+            )*
+        }
+
+        #[derive(Debug, Clone, Default, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+        pub struct VersionEdit {
+            $(
+                #[validate(nested)]
+                pub $field_name: Option<Edit<$ty>>,
+            )*
+        }
+    };
+}
+
+// needed because the `utoipa::ToSchema` macro is broken
+// when you have a `::` in the type path
+type Edit<T> = <T as Component>::Edit;
+type Query<T> = <T as Component>::Query;
+
+define_version_components![
+    (minecraft_java_server, MinecraftJavaServer): minecraft::JavaServerVersion,
+];

apps/labrinth/src/models/mod.rs

@@ -1,4 +1,5 @@
 pub mod error;
+pub mod exp;
 pub mod v2;
 pub mod v3;
 

apps/labrinth/src/models/v3/projects.rs

@@ -4,6 +4,7 @@ use std::mem;
 use crate::database::models::loader_fields::VersionField;
 use crate::database::models::project_item::{LinkUrl, ProjectQueryResult};
 use crate::database::models::version_item::VersionQueryResult;
+use crate::models::exp;
 use crate::models::ids::{
     FileId, OrganizationId, ProjectId, TeamId, ThreadId, VersionId,
 };
@@ -95,6 +96,8 @@ pub struct Project {
     /// The status of the manual review of the migration of side types of this project
     pub side_types_migration_review_status: SideTypesMigrationReviewStatus,
 
+    #[serde(flatten)]
+    pub components: exp::ProjectQuery,
     /// Aggregated loader-fields across its myriad of versions
     #[serde(flatten)]
     pub fields: HashMap<String, Vec<serde_json::Value>>,
@@ -212,6 +215,7 @@ impl From<ProjectQueryResult> for Project {
             side_types_migration_review_status: m
                 .side_types_migration_review_status,
             fields,
+            components: data.components,
         }
     }
 }
@@ -642,7 +646,7 @@ impl SideTypesMigrationReviewStatus {
 }
 
 /// A specific version of a project
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
 pub struct Version {
     /// The ID of the version, encoded as a base62 string.
     pub id: VersionId,
@@ -685,6 +689,8 @@ pub struct Version {
     /// Ordering override, lower is returned first
     pub ordering: Option<i32>,
 
+    #[serde(flatten)]
+    pub components: exp::VersionQuery,
     // All other fields are loader-specific VersionFields
     // These are flattened during serialization
     #[serde(deserialize_with = "skip_nulls")]
@@ -761,6 +767,7 @@ impl From<VersionQueryResult> for Version {
                 .into_iter()
                 .map(|vf| (vf.field_name, vf.value.serialize_internal()))
                 .collect(),
+            components: data.components,
         }
     }
 }
@@ -771,7 +778,9 @@ impl From<VersionQueryResult> for Version {
 /// Draft - Version is not displayed on project, and not accessible by URL
 /// Unlisted - Version is not displayed on project, and accessible by URL
 /// Scheduled - Version is scheduled to be released in the future
-#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Debug)]
+#[derive(
+    Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Debug, utoipa::ToSchema,
+)]
 #[serde(rename_all = "lowercase")]
 pub enum VersionStatus {
     Listed,
@@ -855,7 +864,7 @@ impl VersionStatus {
 }
 
 /// A single project file, with a url for the file and the file's hash
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
 pub struct VersionFile {
     /// The ID of the file. Every file has an ID once created, but it
     /// is not known until it indeed has been created.
@@ -878,7 +887,9 @@ pub struct VersionFile {
 
 /// A dendency which describes what versions are required, break support, or are optional to the
 /// version's functionality
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
+#[derive(
+    Serialize, Deserialize, Clone, Debug, PartialEq, Eq, utoipa::ToSchema,
+)]
 pub struct Dependency {
     /// The specific version id that the dependency uses
     pub version_id: Option<VersionId>,
@@ -890,7 +901,9 @@ pub struct Dependency {
     pub dependency_type: DependencyType,
 }
 
-#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Debug)]
+#[derive(
+    Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Debug, utoipa::ToSchema,
+)]
 #[serde(rename_all = "lowercase")]
 pub enum VersionType {
     Release,
@@ -914,7 +927,9 @@ impl VersionType {
     }
 }
 
-#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(
+    Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, utoipa::ToSchema,
+)]
 #[serde(rename_all = "lowercase")]
 pub enum DependencyType {
     Required,
@@ -951,7 +966,9 @@ impl DependencyType {
     }
 }
 
-#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(
+    Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, utoipa::ToSchema,
+)]
 #[serde(rename_all = "kebab-case")]
 pub enum FileType {
     RequiredResourcePack,
@@ -998,7 +1015,9 @@ impl FileType {
 }
 
 /// A project loader
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
+#[derive(
+    Serialize, Deserialize, Clone, Debug, PartialEq, Eq, utoipa::ToSchema,
+)]
 #[serde(transparent)]
 pub struct Loader(pub String);
 

apps/labrinth/src/queue/mod.rs

@@ -3,5 +3,6 @@ pub mod billing;
 pub mod email;
 pub mod moderation;
 pub mod payouts;
+pub mod server_ping;
 pub mod session;
 pub mod socket;

apps/labrinth/src/queue/server_ping.rs

@@ -0,0 +1,307 @@
+use crate::database::DBProject;
+use crate::database::models::DBProjectId;
+use crate::database::redis::RedisPool;
+use crate::env::ENV;
+use crate::models::exp;
+use crate::models::ids::ProjectId;
+use crate::{database::PgPool, util::error::Context};
+use async_minecraft_ping::ServerDescription;
+use chrono::{TimeDelta, Utc};
+use clickhouse::{Client, Row};
+use serde::Serialize;
+use sqlx::types::Json;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::sync::Semaphore;
+use tokio::task::JoinSet;
+use tracing::{Instrument, debug, info, info_span, warn};
+
+pub struct ServerPingQueue {
+    pub db: PgPool,
+    pub redis: RedisPool,
+    pub clickhouse: Client,
+}
+
+pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping";
+pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings";
+
+impl ServerPingQueue {
+    pub fn new(db: PgPool, redis: RedisPool, clickhouse: Client) -> Self {
+        Self {
+            db,
+            redis,
+            clickhouse,
+        }
+    }
+
+    pub async fn ping_minecraft_java_servers(&self) -> eyre::Result<()> {
+        let server_projects = self.find_servers_to_ping().await?;
+        info!("Found {} servers to ping", server_projects.len());
+
+        let active_pings =
+            Arc::new(Semaphore::new(ENV.SERVER_PING_MAX_CONCURRENT));
+
+        let pings = server_projects
+            .into_iter()
+            .filter_map(|(project_id, java_server)| {
+                let address = java_server.address.to_string();
+                if address.trim().is_empty() {
+                    debug!("Skipping ping for server project {project_id} with empty address");
+                    return None;
+                }
+
+                let port = java_server.port;
+                let span = info_span!("ping", %project_id, %address, %port);
+
+                let active_pings = active_pings.clone();
+                let task = async move {
+                    let _permit = active_pings.acquire().await.expect("semaphore should not be closed now");
+
+                    let mut retries = ENV.SERVER_PING_RETRIES;
+                    let result = loop {
+                        match ping_server(&address, port).await {
+                            Ok(ping) => {
+                                debug!(?ping, "Received successful ping");
+                                break Ok(ping);
+                            }
+                            Err(err) if retries == 0 => {
+                                debug!("Failed to ping server in {:?}ms, no retries left: {err:#}", ENV.SERVER_PING_TIMEOUT_MS);
+                                break Err(err);
+                            }
+                            Err(err) => {
+                                debug!(%retries, "Failed to ping server in {:?}ms, retrying: {err:#}", ENV.SERVER_PING_TIMEOUT_MS);
+                                retries -= 1;
+                                continue;
+                            }
+                        };
+                    };
+
+                    (project_id, exp::minecraft::JavaServerPing {
+                        when: Utc::now(),
+                        address: address.to_string(),
+                        port,
+                        data: result.ok(),
+                    })
+                };
+                Some(tokio::spawn(task.instrument(span)))
+            })
+            .collect::<JoinSet<_>>()
+            .join_all()
+            .await
+            .into_iter()
+            .filter_map(|result| result.ok())
+            .collect::<Vec<_>>();
+
+        if !pings.is_empty() {
+            let mut ch = self
+                .clickhouse
+                .insert::<ServerPingRecord>(CLICKHOUSE_TABLE)
+                .await
+                .wrap_err("failed to begin inserting ping records")?;
+
+            let mut redis = self
+                .redis
+                .connect()
+                .await
+                .wrap_err("failed to connect to redis")?;
+
+            for (project_id, ping) in &pings {
+                let data = ping.data.as_ref();
+
+                let row = ServerPingRecord {
+                    recorded: ping.when.timestamp_nanos_opt().unwrap()
+                        / 100_000,
+                    project_id: project_id.0,
+                    address: ping.address.clone(),
+                    port: ping.port,
+                    latency_ms: data.map(|d| d.latency.as_millis() as u32),
+                    description: data.map(|d| d.description.clone()),
+                    version_name: data.map(|d| d.version_name.clone()),
+                    version_protocol: data.map(|d| d.version_protocol),
+                    players_online: data.map(|d| d.players_online),
+                    players_max: data.map(|d| d.players_max),
+                };
+
+                ch.write(&row)
+                    .await
+                    .wrap_err("failed to write ping record")?;
+
+                redis
+                    .set_serialized_to_json(
+                        REDIS_NAMESPACE,
+                        project_id,
+                        ping,
+                        None,
+                    )
+                    .await
+                    .wrap_err("failed to set redis key")?;
+
+                DBProject::clear_cache(
+                    (*project_id).into(),
+                    None,
+                    None,
+                    &self.redis,
+                )
+                .await
+                .inspect_err(|err| {
+                    warn!("failed to clear project cache: {err:#}")
+                })
+                .ok();
+            }
+
+            ch.end()
+                .await
+                .wrap_err("failed to end inserting ping records")?;
+        }
+
+        let num_success =
+            pings.iter().filter(|(_, ping)| ping.data.is_some()).count();
+        let num_total = pings.len();
+
+        info!(
+            "Inserted ping results for {} servers - {num_success}/{num_total} successful",
+            pings.len()
+        );
+        Ok(())
+    }
+
+    async fn find_servers_to_ping(
+        &self,
+    ) -> eyre::Result<Vec<(ProjectId, exp::minecraft::JavaServerProject)>> {
+        // first select all java servers
+        let all_server_projects = sqlx::query!(
+            r#"
+            SELECT id, components AS "components: Json<exp::ProjectSerial>"
+            FROM mods
+            WHERE components ? 'minecraft_java_server'
+            "#
+        )
+        .fetch_all(&self.db)
+        .await
+        .wrap_err("failed to fetch servers to ping")?;
+
+        if all_server_projects.is_empty() {
+            // we must early-exit, otherwise we'll run `redis.get_many()`,
+            // which runs `MGET` with no args; this gives:
+            // "ResponseError: wrong number of arguments for 'mget' command"
+            return Ok(Vec::new());
+        }
+
+        let mut redis = self
+            .redis
+            .connect()
+            .await
+            .wrap_err("failed to connect to redis")?;
+
+        // get the last ping info for all of them
+        // querying redis here, which is a cache, not the source of truth (clickhouse),
+        // but it should be fine since we don't usually flush redis
+        // and if we do miss an entry that we shouldn't, we just ping it again
+        let all_project_ids = all_server_projects
+            .iter()
+            .map(|row| ProjectId::from(DBProjectId(row.id)).to_string())
+            .collect::<Vec<_>>();
+
+        let all_server_last_pings = redis
+            .get_many_deserialized_from_json::<exp::minecraft::JavaServerPing>(
+                REDIS_NAMESPACE,
+                &all_project_ids,
+            )
+            .await
+            .wrap_err("failed to fetch server project last pings")?;
+
+        let now = Utc::now();
+        let projects_to_ping = all_server_projects
+            .into_iter()
+            .zip(all_server_last_pings)
+            // only include projects which:
+            // - have not had a ping in redis yet
+            // - OR their last ping was a failure
+            // - OR their last successful ping was more than `SERVER_PING_MIN_INTERVAL_SEC` seconds ago
+            .filter(|(_, ping)| {
+                let Some(ping) = ping else { return true };
+                if ping.data.is_none() {
+                    return true;
+                };
+                ping.when.signed_duration_since(now)
+                    > TimeDelta::seconds(
+                        ENV.SERVER_PING_MIN_INTERVAL_SEC as i64,
+                    )
+            })
+            .filter_map(|(row, _)| {
+                let server = row.components.0.minecraft_java_server?;
+                Some((ProjectId::from(DBProjectId(row.id)), server))
+            })
+            .collect::<Vec<_>>();
+
+        Ok(projects_to_ping)
+    }
+}
+
+async fn ping_server(
+    address: &str,
+    port: u16,
+) -> eyre::Result<exp::minecraft::JavaServerPingData> {
+    let start = Instant::now();
+
+    let task = async move {
+        let conn = async_minecraft_ping::ConnectionConfig::build(address)
+            .with_port(port)
+            .connect()
+            .await
+            .wrap_err("failed to connect to server")?;
+
+        let status = conn
+            .status()
+            .await
+            .wrap_err("failed to get server status")?
+            .status;
+
+        Ok(exp::minecraft::JavaServerPingData {
+            latency: start.elapsed(),
+            version_name: status.version.name,
+            version_protocol: status.version.protocol,
+            description: match status.description {
+                ServerDescription::Plain(text)
+                | ServerDescription::Object { text } => text,
+            },
+            players_online: status.players.online,
+            players_max: status.players.max,
+        })
+    };
+
+    let timeout = Duration::from_millis(ENV.SERVER_PING_TIMEOUT_MS);
+    tokio::time::timeout(timeout, task)
+        .await
+        .wrap_err("server ping timed out")
+        .flatten()
+}
+
+#[derive(Debug, Row, Serialize, Clone)]
+struct ServerPingRecord {
+    recorded: i64,
+    project_id: u64,
+    address: String,
+    port: u16,
+    latency_ms: Option<u32>,
+    description: Option<String>,
+    version_name: Option<String>,
+    version_protocol: Option<u32>,
+    players_online: Option<u32>,
+    players_max: Option<u32>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[actix_rt::test]
+    async fn test_ping_server_success() {
+        let _status = ping_server("mc.hypixel.net", 25565).await.unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn test_ping_server_invalid_address() {
+        _ = ping_server("invalid.invalid", 25565).await.unwrap_err();
+    }
+}

apps/labrinth/src/routes/internal/admin.rs

@@ -9,6 +9,7 @@ use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::search::SearchConfig;
 use crate::util::date::get_current_tenths_of_ms;
+use crate::util::error::Context;
 use crate::util::guards::admin_key_guard;
 use actix_web::{HttpRequest, HttpResponse, patch, post, web};
 use serde::Deserialize;
@@ -156,6 +157,8 @@ pub async fn force_reindex(
 ) -> Result<HttpResponse, ApiError> {
     use crate::search::indexing::index_projects;
     let redis = redis.get_ref();
-    index_projects(pool.as_ref().clone(), redis.clone(), &config).await?;
+    index_projects(pool.as_ref().clone(), redis.clone(), &config)
+        .await
+        .wrap_internal_err("failed to index projects")?;
     Ok(HttpResponse::NoContent().finish())
 }

apps/labrinth/src/routes/mod.rs

@@ -216,9 +216,9 @@ impl ApiError {
                 }
             },
             description: match self {
-                Self::Internal(e) => format!("{e:#?}"),
-                Self::Request(e) => format!("{e:#?}"),
-                Self::Auth(e) => format!("{e:#?}"),
+                Self::Internal(e) => format!("{e:#}"),
+                Self::Request(e) => format!("{e:#}"),
+                Self::Auth(e) => format!("{e:#}"),
                 _ => self.to_string(),
             },
             details: match self {

apps/labrinth/src/routes/v2/projects.rs

@@ -214,7 +214,7 @@ pub async fn project_get(
 ) -> Result<HttpResponse, ApiError> {
     // Convert V2 data to V3 data
     // Call V3 project creation
-    let response = v3::projects::project_get(
+    let project = match v3::projects::project_get_internal(
         req,
         info,
         pool.clone(),
@@ -222,23 +222,21 @@ pub async fn project_get(
         session_queue,
     )
     .await
-    .or_else(v2_reroute::flatten_404_error)?;
+    {
+        Ok(resp) => resp.0,
+        Err(ApiError::NotFound) => return Ok(HttpResponse::NotFound().body("")),
+        Err(err) => return Err(err),
+    };
 
     // Convert response to V2 format
-    match v2_reroute::extract_ok_json::<Project>(response).await {
-        Ok(project) => {
-            let version_item = match project.versions.first() {
-                Some(vid) => {
-                    version_item::DBVersion::get((*vid).into(), &**pool, &redis)
-                        .await?
-                }
-                None => None,
-            };
-            let project = LegacyProject::from(project, version_item);
-            Ok(HttpResponse::Ok().json(project))
+    let version_item = match project.versions.first() {
+        Some(vid) => {
+            version_item::DBVersion::get((*vid).into(), &**pool, &redis).await?
         }
-        Err(response) => Ok(response),
-    }
+        None => None,
+    };
+    let project = LegacyProject::from(project, version_item);
+    Ok(HttpResponse::Ok().json(project))
 }
 
 //checks the validity of a project id or slug
@@ -249,7 +247,7 @@ pub async fn project_get_check(
     redis: web::Data<RedisPool>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns an id only, do not need to convert
-    v3::projects::project_get_check(info, pool, redis)
+    v3::projects::project_get_check_internal(info, pool, redis)
         .await
         .or_else(v2_reroute::flatten_404_error)
 }
@@ -269,7 +267,7 @@ pub async fn dependency_list(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // TODO: tests, probably
-    let response = v3::projects::dependency_list(
+    let response = v3::projects::dependency_list_internal(
         req,
         info,
         pool.clone(),
@@ -512,12 +510,16 @@ pub async fn project_edit(
         moderation_message_body: v2_new_project.moderation_message_body,
         monetization_status: v2_new_project.monetization_status,
         side_types_migration_review_status: None, // Not to be exposed in v2
-        loader_fields: HashMap::new(), // Loader fields are not a thing in v2
+        // None of the below is present in v2
+        loader_fields: HashMap::new(),
+        minecraft_server: None,
+        minecraft_java_server: None,
+        minecraft_bedrock_server: None,
     };
 
     // This returns 204 or failure so we don't need to do anything with it
     let project_id = info.clone().0;
-    let mut response = v3::projects::project_edit(
+    let mut response = v3::projects::project_edit_internal(
         req.clone(),
         info,
         pool.clone(),
@@ -754,7 +756,7 @@ pub async fn project_icon_edit(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::project_icon_edit(
+    v3::projects::project_icon_edit_internal(
         web::Query(v3::projects::Extension { ext: ext.ext }),
         req,
         info,
@@ -778,7 +780,7 @@ pub async fn delete_project_icon(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::delete_project_icon(
+    v3::projects::delete_project_icon_internal(
         req,
         info,
         pool,
@@ -814,7 +816,7 @@ pub async fn add_gallery_item(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::add_gallery_item(
+    v3::projects::add_gallery_item_internal(
         web::Query(v3::projects::Extension { ext: ext.ext }),
         req,
         web::Query(v3::projects::GalleryCreateQuery {
@@ -865,7 +867,7 @@ pub async fn edit_gallery_item(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::edit_gallery_item(
+    v3::projects::edit_gallery_item_internal(
         req,
         web::Query(v3::projects::GalleryEditQuery {
             url: item.url,
@@ -897,7 +899,7 @@ pub async fn delete_gallery_item(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::delete_gallery_item(
+    v3::projects::delete_gallery_item_internal(
         req,
         web::Query(v3::projects::GalleryDeleteQuery { url: item.url }),
         pool,
@@ -919,7 +921,7 @@ pub async fn project_delete(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::project_delete(
+    v3::projects::project_delete_internal(
         req,
         info,
         pool,
@@ -941,7 +943,7 @@ pub async fn project_follow(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::project_follow(req, info, pool, redis, session_queue)
+    v3::projects::project_follow_internal(req, info, pool, redis, session_queue)
         .await
         .or_else(v2_reroute::flatten_404_error)
 }
@@ -955,7 +957,13 @@ pub async fn project_unfollow(
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
-    v3::projects::project_unfollow(req, info, pool, redis, session_queue)
-        .await
-        .or_else(v2_reroute::flatten_404_error)
+    v3::projects::project_unfollow_internal(
+        req,
+        info,
+        pool,
+        redis,
+        session_queue,
+    )
+    .await
+    .or_else(v2_reroute::flatten_404_error)
 }

apps/labrinth/src/routes/v2/teams.rs

@@ -39,7 +39,7 @@ pub async fn team_members_get_project(
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
-    let response = v3::teams::team_members_get_project(
+    let response = v3::teams::team_members_get_project_internal(
         req,
         info,
         pool,

apps/labrinth/src/routes/v2/versions.rs

@@ -104,7 +104,7 @@ pub async fn version_list(
         include_changelog: filters.include_changelog,
     };
 
-    let response = v3::versions::version_list(
+    let response = v3::versions::version_list_internal(
         req,
         info,
         web::Query(filters),
@@ -211,6 +211,7 @@ pub async fn version_get(
     let response =
         v3::versions::version_get_helper(req, id, pool, redis, session_queue)
             .await
+            .map(|b| HttpResponse::Ok().json(b))
             .or_else(v2_reroute::flatten_404_error)?;
     // Convert response to V2 format
     match v2_reroute::extract_ok_json::<Version>(response).await {
@@ -277,7 +278,7 @@ pub async fn version_edit(
     }
 
     // Get the older version to get info from
-    let old_version = v3::versions::version_get_helper(
+    let old_version = match v3::versions::version_get_helper(
         req.clone(),
         (*info).0,
         pool.clone(),
@@ -285,12 +286,19 @@ pub async fn version_edit(
         session_queue.clone(),
     )
     .await
-    .or_else(v2_reroute::flatten_404_error)?;
-    let old_version =
-        match v2_reroute::extract_ok_json::<Version>(old_version).await {
-            Ok(version) => version,
-            Err(response) => return Ok(response),
-        };
+    {
+        Ok(resp) => resp,
+        Err(ApiError::NotFound) => return Ok(HttpResponse::NotFound().body("")),
+        Err(err) => return Err(err),
+    };
+    let old_version = match v2_reroute::extract_ok_json::<Version>(
+        HttpResponse::Ok().json(old_version.0),
+    )
+    .await
+    {
+        Ok(version) => version,
+        Err(response) => return Ok(response),
+    };
 
     // If this has 'mrpack_loaders' as a loader field previously, this is a modpack.
     // Therefore, if we are modifying the 'loader' field in this case,

apps/labrinth/src/routes/v3/collections.rs

@@ -99,7 +99,8 @@ pub async fn collection_create(
         &mut transaction,
         &redis,
     )
-    .await?
+    .await
+    .wrap_internal_err("failed to fetch created projects")?
     .into_iter()
     .map(|x| x.inner.id.into())
     .collect::<Vec<ProjectId>>();

apps/labrinth/src/routes/v3/mod.rs

@@ -36,7 +36,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .configure(images::config)
             .configure(notifications::config)
             .configure(organizations::config)
-            .configure(project_creation::config)
             .configure(projects::config)
             .configure(reports::config)
             .configure(shared_instance_version_creation::config)
@@ -65,6 +64,12 @@ pub fn utoipa_config(
             .wrap(default_cors())
             .configure(payouts::config),
     );
+    cfg.service(
+        utoipa_actix_web::scope("/v3/project")
+            .wrap(default_cors())
+            .configure(projects::utoipa_config)
+            .configure(project_creation::config),
+    );
 }
 
 pub async fn hello_world() -> Result<HttpResponse, ApiError> {

apps/labrinth/src/routes/v3/project_creation.rs

@@ -10,6 +10,7 @@ use crate::database::models::{self, DBUser, image_item};
 use crate::database::redis::RedisPool;
 use crate::file_hosting::{FileHost, FileHostPublicity, FileHostingError};
 use crate::models::error::ApiError;
+use crate::models::exp;
 use crate::models::ids::{ImageId, OrganizationId, ProjectId, VersionId};
 use crate::models::images::{Image, ImageContext};
 use crate::models::pats::Scopes;
@@ -43,8 +44,12 @@ use std::sync::Arc;
 use thiserror::Error;
 use validator::Validate;
 
-pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
-    cfg.service(project_create).service(project_create_with_id);
+mod new;
+
+pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
+    cfg.service(project_create)
+        .service(project_create_with_id)
+        .configure(new::config);
 }
 
 #[derive(Error, Debug)]
@@ -93,6 +98,30 @@ pub enum CreateError {
     LimitReached,
 }
 
+impl From<crate::routes::ApiError> for CreateError {
+    fn from(value: crate::routes::ApiError) -> Self {
+        match value {
+            crate::routes::ApiError::Database(err) => Self::DatabaseError(err),
+            crate::routes::ApiError::SqlxDatabase(err) => {
+                Self::SqlxDatabaseError(err)
+            }
+            crate::routes::ApiError::Authentication(err) => {
+                Self::Unauthorized(err)
+            }
+            crate::routes::ApiError::CustomAuthentication(err) => {
+                Self::CustomAuthenticationError(err)
+            }
+            crate::routes::ApiError::InvalidInput(err)
+            | crate::routes::ApiError::Validation(err) => {
+                Self::InvalidInput(err)
+            }
+            err => Self::DatabaseError(models::DatabaseError::SchemaError(
+                err.to_string(),
+            )),
+        }
+    }
+}
+
 impl actix_web::ResponseError for CreateError {
     fn status_code(&self) -> StatusCode {
         match self {
@@ -262,7 +291,8 @@ pub async fn undo_uploads(
     Ok(())
 }
 
-#[post("/project")]
+#[utoipa::path]
+#[post("")]
 pub async fn project_create(
     req: HttpRequest,
     payload: Multipart,
@@ -327,7 +357,8 @@ pub async fn project_create_internal(
 /// Allows creating a project with a specific ID.
 ///
 /// This is a testing endpoint only accessible behind an admin key.
-#[post("/project/{id}", guard = "admin_key_guard")]
+#[utoipa::path]
+#[post("/{id}", guard = "admin_key_guard")]
 pub async fn project_create_with_id(
     req: HttpRequest,
     mut payload: Multipart,
@@ -870,6 +901,7 @@ async fn project_create_inner(
                 .collect(),
             color: icon_data.and_then(|x| x.2),
             monetization_status: MonetizationStatus::Monetized,
+            components: exp::ProjectSerial::default(),
         };
         let project_builder = project_builder_actual.clone();
 
@@ -992,6 +1024,7 @@ async fn project_create_inner(
             side_types_migration_review_status:
                 SideTypesMigrationReviewStatus::Reviewed,
             fields: HashMap::new(), // Fields instantiate to empty
+            components: exp::ProjectQuery::default(),
         };
 
         Ok(HttpResponse::Ok().json(response))
@@ -1076,6 +1109,7 @@ async fn create_initial_version(
         version_type: version_data.release_channel.to_string(),
         requested_status: None,
         ordering: version_data.ordering,
+        components: exp::VersionSerial::default(),
     };
 
     Ok(version)

apps/labrinth/src/routes/v3/project_creation/new.rs

@@ -0,0 +1,337 @@
+use actix_http::StatusCode;
+use actix_web::{HttpRequest, HttpResponse, ResponseError, put, web};
+use eyre::eyre;
+use rust_decimal::Decimal;
+use serde::{Deserialize, Serialize};
+use validator::Validate;
+
+use crate::{
+    auth::get_user_from_headers,
+    database::{
+        PgPool,
+        models::{
+            self, DBOrganization, DBTeamMember, DBUser,
+            project_item::ProjectBuilder, thread_item::ThreadBuilder,
+            version_item::VersionBuilder,
+        },
+        redis::RedisPool,
+    },
+    models::{
+        exp::{self, ProjectComponentKind, component::ComponentRelationError},
+        ids::ProjectId,
+        pats::Scopes,
+        projects::{
+            MonetizationStatus, ProjectStatus, VersionStatus, VersionType,
+        },
+        teams::{OrganizationPermissions, ProjectPermissions},
+        threads::ThreadType,
+        v3::user_limits::UserLimits,
+    },
+    queue::session::AuthQueue,
+    routes::ApiError,
+    util::{error::Context, validate::validation_errors_to_string},
+};
+
+pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
+    cfg.service(create);
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CreateError {
+    #[error("project limit reached")]
+    LimitReached,
+    #[error("invalid component kinds")]
+    ComponentKinds(ComponentRelationError<ProjectComponentKind>),
+    #[error("failed to validate request: {0}")]
+    Validation(String),
+    #[error("slug collision")]
+    SlugCollision,
+    #[error(transparent)]
+    Api(#[from] ApiError),
+}
+
+impl CreateError {
+    pub fn as_api_error(&self) -> crate::models::error::ApiError<'_> {
+        match self {
+            Self::LimitReached => crate::models::error::ApiError {
+                error: "limit_reached",
+                description: self.to_string(),
+                details: None,
+            },
+            Self::ComponentKinds(err) => crate::models::error::ApiError {
+                error: "component_kinds",
+                description: format!("{self}: {err}"),
+                details: Some(
+                    serde_json::to_value(err)
+                        .expect("should never fail to serialize"),
+                ),
+            },
+            Self::Validation(_) => crate::models::error::ApiError {
+                error: "validation",
+                description: self.to_string(),
+                details: None,
+            },
+            Self::SlugCollision => crate::models::error::ApiError {
+                error: "slug_collision",
+                description: self.to_string(),
+                details: None,
+            },
+            Self::Api(err) => err.as_api_error(),
+        }
+    }
+}
+
+impl ResponseError for CreateError {
+    fn status_code(&self) -> actix_http::StatusCode {
+        match self {
+            Self::LimitReached
+            | Self::ComponentKinds(_)
+            | Self::Validation(_)
+            | Self::SlugCollision => StatusCode::BAD_REQUEST,
+            Self::Api(err) => err.status_code(),
+        }
+    }
+
+    fn error_response(&self) -> HttpResponse {
+        HttpResponse::build(self.status_code()).json(self.as_api_error())
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
+pub struct ProjectCreate {
+    pub base: exp::base::Project,
+    #[serde(flatten)]
+    #[validate(nested)]
+    pub components: exp::ProjectEdit,
+}
+
+/// Creates a new project with the given components.
+///
+/// Components must include `base` ([`exp::base::Project`]), and at least one
+/// other component.
+#[utoipa::path]
+#[put("")]
+pub async fn create(
+    req: HttpRequest,
+    db: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    web::Json(create): web::Json<ProjectCreate>,
+) -> Result<web::Json<ProjectId>, CreateError> {
+    // check that the user can make a project
+    let (_, user) = get_user_from_headers(
+        &req,
+        &**db,
+        &redis,
+        &session_queue,
+        Scopes::PROJECT_CREATE,
+    )
+    .await
+    .map_err(ApiError::from)?;
+
+    let limits = UserLimits::get_for_projects(&user, &db)
+        .await
+        .map_err(ApiError::from)?;
+    if limits.current >= limits.max {
+        return Err(CreateError::LimitReached);
+    }
+
+    // check if the given details are valid
+
+    create.validate().map_err(|err| {
+        CreateError::Validation(validation_errors_to_string(err, None))
+    })?;
+
+    let ProjectCreate { base, components } = create;
+
+    exp::component::kinds_valid(
+        &components.component_kinds(),
+        &exp::PROJECT_COMPONENT_RELATIONS,
+    )
+    .map_err(CreateError::ComponentKinds)?;
+
+    // get component-specific data
+    // use struct destructor syntax, so we get a compile error
+    // if we add a new field and don't add it here
+    let exp::base::Project {
+        name,
+        slug,
+        summary,
+        description,
+        requested_status,
+        organization_id,
+    } = base;
+
+    // check if this won't conflict with an existing project
+
+    let mut txn = db
+        .begin()
+        .await
+        .wrap_internal_err("failed to begin transaction")?;
+
+    let same_slug_record = sqlx::query!(
+        "SELECT EXISTS(
+            SELECT 1 FROM mods WHERE slug = $1 OR text_id_lower = $1
+        )",
+        slug.to_lowercase()
+    )
+    .fetch_one(&mut txn)
+    .await
+    .wrap_internal_err("failed to query if slug already exists")?;
+
+    if same_slug_record.exists.unwrap_or(false) {
+        return Err(CreateError::SlugCollision);
+    }
+
+    // create project and supporting records in db
+
+    let team = if let Some(organization_id) = organization_id {
+        let org = DBOrganization::get_id(organization_id.into(), &**db, &redis)
+            .await
+            .wrap_internal_err("failed to get organization")?
+            .wrap_request_err("invalid organization ID")?;
+
+        let team_member =
+            DBTeamMember::get_from_user_id(org.team_id, user.id.into(), &**db)
+                .await
+                .wrap_internal_err(
+                    "failed to get team member of user for organization",
+                )?;
+
+        let perms = OrganizationPermissions::get_permissions_by_role(
+            &user.role,
+            &team_member,
+        );
+
+        if !perms
+            .is_some_and(|p| p.contains(OrganizationPermissions::ADD_PROJECT))
+        {
+            return Err(ApiError::Auth(eyre!(
+                "no permission to create projects in this organization"
+            ))
+            .into());
+        }
+
+        models::team_item::TeamBuilder {
+            members: Vec::new(),
+        }
+    } else {
+        let members = vec![models::team_item::TeamMemberBuilder {
+            user_id: user.id.into(),
+            role: crate::models::teams::DEFAULT_ROLE.to_owned(),
+            is_owner: true,
+            permissions: ProjectPermissions::all(),
+            organization_permissions: None,
+            accepted: true,
+            payouts_split: Decimal::ONE_HUNDRED,
+            ordering: 0,
+        }];
+
+        models::team_item::TeamBuilder { members }
+    };
+    let team_id = team
+        .insert(&mut txn)
+        .await
+        .wrap_internal_err("failed to insert team")?;
+
+    let project_id: ProjectId = models::generate_project_id(&mut txn)
+        .await
+        .wrap_internal_err("failed to generate project ID")?
+        .into();
+
+    // TODO: special casing certain components
+    let mut monetization_status = MonetizationStatus::Monetized;
+    let mut version_builder = None::<VersionBuilder>;
+
+    if components.minecraft_server.is_some() {
+        // servers are not part of the monetization pool;
+        // they generate no payouts for their owners
+        monetization_status = MonetizationStatus::ForceDemonetized;
+
+        // servers may never have a version added, if e.g. they are a vanilla server
+        // but we need at least 1 version on this project for certain features,
+        // like search indexing, to work.
+        // so we generate a synthetic initial version.
+        let version_id = models::generate_version_id(&mut txn)
+            .await
+            .wrap_internal_err("failed to generate project ID")?;
+        version_builder = Some(VersionBuilder {
+            version_id,
+            project_id: project_id.into(),
+            author_id: user.id.into(),
+            name: "__synthetic".into(),
+            version_number: String::new(),
+            changelog: String::new(),
+            files: Vec::new(),
+            dependencies: Vec::new(),
+            loaders: Vec::new(),
+            version_fields: Vec::new(),
+            version_type: VersionType::Release.to_string(),
+            featured: false,
+            status: VersionStatus::Listed,
+            requested_status: None,
+            ordering: None,
+            components: exp::VersionSerial::default(),
+        });
+    }
+
+    let project_builder = ProjectBuilder {
+        project_id: project_id.into(),
+        team_id,
+        organization_id: organization_id.map(From::from),
+        name: name.clone(),
+        summary: summary.clone(),
+        description: description.clone(),
+        icon_url: None,
+        raw_icon_url: None,
+        license_url: None,
+        categories: vec![],
+        additional_categories: vec![],
+        initial_versions: vec![],
+        status: ProjectStatus::Draft,
+        requested_status: Some(requested_status),
+        license: "LicenseRef-Unknown".into(),
+        slug: Some(slug.clone()),
+        link_urls: vec![],
+        gallery_items: vec![],
+        color: None,
+        monetization_status,
+        components: components
+            .create()
+            .wrap_request_err("failed to create components")?,
+    };
+
+    project_builder
+        .insert(&mut txn)
+        .await
+        .wrap_internal_err("failed to insert project")?;
+
+    if let Some(version_builder) = version_builder {
+        version_builder
+            .insert(&mut txn)
+            .await
+            .wrap_internal_err("failed to insert initial version")?;
+    }
+
+    DBUser::clear_project_cache(&[user.id.into()], &redis)
+        .await
+        .wrap_internal_err("failed to clear user project cache")?;
+
+    ThreadBuilder {
+        type_: ThreadType::Project,
+        members: vec![],
+        project_id: Some(project_id.into()),
+        report_id: None,
+    }
+    .insert(&mut txn)
+    .await
+    .wrap_internal_err("failed to insert thread")?;
+
+    // and commit!
+
+    txn.commit()
+        .await
+        .wrap_internal_err("failed to commit transaction")?;
+
+    Ok(web::Json(project_id))
+}

apps/labrinth/src/routes/v3/projects.rs

@@ -1,3 +1,4 @@
+use std::any::type_name;
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -7,14 +8,13 @@ use crate::database::models::notification_item::NotificationBuilder;
 use crate::database::models::project_item::{DBGalleryItem, DBModCategory};
 use crate::database::models::thread_item::ThreadMessageBuilder;
 use crate::database::models::{
-    DBModerationLock, DBTeamMember, ids as db_ids, image_item,
+    DBModerationLock, DBProjectId, DBTeamMember, ids as db_ids, image_item,
 };
 use crate::database::redis::RedisPool;
 use crate::database::{self, models as db_models};
 use crate::database::{PgPool, PgTransaction};
 use crate::env::ENV;
 use crate::file_hosting::{FileHost, FileHostPublicity};
-use crate::models;
 use crate::models::ids::{ProjectId, VersionId};
 use crate::models::images::ImageContext;
 use crate::models::notifications::NotificationBody;
@@ -25,6 +25,7 @@ use crate::models::projects::{
 };
 use crate::models::teams::ProjectPermissions;
 use crate::models::threads::MessageBody;
+use crate::models::{self, exp};
 use crate::queue::moderation::AutomatedModerationQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
@@ -36,7 +37,7 @@ use crate::util::img;
 use crate::util::img::{delete_old_images, upload_image_optimized};
 use crate::util::routes::read_limited_from_payload;
 use crate::util::validate::validation_errors_to_string;
-use actix_web::{HttpRequest, HttpResponse, web};
+use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
 use chrono::Utc;
 use eyre::eyre;
 use futures::TryStreamExt;
@@ -50,38 +51,27 @@ pub fn config(cfg: &mut web::ServiceConfig) {
     cfg.route("projects", web::get().to(projects_get));
     cfg.route("projects", web::patch().to(projects_edit));
     cfg.route("projects_random", web::get().to(random_projects_get));
+}
 
-    cfg.service(
-        web::scope("project")
-            .route("{id}", web::get().to(project_get))
-            .route("{id}/check", web::get().to(project_get_check))
-            .route("{id}", web::delete().to(project_delete))
-            .route("{id}", web::patch().to(project_edit))
-            .route("{id}/icon", web::patch().to(project_icon_edit))
-            .route("{id}/icon", web::delete().to(delete_project_icon))
-            .route("{id}/gallery", web::post().to(add_gallery_item))
-            .route("{id}/gallery", web::patch().to(edit_gallery_item))
-            .route("{id}/gallery", web::delete().to(delete_gallery_item))
-            .route("{id}/follow", web::post().to(project_follow))
-            .route("{id}/follow", web::delete().to(project_unfollow))
-            .route("{id}/organization", web::get().to(project_get_organization))
-            .service(
-                web::scope("{project_id}")
-                    .route(
-                        "members",
-                        web::get().to(super::teams::team_members_get_project),
-                    )
-                    .route(
-                        "version",
-                        web::get().to(super::versions::version_list),
-                    )
-                    .route(
-                        "version/{slug}",
-                        web::get().to(super::versions::version_project_get),
-                    )
-                    .route("dependencies", web::get().to(dependency_list)),
-            ),
-    );
+pub fn utoipa_config(
+    cfg: &mut utoipa_actix_web::service_config::ServiceConfig,
+) {
+    cfg.service(project_get)
+        .service(project_get_check)
+        .service(project_delete)
+        .service(project_edit)
+        .service(project_icon_edit)
+        .service(delete_project_icon)
+        .service(add_gallery_item)
+        .service(edit_gallery_item)
+        .service(delete_gallery_item)
+        .service(project_follow)
+        .service(project_unfollow)
+        .service(project_get_organization)
+        .service(super::teams::team_members_get_project)
+        .service(super::versions::version_list)
+        .service(super::versions::version_project_get)
+        .service(dependency_list);
 }
 
 #[derive(Deserialize, Validate)]
@@ -165,17 +155,30 @@ pub async fn projects_get(
     Ok(HttpResponse::Ok().json(projects))
 }
 
-pub async fn project_get(
+#[utoipa::path]
+#[get("/{id}")]
+async fn project_get(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-) -> Result<HttpResponse, ApiError> {
-    let string = info.into_inner().0;
+) -> Result<web::Json<Project>, ApiError> {
+    project_get_internal(req, info, pool, redis, session_queue).await
+}
 
-    let project_data =
-        db_models::DBProject::get(&string, &**pool, &redis).await?;
+pub async fn project_get_internal(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<web::Json<Project>, ApiError> {
+    let (string,) = info.into_inner();
+
+    let project_data = db_models::DBProject::get(&string, &**pool, &redis)
+        .await
+        .wrap_internal_err("failed to fetch project")?;
     let user_option = get_user_from_headers(
         &req,
         &**pool,
@@ -184,18 +187,20 @@ pub async fn project_get(
         Scopes::PROJECT_READ,
     )
     .await
-    .map(|x| x.1)
+    .map(|(_, user)| user)
     .ok();
 
     if let Some(data) = project_data
-        && is_visible_project(&data.inner, &user_option, &pool, false).await?
+        && is_visible_project(&data.inner, &user_option, &pool, false)
+            .await
+            .wrap_internal_err("failed to check project visibility")?
     {
-        return Ok(HttpResponse::Ok().json(Project::from(data)));
+        return Ok(web::Json(Project::from(data)));
     }
     Err(ApiError::NotFound)
 }
 
-#[derive(Serialize, Deserialize, Validate)]
+#[derive(Debug, Serialize, Deserialize, Validate, utoipa::ToSchema)]
 pub struct EditProject {
     #[validate(
         length(min = 3, max = 64),
@@ -257,15 +262,61 @@ pub struct EditProject {
         Option<SideTypesMigrationReviewStatus>,
     #[serde(flatten)]
     pub loader_fields: HashMap<String, serde_json::Value>,
+
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        with = "serde_with::rust::double_option"
+    )]
+    pub minecraft_server: Option<Option<exp::minecraft::ServerProjectEdit>>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        with = "serde_with::rust::double_option"
+    )]
+    pub minecraft_java_server:
+        Option<Option<exp::minecraft::JavaServerProjectEdit>>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        with = "serde_with::rust::double_option"
+    )]
+    pub minecraft_bedrock_server:
+        Option<Option<exp::minecraft::BedrockServerProjectEdit>>,
 }
 
 #[allow(clippy::too_many_arguments)]
-pub async fn project_edit(
+#[utoipa::path]
+#[patch("/{id}")]
+async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     search_config: web::Data<SearchConfig>,
-    new_project: web::Json<EditProject>,
+    web::Json(new_project): web::Json<EditProject>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    moderation_queue: web::Data<AutomatedModerationQueue>,
+) -> Result<HttpResponse, ApiError> {
+    project_edit_internal(
+        req,
+        info,
+        pool,
+        search_config,
+        web::Json(new_project),
+        redis,
+        session_queue,
+        moderation_queue,
+    )
+    .await
+}
+
+pub async fn project_edit_internal(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    search_config: web::Data<SearchConfig>,
+    web::Json(new_project): web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
     moderation_queue: web::Data<AutomatedModerationQueue>,
@@ -284,7 +335,7 @@ pub async fn project_edit(
         ApiError::Validation(validation_errors_to_string(err, None))
     })?;
 
-    let Some(project_item) =
+    let Some(mut project_item) =
         db_models::DBProject::get(&info.into_inner().0, &**pool, &redis)
             .await?
     else {
@@ -939,6 +990,95 @@ pub async fn project_edit(
         }
     }
 
+    // components
+
+    async fn update<E: exp::component::ComponentEdit>(
+        _txn: &mut PgTransaction<'_>,
+        _project_id: DBProjectId,
+        edit: Option<Option<E>>,
+        mut component: &mut Option<E::Component>,
+    ) -> Result<(), ApiError> {
+        let Some(edit) = edit else {
+            // component is not specified in the input JSON - leave alone
+            return Ok(());
+        };
+
+        match (&mut component, edit) {
+            (None, None) => {}
+            (Some(_), None) => {
+                // component is `null` in the input JSON - remove component
+                *component = None;
+            }
+            (None, Some(edit)) => {
+                // component is specified in the JSON and is non-null - create new component
+                *component =
+                    Some(edit.create().wrap_request_err_with(|| {
+                        eyre!(
+                            "failed to create `{}` component",
+                            type_name::<E::Component>()
+                        )
+                    })?);
+            }
+            (Some(component), Some(edit)) => {
+                // edit component
+                edit.apply_to(component).await.wrap_internal_err_with(
+                    || {
+                        eyre!(
+                            "failed to update `{}` component",
+                            type_name::<E::Component>()
+                        )
+                    },
+                )?;
+            }
+        }
+
+        Ok(())
+    }
+
+    update(
+        &mut transaction,
+        id,
+        new_project.minecraft_server,
+        &mut project_item.inner.components.minecraft_server,
+    )
+    .await?;
+    update(
+        &mut transaction,
+        id,
+        new_project.minecraft_java_server,
+        &mut project_item.inner.components.minecraft_java_server,
+    )
+    .await?;
+    update(
+        &mut transaction,
+        id,
+        new_project.minecraft_bedrock_server,
+        &mut project_item.inner.components.minecraft_bedrock_server,
+    )
+    .await?;
+
+    let components_serial = project_item.inner.components.clone();
+
+    exp::component::kinds_valid(
+        &components_serial.component_kinds(),
+        &exp::PROJECT_COMPONENT_RELATIONS,
+    )
+    .wrap_request_err("invalid component kinds")?;
+
+    sqlx::query!(
+        "
+        UPDATE mods
+        SET components = $1
+        WHERE id = $2
+        ",
+        serde_json::to_value(&components_serial)
+            .expect("serialization shouldn't fail"),
+        id as db_ids::DBProjectId,
+    )
+    .execute(&mut transaction)
+    .await
+    .wrap_internal_err("failed to update components")?;
+
     // check new description and body for links to associated images
     // if they no longer exist in the description or body, delete them
     let checkable_strings: Vec<&str> =
@@ -1059,7 +1199,17 @@ pub async fn project_search(
 }
 
 //checks the validity of a project id or slug
-pub async fn project_get_check(
+#[utoipa::path]
+#[get("/{id}/check")]
+async fn project_get_check(
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+) -> Result<HttpResponse, ApiError> {
+    project_get_check_internal(info, pool, redis).await
+}
+
+pub async fn project_get_check_internal(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
@@ -1084,12 +1234,24 @@ pub struct DependencyInfo {
     pub versions: Vec<models::projects::Version>,
 }
 
+#[utoipa::path]
+#[get("/{project_id}/dependencies")]
 pub async fn dependency_list(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    dependency_list_internal(req, info, pool, redis, session_queue).await
+}
+
+pub async fn dependency_list_internal(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     let string = info.into_inner().0;
 
@@ -1142,7 +1304,11 @@ pub async fn dependency_list(
             .collect::<Vec<db_models::DBVersionId>>();
         let (projects_result, versions_result) = futures::future::try_join(
             database::DBProject::get_many_ids(&project_ids, &**pool, &redis),
-            database::DBVersion::get_many(&dep_version_ids, &**pool, &redis),
+            async {
+                database::DBVersion::get_many(&dep_version_ids, &**pool, &redis)
+                    .await
+                    .wrap_internal_err("failed to fetch dependency versions")
+            },
         )
         .await?;
 
@@ -1494,7 +1660,32 @@ pub struct Extension {
 }
 
 #[allow(clippy::too_many_arguments)]
-pub async fn project_icon_edit(
+#[utoipa::path]
+#[patch("/{id}/icon")]
+async fn project_icon_edit(
+    web::Query(ext): web::Query<Extension>,
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    file_host: web::Data<Arc<dyn FileHost + Send + Sync>>,
+    payload: web::Payload,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    project_icon_edit_internal(
+        web::Query(ext),
+        req,
+        info,
+        pool,
+        redis,
+        file_host,
+        payload,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn project_icon_edit_internal(
     web::Query(ext): web::Query<Extension>,
     req: HttpRequest,
     info: web::Path<(String,)>,
@@ -1609,7 +1800,28 @@ pub async fn project_icon_edit(
     Ok(HttpResponse::NoContent().body(""))
 }
 
-pub async fn delete_project_icon(
+#[utoipa::path]
+#[delete("/{id}/icon")]
+async fn delete_project_icon(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    file_host: web::Data<Arc<dyn FileHost + Send + Sync>>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    delete_project_icon_internal(
+        req,
+        info,
+        pool,
+        redis,
+        file_host,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn delete_project_icon_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
@@ -1710,7 +1922,34 @@ pub struct GalleryCreateQuery {
 }
 
 #[allow(clippy::too_many_arguments)]
+#[utoipa::path]
+#[post("/{id}/gallery")]
 pub async fn add_gallery_item(
+    web::Query(ext): web::Query<Extension>,
+    req: HttpRequest,
+    web::Query(item): web::Query<GalleryCreateQuery>,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    file_host: web::Data<Arc<dyn FileHost + Send + Sync>>,
+    payload: web::Payload,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    add_gallery_item_internal(
+        web::Query(ext),
+        req,
+        web::Query(item),
+        info,
+        pool,
+        redis,
+        file_host,
+        payload,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn add_gallery_item_internal(
     web::Query(ext): web::Query<Extension>,
     req: HttpRequest,
     web::Query(item): web::Query<GalleryCreateQuery>,
@@ -1877,7 +2116,26 @@ pub struct GalleryEditQuery {
     pub ordering: Option<i64>,
 }
 
-pub async fn edit_gallery_item(
+#[utoipa::path]
+#[patch("/{id}/gallery")]
+async fn edit_gallery_item(
+    req: HttpRequest,
+    web::Query(item): web::Query<GalleryEditQuery>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    edit_gallery_item_internal(
+        req,
+        web::Query(item),
+        pool,
+        redis,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn edit_gallery_item_internal(
     req: HttpRequest,
     web::Query(item): web::Query<GalleryEditQuery>,
     pool: web::Data<PgPool>,
@@ -2043,7 +2301,28 @@ pub struct GalleryDeleteQuery {
     pub url: String,
 }
 
-pub async fn delete_gallery_item(
+#[utoipa::path]
+#[delete("/{id}/gallery")]
+async fn delete_gallery_item(
+    req: HttpRequest,
+    web::Query(item): web::Query<GalleryDeleteQuery>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    file_host: web::Data<Arc<dyn FileHost + Send + Sync>>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    delete_gallery_item_internal(
+        req,
+        web::Query(item),
+        pool,
+        redis,
+        file_host,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn delete_gallery_item_internal(
     req: HttpRequest,
     web::Query(item): web::Query<GalleryDeleteQuery>,
     pool: web::Data<PgPool>,
@@ -2153,7 +2432,28 @@ pub async fn delete_gallery_item(
     Ok(HttpResponse::NoContent().body(""))
 }
 
-pub async fn project_delete(
+#[utoipa::path]
+#[delete("/{id}")]
+async fn project_delete(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    search_config: web::Data<SearchConfig>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<(), ApiError> {
+    project_delete_internal(
+        req,
+        info,
+        pool,
+        redis,
+        search_config,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn project_delete_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
@@ -2288,7 +2588,19 @@ pub async fn project_delete(
     }
 }
 
-pub async fn project_follow(
+#[utoipa::path]
+#[post("/{id}/follow")]
+async fn project_follow(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    project_follow_internal(req, info, pool, redis, session_queue).await
+}
+
+pub async fn project_follow_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
@@ -2368,7 +2680,19 @@ pub async fn project_follow(
     }
 }
 
-pub async fn project_unfollow(
+#[utoipa::path]
+#[delete("/{id}/follow")]
+async fn project_unfollow(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    project_unfollow_internal(req, info, pool, redis, session_queue).await
+}
+
+pub async fn project_unfollow_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
@@ -2444,6 +2768,8 @@ pub async fn project_unfollow(
     }
 }
 
+#[utoipa::path]
+#[get("/{id}/organization")]
 pub async fn project_get_organization(
     req: HttpRequest,
     info: web::Path<(String,)>,

apps/labrinth/src/routes/v3/teams.rs

@@ -12,7 +12,7 @@ use crate::models::pats::Scopes;
 use crate::models::teams::{OrganizationPermissions, ProjectPermissions};
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
-use actix_web::{HttpRequest, HttpResponse, web};
+use actix_web::{HttpRequest, HttpResponse, get, web};
 use ariadne::ids::UserId;
 use eyre::eyre;
 use rust_decimal::Decimal;
@@ -40,7 +40,20 @@ pub fn config(cfg: &mut web::ServiceConfig) {
 // also the members of the organization's team if the project is associated with an organization
 // (Unlike team_members_get_project, which only returns the members of the project's team)
 // They can be differentiated by the "organization_permissions" field being null or not
-pub async fn team_members_get_project(
+#[utoipa::path]
+#[get("/{project_id}/members")]
+async fn team_members_get_project(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    team_members_get_project_internal(req, info, pool, redis, session_queue)
+        .await
+}
+
+pub async fn team_members_get_project_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,

apps/labrinth/src/routes/v3/version_creation.rs

@@ -13,6 +13,7 @@ use crate::database::models::{self, DBOrganization, image_item};
 use crate::database::redis::RedisPool;
 use crate::env::ENV;
 use crate::file_hosting::{FileHost, FileHostPublicity};
+use crate::models::exp;
 use crate::models::ids::{ImageId, ProjectId, VersionId};
 use crate::models::images::{Image, ImageContext};
 use crate::models::notifications::NotificationBody;
@@ -325,6 +326,7 @@ async fn version_create_inner(
                     status: version_create_data.status,
                     requested_status: None,
                     ordering: version_create_data.ordering,
+                    components: exp::VersionSerial::default(),
                 });
 
                 return Ok(());
@@ -474,6 +476,7 @@ async fn version_create_inner(
         dependencies: version_data.dependencies,
         loaders: version_data.loaders,
         fields: version_data.fields,
+        components: exp::VersionQuery::default(),
     };
 
     let project_id = builder.project_id;

apps/labrinth/src/routes/v3/versions.rs

@@ -31,7 +31,7 @@ use crate::search::indexing::remove_documents;
 use crate::util::error::Context;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
-use actix_web::{HttpRequest, HttpResponse, web};
+use actix_web::{HttpRequest, HttpResponse, get, web};
 use ariadne::ids::base62_impl::parse_base62;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
@@ -57,6 +57,8 @@ pub fn config(cfg: &mut web::ServiceConfig) {
 }
 
 // Given a project ID/slug and a version slug
+#[utoipa::path]
+#[get("/{project_id}/version/{slug}")]
 pub async fn version_project_get(
     req: HttpRequest,
     info: web::Path<(String, String)>,
@@ -177,7 +179,7 @@ pub async fn version_get(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-) -> Result<HttpResponse, ApiError> {
+) -> Result<web::Json<models::projects::Version>, ApiError> {
     let id = info.into_inner().0;
     version_get_helper(req, id, pool, redis, session_queue).await
 }
@@ -188,7 +190,7 @@ pub async fn version_get_helper(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-) -> Result<HttpResponse, ApiError> {
+) -> Result<web::Json<models::projects::Version>, ApiError> {
     let version_data =
         database::models::DBVersion::get(id.into(), &**pool, &redis).await?;
 
@@ -206,9 +208,7 @@ pub async fn version_get_helper(
     if let Some(data) = version_data
         && is_visible_version(&data.inner, &user_option, &pool, &redis).await?
     {
-        return Ok(
-            HttpResponse::Ok().json(models::projects::Version::from(data))
-        );
+        return Ok(web::Json(models::projects::Version::from(data)));
     }
 
     Err(ApiError::NotFound)
@@ -733,7 +733,28 @@ pub struct VersionListFilters {
     pub include_changelog: bool,
 }
 
-pub async fn version_list(
+#[utoipa::path]
+#[get("/{project_id}/version")]
+async fn version_list(
+    req: HttpRequest,
+    info: web::Path<(String,)>,
+    web::Query(filters): web::Query<VersionListFilters>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    version_list_internal(
+        req,
+        info,
+        web::Query(filters),
+        pool,
+        redis,
+        session_queue,
+    )
+    .await
+}
+
+pub async fn version_list_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     web::Query(filters): web::Query<VersionListFilters>,

apps/labrinth/src/search/indexing/local_import.rs

@@ -15,13 +15,18 @@ use crate::database::models::{
     DBProjectId, DBVersionId, LoaderFieldEnumId, LoaderFieldEnumValueId,
     LoaderFieldId,
 };
+use crate::database::redis::RedisPool;
+use crate::models::exp;
+use crate::models::ids::ProjectId;
 use crate::models::projects::from_duplicate_version_fields;
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
 use crate::search::UploadSearchProject;
+use crate::util::error::Context;
 
 pub async fn index_local(
     pool: &PgPool,
+    redis: &RedisPool,
     cursor: i64,
     limit: i64,
 ) -> Result<(Vec<UploadSearchProject>, i64), IndexingError> {
@@ -40,18 +45,20 @@ pub async fn index_local(
         slug: Option<String>,
         color: Option<i32>,
         license: String,
+        components: exp::ProjectSerial,
     }
 
     let db_projects = sqlx::query!(
-        "
+        r#"
         SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,
-        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color
+        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color,
+        m.components AS "components: sqlx::types::Json<exp::ProjectSerial>"
         FROM mods m
         WHERE m.status = ANY($1) AND m.id > $3
         GROUP BY m.id
         ORDER BY m.id ASC
         LIMIT $2;
-        ",
+        "#,
         &*crate::models::projects::ProjectStatus::iterator()
         .filter(|x| x.is_searchable())
         .map(|x| x.to_string())
@@ -73,12 +80,21 @@ pub async fn index_local(
                 slug: m.slug,
                 color: m.color,
                 license: m.license,
+                components: m.components.0,
             }
         })
         .try_collect::<Vec<PartialProject>>()
         .await?;
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
+    let project_components = db_projects
+        .iter()
+        .map(|project| (ProjectId::from(project.id), &project.components))
+        .collect::<Vec<_>>();
+    let project_query_context =
+        exp::project::fetch_query_context(&project_components, pool, redis)
+            .await
+            .wrap_err("failed to fetch query context")?;
 
     let Some(largest) = project_ids.iter().max() else {
         return Ok((vec![], i64::MAX));
@@ -336,7 +352,12 @@ pub async fn index_local(
                     .collect();
                 let mut loader_fields =
                     from_duplicate_version_fields(version_fields);
-                let project_types = version.project_types;
+                let mut project_types = version.project_types;
+
+                exp::compat::correct_project_types(
+                    &project.components,
+                    &mut project_types,
+                );
 
                 let mut version_loaders = version.loaders;
 
@@ -389,6 +410,15 @@ pub async fn index_local(
                         .insert("server_side".to_string(), vec![server_side]);
                 }
 
+                let components = project
+                    .components
+                    .clone()
+                    .into_query(
+                        ProjectId::from(project.id),
+                        &project_query_context,
+                    )
+                    .wrap_err("failed to populate query components")?;
+
                 let usp = UploadSearchProject {
                     version_id: crate::models::ids::VersionId::from(version.id)
                         .to_string(),
@@ -418,6 +448,7 @@ pub async fn index_local(
                     project_loader_fields: project_loader_fields.clone(),
                     // 'loaders' is aggregate of all versions' loaders
                     loaders: project_loaders.clone(),
+                    components,
                 };
 
                 uploads.push(usp);

apps/labrinth/src/search/indexing/mod.rs

@@ -22,6 +22,8 @@ use tracing::{Instrument, error, info, info_span, instrument};
 
 #[derive(Error, Debug)]
 pub enum IndexingError {
+    #[error(transparent)]
+    Internal(#[from] eyre::Report),
     #[error("Error while connecting to the MeiliSearch database")]
     Indexing(#[from] meilisearch_sdk::errors::Error),
     #[error("Error while serializing or deserializing JSON: {0}")]
@@ -107,31 +109,38 @@ pub async fn index_projects(
     ro_pool: PgPool,
     redis: RedisPool,
     config: &SearchConfig,
-) -> Result<(), IndexingError> {
+) -> eyre::Result<()> {
     info!("Indexing projects.");
 
     info!("Ensuring current indexes exists");
     // First, ensure current index exists (so no error happens- current index should be worst-case empty, not missing)
-    get_indexes_for_indexing(config, false, false).await?;
+    get_indexes_for_indexing(config, false, false)
+        .await
+        .wrap_err("failed to get indexes for indexing")?;
 
     info!("Deleting surplus indexes");
     // Then, delete the next index if it still exists
-    let indices = get_indexes_for_indexing(config, true, false).await?;
+    let indices = get_indexes_for_indexing(config, true, false)
+        .await
+        .wrap_err("failed to get next indexes to delete")?;
     for client_indices in indices {
         for index in client_indices {
-            index.delete().await?;
+            index.delete().await.wrap_err("failed to delete an index")?;
         }
     }
 
     info!("Recreating next index");
     // Recreate the next index for indexing
-    let indices = get_indexes_for_indexing(config, true, true).await?;
+    let indices = get_indexes_for_indexing(config, true, true)
+        .await
+        .wrap_internal_err("failed to recreate next index")?;
 
     let all_loader_fields =
         crate::database::models::loader_fields::LoaderField::get_fields_all(
             &ro_pool, &redis,
         )
-        .await?
+        .await
+        .wrap_internal_err("failed to get all loader fields")?
         .into_iter()
         .map(|x| x.field)
         .collect::<Vec<_>>();
@@ -147,7 +156,7 @@ pub async fn index_projects(
         idx += 1;
 
         let (uploads, next_cursor) =
-            index_local(&ro_pool, cursor, 10000).await?;
+            index_local(&ro_pool, &redis, cursor, 10000).await?;
         total += uploads.len();
 
         if uploads.is_empty() {
@@ -599,6 +608,10 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
     "gallery_items",
     "loaders", // search uses loaders as categories- this is purely for the Project model.
     "project_loader_fields",
+    "minecraft_mod",
+    "minecraft_server",
+    "minecraft_java_server",
+    "minecraft_bedrock_server",
 ];
 
 const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] =

apps/labrinth/src/search/mod.rs

@@ -1,4 +1,5 @@
 use crate::env::ENV;
+use crate::models::exp;
 use crate::models::projects::SearchRequest;
 use crate::{models::error::ApiError, search::indexing::IndexingError};
 use actix_web::HttpResponse;
@@ -206,6 +207,8 @@ pub struct UploadSearchProject {
     pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
     pub project_loader_fields: HashMap<String, Vec<serde_json::Value>>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model.
 
+    #[serde(flatten)]
+    pub components: exp::ProjectQuery,
     #[serde(flatten)]
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
@@ -245,6 +248,8 @@ pub struct ResultSearchProject {
     pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
     pub project_loader_fields: HashMap<String, Vec<serde_json::Value>>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model.
 
+    #[serde(flatten)]
+    pub components: exp::ProjectQuery,
     #[serde(flatten)]
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }

packages/labrinth-derive/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "labrinth-derive"
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+darling = { workspace = true }
+proc-macro2 = { workspace = true }
+quote = { workspace = true }
+syn = { workspace = true }
+
+[lints]
+workspace = true

packages/labrinth-derive/src/component.rs

@@ -0,0 +1,161 @@
+use darling::{FromDeriveInput, FromField};
+use proc_macro2::TokenStream;
+use quote::{format_ident, quote};
+use syn::{Attribute, DeriveInput, Ident, Result, Type, Visibility};
+
+#[derive(Debug, FromDeriveInput)]
+#[darling(supports(struct_named))]
+struct Component {
+    ident: Ident,
+    vis: Visibility,
+    data: darling::ast::Data<(), ComponentField>,
+}
+
+#[derive(Debug, FromField)]
+#[darling(attributes(component), forward_attrs)]
+struct ComponentField {
+    ident: Option<Ident>,
+    vis: Visibility,
+    ty: Type,
+    attrs: Vec<Attribute>,
+    #[darling(default)]
+    synthetic: bool,
+}
+
+pub fn derive(input: &DeriveInput) -> Result<TokenStream> {
+    let Component { ident, vis, data } = Component::from_derive_input(input)?;
+    let fields = data
+        .take_struct()
+        .expect("macro only works on structs with named fields");
+
+    let fields = &fields.fields;
+    let struct_serial = struct_serial(&vis, &ident, fields)?;
+    let struct_edit = struct_edit(&vis, &ident, fields)?;
+
+    Ok(quote! {
+        #struct_serial
+        #struct_edit
+    })
+}
+
+fn struct_serial(
+    vis: &Visibility,
+    ident: &Ident,
+    fields: &[ComponentField],
+) -> Result<TokenStream> {
+    let ident_serial = format_ident!("{ident}Serial");
+
+    let fields = fields
+        .iter()
+        .filter_map(|field| {
+            if field.synthetic {
+                return None;
+            }
+
+            let ident = &field
+                .ident
+                .as_ref()
+                .expect("macro only works on structs with named fields");
+            let vis = &field.vis;
+            let ty = &field.ty;
+            let attrs = &field.attrs;
+
+            Some(quote! {
+                #(#attrs)*
+                #vis #ident: #ty
+            })
+        })
+        .collect::<Vec<_>>();
+
+    Ok(quote! {
+        #[derive(
+            Debug,
+            Clone,
+            ::serde::Serialize,
+            ::serde::Deserialize,
+            ::validator::Validate,
+            ::utoipa::ToSchema,
+        )]
+        #vis struct #ident_serial {
+            #(#fields),*
+        }
+    })
+}
+
+fn struct_edit(
+    vis: &Visibility,
+    ident: &Ident,
+    fields: &[ComponentField],
+) -> Result<TokenStream> {
+    let ident_edit = format_ident!("{ident}Edit");
+
+    let (fields, apply_fields): (Vec<_>, Vec<_>) = fields
+        .iter()
+        .filter_map(|field| {
+            if field.synthetic {
+                return None;
+            }
+
+            let ident = &field
+                .ident
+                .as_ref()
+                .expect("macro only works on structs with named fields");
+            let vis = &field.vis;
+            let ty = &field.ty;
+            let attrs = &field.attrs;
+
+            let serde_attr = if let Type::Path(path) = ty
+                && let Some(root_ident) = path.path.segments.first()
+                && root_ident.ident == "Option"
+            {
+                quote! {
+                    #[serde(
+                        default,
+                        skip_serializing_if = "::core::option::Option::is_none",
+                        with = "::serde_with::rust::double_option"
+                    )]
+                }
+            } else {
+                quote! {
+                    #[serde(default)]
+                }
+            };
+
+            Some((
+                quote! {
+                    #(#attrs)*
+                    #serde_attr
+                    #vis #ident: ::core::option::Option<#ty>
+                },
+                quote! {
+                    if let Some(t) = self.#ident {
+                        component.#ident = t;
+                    }
+                },
+            ))
+        })
+        .unzip();
+
+    Ok(quote! {
+        #[derive(
+            Debug,
+            Clone,
+            ::serde::Serialize,
+            ::serde::Deserialize,
+            ::validator::Validate,
+            ::utoipa::ToSchema,
+        )]
+        #vis struct #ident_edit {
+            #(#fields),*
+        }
+
+        impl #ident_edit {
+            pub fn apply_to(
+                self,
+                component: &mut #ident,
+            ) {
+                #(#apply_fields)*
+            }
+        }
+    })
+}

packages/labrinth-derive/src/lib.rs

@@ -0,0 +1,15 @@
+//! This crate is currently unused, but will replace the `macro_rules!` component
+//! logic in Labrinth experimental API.
+
+use proc_macro::TokenStream;
+use syn::{DeriveInput, parse_macro_input};
+
+mod component;
+
+#[proc_macro_derive(Component, attributes(component))]
+pub fn component(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    component::derive(&input)
+        .unwrap_or_else(|err| err.to_compile_error())
+        .into()
+}

prompt.md

@@ -0,0 +1,11 @@
+read the last commit. basically what we're doing, is we want to store a project's `components` into the search indexing. and when we fetch a project, we fetch its components from the db (the `components` jsonb column), AND each component can fetch some extra stuff for itself. for example, the `minecraft_java_server` project component has a content field which has a modpack id `version_id`. when we query this (either for returning as an API request, or for search indexing), we want to also fetch extra data associated with that version id. BUT, 2 problems:
+- we want 1 logical unified place to store the logic for this extra fetch
+- BUT in the dbproject::get_many fn (where the fetch is done), the function is fucking massive, and there's no easy way to add a "hook" into there for components
+- and I don't want components to have access to the `db: &PgPool` themselves, since then each component does an extra round trip to the db. ideally, we fetch all projects as batch, and any extra data required by components at the same time, then components figure out how to take this extra-fetch data and map it into themselves.
+- this same issue is repeated for search - but if we fix the 2 points above, we can reuse this for search
+
+figure out a solution to this. ideas:
+- have a fn in `exp.rs` which is essentially `DBProject::get_many`, and it defines a place where all components define what they fetch
+- store that extra fetch data as like a `ProjectFetchContext`
+- each component type has a `fn map(&ProjectFetchContext, ProjecetId) -> Result<Self>` which takes some of that fetched context, and gets the appropriate data to make an instance of itself
+- call this fn from DBProject::get_many and from search indexing ( [@local_import.rs](file:///home/boris/Projects/code3/apps/labrinth/src/search/indexing/local_import.rs) )