Merge branch 'main' into fetch/batch-search-indexing

This reverts commit a21bea9654.

apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                SELECT\n                    id,\n                    status AS \"status: PayoutStatus\"\n                FROM payouts\n                ORDER BY id\n                ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "status: PayoutStatus",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false
-    ]
-  },
-  "hash": "1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286"
-}

apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n        FROM mods m\n        WHERE m.status = ANY($1)\n        GROUP BY m.id;\n        ",
+  "query": "\n        SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n        m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n        FROM mods m\n        WHERE m.status = ANY($1) AND m.id > $2\n        GROUP BY m.id\n        LIMIT $3\n        ",
   "describe": {
     "columns": [
       {
@@ -66,7 +66,9 @@
     ],
     "parameters": {
       "Left": [
-        "TextArray"
+        "TextArray",
+        "Int8",
+        "Int8"
       ]
     },
     "nullable": [
@@ -84,5 +86,5 @@
       true
     ]
   },
-  "hash": "b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5"
+  "hash": "2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa"
 }

apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT status AS \"status: PayoutStatus\" FROM payouts WHERE id = 1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "status: PayoutStatus",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3"
-}

apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n                VALUES ($1, $2, $3, $4, $5, 10.0, NOW())\n                ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text",
-        "Text",
-        "Varchar",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02"
-}

apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                    INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n                    VALUES ($1, $2, NULL, $3, $4, 10.00, NOW())\n                    ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text",
-        "Varchar",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606"
-}

apps/labrinth/src/search/indexing/local_import.rs

@@ -20,10 +20,12 @@ use crate::routes::v2_reroute;
 use crate::search::UploadSearchProject;
 use sqlx::postgres::PgPool;
 
-pub async fn index_local(
+pub async fn index_local_chunk(
     pool: &PgPool,
-) -> Result<Vec<UploadSearchProject>, IndexingError> {
-    info!("Indexing local projects!");
+    cursor: i64,
+    limit: i64,
+) -> Result<(Vec<UploadSearchProject>, i64), IndexingError> {
+    info!(cursor = cursor, limit = limit, "Indexing local projects");
 
     // todo: loaders, project type, game versions
     struct PartialProject {
@@ -45,13 +47,16 @@ pub async fn index_local(
         SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,
         m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color
         FROM mods m
-        WHERE m.status = ANY($1)
-        GROUP BY m.id;
+        WHERE m.status = ANY($1) AND m.id > $2
+        GROUP BY m.id
+        LIMIT $3
         ",
         &*crate::models::projects::ProjectStatus::iterator()
         .filter(|x| x.is_searchable())
         .map(|x| x.to_string())
         .collect::<Vec<String>>(),
+        cursor,
+        limit,
     )
         .fetch(pool)
         .map_ok(|m| {
@@ -74,6 +79,10 @@ pub async fn index_local(
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
 
+    let Some(largest) = project_ids.iter().max() else {
+        return Ok((vec![], i64::MAX));
+    };
+
     struct PartialGallery {
         url: String,
         featured: bool,
@@ -412,7 +421,7 @@ pub async fn index_local(
         }
     }
 
-    Ok(uploads)
+    Ok((uploads, *largest))
 }
 
 struct PartialVersion {

apps/labrinth/src/search/indexing/mod.rs

@@ -8,7 +8,7 @@ use crate::search::{SearchConfig, UploadSearchProject};
 use ariadne::ids::base62_impl::to_base62;
 use futures::StreamExt;
 use futures::stream::FuturesOrdered;
-use local_import::index_local;
+use local_import::index_local_chunk;
 use meilisearch_sdk::client::{Client, SwapIndexes};
 use meilisearch_sdk::indexes::Index;
 use meilisearch_sdk::settings::{PaginationSetting, Settings};
@@ -34,8 +34,8 @@ pub enum IndexingError {
 
 // The chunk size for adding projects to the indexing database. If the request size
 // is too large (>10MiB) then the request fails with an error.  This chunk size
-// assumes a max average size of 4KiB per project to avoid this cap.
-const MEILISEARCH_CHUNK_SIZE: usize = 10000000;
+// assumes a max average size of 8KiB per project to avoid this cap.
+const MEILISEARCH_CHUNK_SIZE: usize = 5000000;
 const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
 
 pub async fn remove_documents(
@@ -115,15 +115,35 @@ pub async fn index_projects(
         .map(|x| x.field)
         .collect::<Vec<_>>();
 
-    let uploads = index_local(&pool).await?;
+    let mut cursor = 0;
+    let mut idx = 0;
+    let mut total = 0;
 
-    add_projects_batch_client(
-        &indices,
-        uploads,
-        all_loader_fields.clone(),
-        config,
-    )
-    .await?;
+    loop {
+        info!("Gathering index data chunk {idx}");
+        idx += 1;
+
+        let (uploads, next_cursor) =
+            index_local_chunk(&pool, cursor, 10000).await?;
+        total += uploads.len();
+
+        if uploads.is_empty() {
+            info!(
+                "No more projects to index, indexed {total} projects after {idx} chunks"
+            );
+            break;
+        }
+
+        cursor = next_cursor;
+
+        add_projects_batch_client(
+            &indices,
+            uploads,
+            all_loader_fields.clone(),
+            config,
+        )
+        .await?;
+    }
 
     // Swap the index
     swap_index(config, "projects").await?;