Incremental search indexing (#6453)

* wip: initial kafka support

* wip: kafka search queue

* wip: kafka consumer

* wip: create/delete docs

* reindexing works

* reindex in more places

* also incrementally update server pings

* cleanup

* tombi

* fix ci

* update tombi

* prepare

* fix ci

* fix

* fix PLEASE

* 🙏

* add route to force reindex project

apps/labrinth/.sqlx/query-0ffebf149d50149d0c128478e6267a2296efc16763cdbd5d8c295a1a7ca08021.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                INSERT INTO users (id, username, email, role)\n                VALUES\n                    (1000, 'userland', 'userland@modrinth.com', 'developer'),\n                    (1001, 'useful', 'useful@modrinth.com', 'developer'),\n                    (1002, 'Useless', 'useless@modrinth.com', 'developer')\n                ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "0ffebf149d50149d0c128478e6267a2296efc16763cdbd5d8c295a1a7ca08021"
+}

apps/labrinth/.sqlx/query-329c009635ad2a4c214da2f4651b8c14fe329deac983bc2f43ba74e066f9cf8d.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mods SET status = 'processing' WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "329c009635ad2a4c214da2f4651b8c14fe329deac983bc2f43ba74e066f9cf8d"
+}

apps/labrinth/.sqlx/query-38ace0abad724d1111b79a72c7c8518f1272690ce7766dbdc41ec1e98f7f1fec.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                INSERT INTO users (id, username, email, role)\n                VALUES (2100, 'prefix_under_score', 'prefix_under_score@modrinth.com', 'developer')\n                ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "38ace0abad724d1111b79a72c7c8518f1272690ce7766dbdc41ec1e98f7f1fec"
+}

apps/labrinth/.sqlx/query-5cb65218a2a4a7343130d838f8a78df08d25844f1019d539712b0cb686e4ae81.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                    INSERT INTO users (id, username, email, role)\n                    VALUES ($1, $2, $3, 'developer')\n                    ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5cb65218a2a4a7343130d838f8a78df08d25844f1019d539712b0cb686e4ae81"
+}

apps/labrinth/.sqlx/query-66868d0b35229abffa3f668eaef19dc0cdef502935dd5939a2f3ab89b0766ad2.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "CREATE SCHEMA public;",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "66868d0b35229abffa3f668eaef19dc0cdef502935dd5939a2f3ab89b0766ad2"
+}

apps/labrinth/.sqlx/query-881de045a10c1debe041ec558fd92fdfbf8b8ffa1b21d3d4a271ab6f9c8adb78.json

@@ -0,0 +1,95 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n\t\tSELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n\t\tm.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color,\n\t\tm.components AS \"components: sqlx::types::Json<exp::ProjectSerial>\"\n\t\tFROM mods m\n\t\tWHERE m.status = ANY($1) AND m.id = ANY($2)\n\t\tGROUP BY m.id\n\t\tORDER BY m.id ASC;\n\t\t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "downloads",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "follows",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 5,
+        "name": "icon_url",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "updated",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 7,
+        "name": "approved",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 8,
+        "name": "published",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 9,
+        "name": "license",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 10,
+        "name": "slug",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "color",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 12,
+        "name": "components: sqlx::types::Json<exp::ProjectSerial>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "TextArray",
+        "Int8Array"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      false,
+      true,
+      false,
+      false,
+      true,
+      true,
+      false
+    ]
+  },
+  "hash": "881de045a10c1debe041ec558fd92fdfbf8b8ffa1b21d3d4a271ab6f9c8adb78"
+}

apps/labrinth/.sqlx/query-8ca7f46618bd72cfd5fe1f246d41a31da1885cc22bb61be6853950ba52be9aa7.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE moderation_locks SET locked_at = NOW() - ($1::bigint * INTERVAL '1 minute') - INTERVAL '1 second' WHERE project_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8ca7f46618bd72cfd5fe1f246d41a31da1885cc22bb61be6853950ba52be9aa7"
+}

apps/labrinth/.sqlx/query-a34294b38639cb96d0c2490773de6c5e7c4bdcbeb85c84d0b23550ddb83c431f.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            INSERT INTO moderation_locks (project_id, moderator_id, locked_at)\n            VALUES ($1, $2, NOW())\n            ON CONFLICT (project_id) DO UPDATE SET\n                moderator_id = EXCLUDED.moderator_id,\n                locked_at = EXCLUDED.locked_at\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a34294b38639cb96d0c2490773de6c5e7c4bdcbeb85c84d0b23550ddb83c431f"
+}

apps/labrinth/.sqlx/query-b7a3ebd72c8685c590298c154fb3da625f34589540efbb08201c0921f96d510b.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                UPDATE versions v\n                SET downloads = v.downloads + x.amount\n                FROM unnest($1::BIGINT[], $2::int[]) AS x(id, amount)\n                WHERE v.id = x.id\n                ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8Array",
+        "Int4Array"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b7a3ebd72c8685c590298c154fb3da625f34589540efbb08201c0921f96d510b"
+}

apps/labrinth/.sqlx/query-c65295f61b18f7be4fdf30b3e2c2c36b9db3770508be4acbb6b441ed8006d9c7.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DROP SCHEMA public CASCADE;",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "c65295f61b18f7be4fdf30b3e2c2c36b9db3770508be4acbb6b441ed8006d9c7"
+}

apps/labrinth/.sqlx/query-e6287177bd5725f587327a9bd5bcce60358fe9401e0477c39c8a910412c3865c.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                    UPDATE mods m\n                    SET downloads = m.downloads + x.amount\n                    FROM unnest($1::BIGINT[], $2::int[]) AS x(id, amount)\n                    WHERE m.id = x.id\n                    ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8Array",
+        "Int4Array"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e6287177bd5725f587327a9bd5bcce60358fe9401e0477c39c8a910412c3865c"
+}

apps/labrinth/.sqlx/query-fabba52b34e14196b6305671c060d934b9afc6dccc5fc77454fbe45365bdf429.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_advisory_unlock(1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "pg_advisory_unlock",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "fabba52b34e14196b6305671c060d934b9afc6dccc5fc77454fbe45365bdf429"
+}