sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-08-27 01:54:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Incremental search indexing (#6453)

Browse Source 
* wip: initial kafka support

* wip: kafka search queue

* wip: kafka consumer

* wip: create/delete docs

* reindexing works

* reindex in more places

* also incrementally update server pings

* cleanup

* tombi

* fix ci

* update tombi

* prepare

* fix ci

* fix

* fix PLEASE

* 🙏

* add route to force reindex project
This commit is contained in:
aecsocket
2026-06-21 10:03:20 +00:00
committed by GitHub
parent be65589e4d
commit c6ef10e798
53 changed files with 1212 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/labrinth/src/queue/analytics/mod.rs
+6 -6
View File
@@ -339,29 +339,29 @@ impl AnalyticsQueue {
downloads.write(&download).await?;
}
sqlx::query(
sqlx::query!(
"
UPDATE versions v
SET downloads = v.downloads + x.amount
FROM unnest($1::BIGINT[], $2::int[]) AS x(id, amount)
WHERE v.id = x.id
",
&version_downloads.keys().copied().collect::<Vec<_>>(),
&version_downloads.values().copied().collect::<Vec<_>>(),
)
.bind(version_downloads.keys().copied().collect::<Vec<_>>())
.bind(version_downloads.values().copied().collect::<Vec<_>>())
.execute(&mut transaction)
.await?;
sqlx::query(
sqlx::query!(
"
UPDATE mods m
SET downloads = m.downloads + x.amount
FROM unnest($1::BIGINT[], $2::int[]) AS x(id, amount)
WHERE m.id = x.id
",
&project_downloads.keys().copied().collect::<Vec<_>>(),
&project_downloads.values().copied().collect::<Vec<_>>(),
)
.bind(project_downloads.keys().copied().collect::<Vec<_>>())
.bind(project_downloads.values().copied().collect::<Vec<_>>())
.execute(&mut transaction)
.await?;
apps/labrinth/src/queue/server_ping.rs
+22 -8
View File
@@ -5,10 +5,12 @@ use crate::env::ENV;
use crate::models::exp;
use crate::models::ids::ProjectId;
use crate::models::projects::ProjectStatus;
use crate::search::incremental::IncrementalSearchQueue;
use crate::{database::PgPool, util::error::Context};
use async_minecraft_ping::ServerDescription;
use chrono::{TimeDelta, Utc};
use clickhouse::{Client, Row};
use futures::future::join;
use serde::Serialize;
use sqlx::types::Json;
use std::sync::Arc;
@@ -21,6 +23,7 @@ pub struct ServerPingQueue {
pub db: PgPool,
pub redis: RedisPool,
pub clickhouse: Client,
pub incremental_search_queue: IncrementalSearchQueue,
}
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping";
@@ -28,11 +31,17 @@ pub const REDIS_FAILURE_NAMESPACE: &str = "minecraft_java_server_ping_failures";
pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings";
impl ServerPingQueue {
pub fn new(db: PgPool, redis: RedisPool, clickhouse: Client) -> Self {
pub fn new(
db: PgPool,
redis: RedisPool,
clickhouse: Client,
incremental_search_queue: IncrementalSearchQueue,
) -> Self {
Self {
db,
redis,
clickhouse,
incremental_search_queue,
}
}
@@ -166,17 +175,22 @@ impl ServerPingQueue {
}
if updated_project {
DBProject::clear_cache(
let clear_cache = DBProject::clear_cache(
(*project_id).into(),
None,
None,
&self.redis,
)
.await
.inspect_err(|err| {
warn!("failed to clear project cache: {err:#}")
})
.ok();
);
let queue_search =
self.incremental_search_queue.push(*project_id);
let (clear_cache_result, _) =
join(clear_cache, queue_search).await;
clear_cache_result
.inspect_err(|err| {
warn!("failed to clear project cache: {err:#}")
})
.ok();
}
}