sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-07-31 13:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(labrinth): Redis Cluster (#6771)

Browse Source 
* chore(labrinth): bump to redis 1.4.1

* feat(labrinth): redis cluster

* chore: cleanup

* feat(labrinth): cache locking

* fix(labrinth): clippy

* chore(labrinth): cleanup env, remove postcard support

* chore(ci): fix test env for labrinth

* chore(labrinth): bump all key versions

* chore(labrinth): improve redis key identities handling

* chore(labrinth): simplify deadline handling

* chore(labrinth): remove unused lease tracking

* chore(labrinth): remove distributed cache locking for now

* chore(labrinth): improve redis backend init error

* feat(labrinth): expose redis read replica strategy

* chore(ci): remove other connection mode tests

* chore: split xredis crate

* feat(xredis): primaries routing

* chore: tombi fmt

* chore: clippy

* chore: update query cache
This commit is contained in:
François-Xavier Talbot
2026-07-23 11:35:02 +02:00
committed by GitHub
parent 11af2651ef
commit b4d681e713
146 changed files with 4741 additions and 2000 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/turbo-ci.yml
+7 -1
View File
@@ -82,6 +82,12 @@ jobs:
SCCACHE_DIR: ${{ needs.skip-if-clean.outputs.internal == 'true' && '/mnt/sccache' || '' }}
SCCACHE_CACHE_SIZE: ${{ needs.skip-if-clean.outputs.internal == 'true' && '10G' || '' }}
RUSTC_WRAPPER: ${{ needs.skip-if-clean.outputs.internal == 'true' && 'sccache' || '' }}
# Redis config for tests
REDIS_TOPOLOGY: cluster
REDIS_CONNECTION_TYPE: multiplexed
REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005
# Avoid stack overflows in tests
RUST_MIN_STACK: 16777216
steps:
- name: Check out code
@@ -185,7 +191,7 @@ jobs:
- name: Start services
if: steps.check-labrinth.outputs.needs_services == 'true'
run: docker compose up --wait
run: docker compose --profile clustered-redis up --wait
- name: Setup labrinth environment and database
if: steps.check-labrinth.outputs.needs_services == 'true'
Cargo.lock
Generated
+71 -43
View File
@@ -244,7 +244,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"smallvec",
"socket2 0.6.1",
"socket2 0.6.5",
"time",
"tracing",
"url",
@@ -458,6 +458,12 @@ version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
[[package]]
name = "arcstr"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d"
[[package]]
name = "arg_enum_proc_macro"
version = "0.3.4"
@@ -2478,6 +2484,12 @@ dependencies = [
"spin 0.10.0",
]
[[package]]
name = "crc16"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -2801,7 +2813,7 @@ checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"
[[package]]
name = "deadpool"
version = "0.12.3"
source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde"
source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b"
dependencies = [
"deadpool-runtime",
"num_cpus",
@@ -2811,8 +2823,8 @@ dependencies = [
[[package]]
name = "deadpool-redis"
version = "0.22.1"
source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde"
version = "0.23.0"
source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b"
dependencies = [
"deadpool",
"redis",
@@ -2822,7 +2834,7 @@ dependencies = [
[[package]]
name = "deadpool-runtime"
version = "0.1.5"
source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde"
source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b"
dependencies = [
"tokio",
]
@@ -4794,7 +4806,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.1",
"socket2 0.5.10",
"system-configuration",
"tokio",
"tower-service",
@@ -5135,17 +5147,6 @@ dependencies = [
"syn 2.0.106",
]
[[package]]
name = "io-uring"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
dependencies = [
"bitflags 2.9.4",
"cfg-if",
"libc",
]
[[package]]
name = "ipconfig"
version = "0.3.2"
@@ -5470,7 +5471,6 @@ dependencies = [
"color-thief",
"const_format",
"dashmap",
"deadpool-redis",
"derive_more 2.1.1",
"dotenv-build",
"dotenvy",
@@ -5490,7 +5490,6 @@ dependencies = [
"json-patch 4.1.0",
"labrinth",
"lettre",
"lz4_flex",
"meilisearch-sdk",
"modrinth-content-management",
"modrinth-util",
@@ -5540,6 +5539,7 @@ dependencies = [
"webauthn-rs-proto",
"webp",
"woothee",
"xredis",
"yaserde",
"zip 6.0.0",
"zxcvbn",
@@ -5605,7 +5605,7 @@ dependencies = [
"quoted_printable",
"rustls 0.23.32",
"rustls-native-certs 0.8.1",
"socket2 0.6.1",
"socket2 0.6.5",
"tokio",
"tokio-rustls 0.26.4",
"url",
@@ -5643,9 +5643,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
version = "0.2.177"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libfuzzer-sys"
@@ -6059,14 +6059,14 @@ dependencies = [
[[package]]
name = "mio"
version = "1.0.4"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"log",
"wasi 0.11.1+wasi-snapshot-preview1",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -8014,7 +8014,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls 0.23.32",
"socket2 0.6.1",
"socket2 0.5.10",
"thiserror 2.0.17",
"tokio",
"tracing",
@@ -8051,7 +8051,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.1",
"socket2 0.5.10",
"tracing",
"windows-sys 0.60.2",
]
@@ -8345,26 +8345,32 @@ dependencies = [
[[package]]
name = "redis"
version = "0.32.7"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44"
checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a"
dependencies = [
"ahash 0.8.12",
"arcstr",
"async-lock",
"bytes",
"cfg-if",
"combine",
"crc16",
"futures-util",
"itoa",
"log",
"num-bigint",
"percent-encoding",
"pin-project-lite",
"r2d2",
"rand 0.10.1",
"ryu",
"sha1_smol",
"socket2 0.6.1",
"socket2 0.6.5",
"tokio",
"tokio-util",
"url",
"xxhash-rust",
]
[[package]]
@@ -9817,12 +9823,12 @@ dependencies = [
[[package]]
name = "socket2"
version = "0.6.1"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -11272,30 +11278,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.47.1"
version = "1.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
dependencies = [
"backtrace",
"bytes",
"io-uring",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"slab",
"socket2 0.6.1",
"socket2 0.6.5",
"tokio-macros",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.5.0"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
dependencies = [
"proc-macro2",
"quote",
@@ -11499,7 +11502,7 @@ dependencies = [
"hyper-util",
"percent-encoding",
"pin-project",
"socket2 0.6.1",
"socket2 0.6.5",
"sync_wrapper",
"tokio",
"tokio-stream",
@@ -13429,6 +13432,31 @@ version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xredis"
version = "0.1.0"
dependencies = [
"ariadne",
"chrono",
"dashmap",
"deadpool-redis",
"futures",
"lz4_flex",
"prometheus",
"redis",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tracing",
]
[[package]]
name = "xxhash-rust"
version = "0.8.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72"
[[package]]
name = "yaserde"
version = "0.12.0"
Cargo.toml
+4 -2
View File
@@ -15,6 +15,7 @@ members = [
"packages/modrinth-util",
"packages/neverbounce",
"packages/path-util",
"packages/xredis",
]
[workspace.package]
@@ -72,7 +73,7 @@ daedalus = { path = "packages/daedalus" }
darling = { version = "0.23" }
dashmap = "6.1.0"
data-url = "0.3.2"
deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "db5fb00b036ecc8fe5f18853c559b745ffe47bde", version = "0.22.1" }
deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "c7209b086829572f368f1194aba54bdea8dc540b", version = "0.23.0" }
derive_more = "2.1.1"
directories = "6.0.0"
dirs = "6.0.0"
@@ -154,7 +155,7 @@ quote = { version = "1.0" }
rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9
rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9
rdkafka = { version = "0.36.2", features = ["cmake-build"] }
redis = "0.32.7"
redis = "1.4.1"
regex = "1.12.2"
reqwest = { version = "0.12.24", default-features = false }
rgb = "0.8.52"
@@ -241,6 +242,7 @@ windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62
windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62
winreg = "0.55.0"
woothee = "0.13.0"
xredis = { path = "packages/xredis" }
yaserde = "0.12.0"
zbus = "5.11.0"
zip = { version = "6.0.0", default-features = false, features = [
apps/labrinth/.env.docker-compose
+5
View File
@@ -30,9 +30,14 @@ TYPESENSE_URL=http://localhost:8108
TYPESENSE_API_KEY=modrinth
TYPESENSE_INDEX_PREFIX=labrinth
REDIS_MODE=standalone
REDIS_CONNECTION_TYPE=pooled
# Cache fill coordination is local to each process
REDIS_CACHE_LOCKING_STRATEGY=local
REDIS_URL=redis://labrinth-redis
REDIS_MIN_CONNECTIONS=0
REDIS_MAX_CONNECTIONS=10000
REDIS_BLOCKING_MAX_CONNECTIONS=8
KAFKA_BOOTSTRAP_SERVERS=redpanda:9092
KAFKA_CLIENT_ID=labrinth
apps/labrinth/.env.local
+10
View File
@@ -48,9 +48,19 @@ TYPESENSE_URL=http://localhost:8108
TYPESENSE_API_KEY=modrinth
TYPESENSE_INDEX_PREFIX=labrinth
REDIS_MODE=standalone
REDIS_CONNECTION_TYPE=pooled
# Cache fill coordination is local to each process
REDIS_CACHE_LOCKING_STRATEGY=local
REDIS_URL=redis://localhost
REDIS_MIN_CONNECTIONS=0
REDIS_MAX_CONNECTIONS=10000
REDIS_BLOCKING_MAX_CONNECTIONS=8
# For a clustered Redis setup (`clustered-redis` Docker Compose profile)
# REDIS_MODE=cluster
# REDIS_CONNECTION_TYPE=multiplexed
# REDIS_URL=redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005
KAFKA_BOOTSTRAP_SERVERS=localhost:19092
KAFKA_CLIENT_ID=labrinth
apps/labrinth/.sqlx/query-cb626a36deffd73e67de2dc4789bd875675779a173fb2cd41ba365c1acca96f9.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE users SET email_verified = TRUE WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "cb626a36deffd73e67de2dc4789bd875675779a173fb2cd41ba365c1acca96f9"
}
apps/labrinth/Cargo.toml
+10 -4
View File
@@ -41,7 +41,6 @@ color-eyre = { workspace = true }
color-thief = { workspace = true }
const_format = { workspace = true }
dashmap = { workspace = true }
deadpool-redis.workspace = true
derive_more = { workspace = true, features = ["deref", "deref_mut"] }
dotenvy = { workspace = true }
either = { workspace = true }
@@ -73,7 +72,6 @@ image = { workspace = true, features = [
itertools = { workspace = true }
json-patch = { workspace = true }
lettre = { workspace = true }
lz4_flex = { workspace = true }
meilisearch-sdk = { workspace = true, features = ["reqwest"] }
modrinth-content-management = { workspace = true }
modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] }
@@ -88,7 +86,13 @@ quick-xml = { workspace = true }
rand = { workspace = true }
rand_chacha = { workspace = true }
rdkafka = { workspace = true }
redis = { workspace = true, features = ["ahash", "r2d2", "tokio-comp"] }
redis = { workspace = true, features = [
"ahash",
"cluster",
"cluster-async",
"r2d2",
"tokio-comp"
] }
regex = { workspace = true }
reqwest = { workspace = true, features = [
"http2",
@@ -141,12 +145,14 @@ webauthn-rs = { workspace = true, features = [
webauthn-rs-proto = { workspace = true }
webp = { workspace = true }
woothee = { workspace = true }
xredis = { workspace = true }
yaserde = { workspace = true, features = ["derive"] }
zip = { workspace = true }
zxcvbn = { workspace = true }
[dev-dependencies]
labrinth = { path = ".", features = ["test"] }
tokio = { workspace = true, features = ["test-util"] }
[build-dependencies]
chrono = { workspace = true }
@@ -162,7 +168,7 @@ tikv-jemallocator = { workspace = true, features = [
] }
[features]
test = []
test = ["xredis/test"]
[lints]
workspace = true
apps/labrinth/src/auth/checks.rs
+1 -1
View File
@@ -2,7 +2,6 @@ use crate::database;
use crate::database::models::project_item::ProjectQueryResult;
use crate::database::models::version_item::VersionQueryResult;
use crate::database::models::{DBCollection, DBOrganization, DBTeamMember};
use crate::database::redis::RedisPool;
use crate::database::{DBProject, DBVersion, models};
use crate::database::{PgPool, ReadOnlyPgPool};
use crate::models::ids::FileId;
@@ -14,6 +13,7 @@ use crate::queue::file_scan::get_files_missing_attribution;
use crate::routes::ApiError;
use futures::TryStreamExt;
use itertools::Itertools;
use xredis::RedisPool;
pub fn require_verified_email(user: &User) -> Result<(), ApiError> {
if !user.email_verified.unwrap_or(false) {
apps/labrinth/src/auth/mod.rs
+6
View File
@@ -59,6 +59,12 @@ pub enum AuthenticationError {
Url,
}
impl From<xredis::Error> for AuthenticationError {
fn from(error: xredis::Error) -> Self {
Self::Database(error.into())
}
}
impl actix_web::ResponseError for AuthenticationError {
fn status_code(&self) -> StatusCode {
match self {
apps/labrinth/src/auth/oauth/mod.rs
+1 -1
View File
@@ -12,7 +12,6 @@ use crate::database::models::{
DBOAuthClientAuthorizationId, generate_oauth_access_token_id,
generate_oauth_client_authorization_id,
};
use crate::database::redis::RedisPool;
use crate::models;
use crate::models::ids::OAuthClientId;
use crate::models::pats::Scopes;
@@ -25,6 +24,7 @@ use rand::distributions::Alphanumeric;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
use self::errors::{OAuthError, OAuthErrorType};
apps/labrinth/src/auth/validate.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use super::AuthProvider;
use crate::auth::AuthenticationError;
use crate::database::models::{DBUser, user_item};
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::pats::Scopes;
use crate::models::users::User;
@@ -10,6 +9,7 @@ use crate::routes::internal::session::get_session_metadata;
use actix_web::HttpRequest;
use actix_web::http::header::{AUTHORIZATION, HeaderValue};
use chrono::Utc;
use xredis::RedisPool;
pub async fn get_maybe_user_from_headers<'a, E>(
req: &HttpRequest,
apps/labrinth/src/background_task.rs
+2 -2
View File
@@ -2,7 +2,6 @@ use crate::database;
use crate::database::PgPool;
use crate::database::models::ids::DBUserId;
use crate::database::models::notification_item::NotificationBuilder;
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::notifications::NotificationBody;
use crate::queue::analytics::cache::cache_analytics;
@@ -20,6 +19,7 @@ use actix_web::web;
use clap::ValueEnum;
use eyre::WrapErr;
use tracing::info;
use xredis::RedisPool;
#[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
#[clap(rename_all = "kebab_case")]
@@ -386,11 +386,11 @@ mod version_updater {
use crate::database::PgPool;
use crate::database::models::legacy_loader_fields::MinecraftGameVersion;
use crate::database::redis::RedisPool;
use chrono::{DateTime, Utc};
use serde::Deserialize;
use thiserror::Error;
use tracing::warn;
use xredis::RedisPool;
#[derive(Deserialize)]
struct InputFormat<'a> {
apps/labrinth/src/database/models/analytics_event_item.rs
+12 -23
View File
@@ -1,17 +1,15 @@
use chrono::{DateTime, Utc};
use futures::{StreamExt, TryStreamExt};
use sqlx::types::Json;
use xredis::RedisPool;
use crate::{
database::{
models::{DBAnalyticsEventId, DatabaseError},
redis::RedisPool,
},
database::models::{DBAnalyticsEventId, DatabaseError},
models::v3::analytics_event::AnalyticsEventMeta,
};
use serde::{Deserialize, Serialize};
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v1";
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v3";
const ANALYTICS_EVENTS_ALL_KEY: &str = "all";
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -86,14 +84,11 @@ impl DBAnalyticsEvent {
redis: &RedisPool,
) -> Result<Vec<DBAnalyticsEvent>, DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis
.key()
.metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY);
if let Some(events) = redis
.get_deserialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
)
.await?
{
if let Some(events) = redis.get_deserialized(&key).await? {
return Ok(events);
}
@@ -118,23 +113,17 @@ impl DBAnalyticsEvent {
.try_collect::<Vec<_>>()
.await?;
redis
.set_serialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
&events,
None,
)
.await?;
redis.set_serialized(&key, &events, None).await?;
Ok(events)
}
pub async fn clear_cache(redis: &RedisPool) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY)
.await?;
let key = redis
.key()
.metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY);
redis.delete(&key).await?;
Ok(())
}
}
apps/labrinth/src/database/models/categories.rs
+19 -24
View File
@@ -1,13 +1,13 @@
use std::collections::HashMap;
use crate::database::redis::RedisPool;
use xredis::RedisPool;
use super::DatabaseError;
use super::ids::*;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const TAGS_NAMESPACE: &str = "tags:v1";
const TAGS_NAMESPACE: &str = "tags:v3";
pub struct ProjectType {
pub id: ProjectTypeId,
@@ -95,9 +95,10 @@ impl Category {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "category");
let res: Option<Vec<Category>> =
redis.get_deserialized(TAGS_NAMESPACE, "category").await?;
redis.get_deserialized(&key).await?;
if let Some(res) = res {
return Ok(res);
@@ -124,10 +125,9 @@ impl Category {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "category");
redis
.set_serialized(TAGS_NAMESPACE, "category", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -163,10 +163,10 @@ impl LinkPlatform {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "link_platform");
let res: Option<Vec<LinkPlatform>> = redis
.get_deserialized(TAGS_NAMESPACE, "link_platform")
.await?;
let res: Option<Vec<LinkPlatform>> =
redis.get_deserialized(&key).await?;
if let Some(res) = res {
return Ok(res);
@@ -188,10 +188,9 @@ impl LinkPlatform {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "link_platform");
redis
.set_serialized(TAGS_NAMESPACE, "link_platform", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -227,10 +226,9 @@ impl ReportType {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "report_type");
let res: Option<Vec<String>> = redis
.get_deserialized(TAGS_NAMESPACE, "report_type")
.await?;
let res: Option<Vec<String>> = redis.get_deserialized(&key).await?;
if let Some(res) = res {
return Ok(res);
@@ -248,10 +246,9 @@ impl ReportType {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "report_type");
redis
.set_serialized(TAGS_NAMESPACE, "report_type", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -287,10 +284,9 @@ impl ProjectType {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "project_type");
let res: Option<Vec<String>> = redis
.get_deserialized(TAGS_NAMESPACE, "project_type")
.await?;
let res: Option<Vec<String>> = redis.get_deserialized(&key).await?;
if let Some(res) = res {
return Ok(res);
@@ -308,10 +304,9 @@ impl ProjectType {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TAGS_NAMESPACE, "project_type");
redis
.set_serialized(TAGS_NAMESPACE, "project_type", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
apps/labrinth/src/database/models/collection_item.rs
+5 -4
View File
@@ -1,14 +1,14 @@
use super::ids::*;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::database::{PgTransaction, models};
use crate::models::collections::CollectionStatus;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const COLLECTIONS_NAMESPACE: &str = "collections:v1";
const COLLECTIONS_NAMESPACE: &str = "collections:v3";
#[derive(Clone)]
pub struct CollectionBuilder {
@@ -204,7 +204,7 @@ impl DBCollection {
})
.await?;
Ok(collections)
Ok::<_, DatabaseError>(collections)
},
)
.await?;
@@ -217,8 +217,9 @@ impl DBCollection {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().entity(COLLECTIONS_NAMESPACE, id.0);
redis.delete(COLLECTIONS_NAMESPACE, id.0).await?;
redis.delete(&key).await?;
Ok(())
}
}
apps/labrinth/src/database/models/flow_item.rs
+8 -10
View File
@@ -1,7 +1,6 @@
use super::ids::*;
use crate::auth::oauth::uris::OAuthRedirectUris;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::{auth::AuthProvider, routes::internal::flows::TempUser};
use chrono::Duration;
@@ -12,8 +11,9 @@ use rand_chacha::rand_core::SeedableRng;
use serde::{Deserialize, Serialize};
use url::Url;
use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration};
use xredis::RedisPool;
const FLOWS_NAMESPACE: &str = "flows:v1";
const FLOWS_NAMESPACE: &str = "flows:v3";
#[derive(Deserialize, Serialize)]
pub enum DBFlow {
@@ -75,14 +75,10 @@ impl DBFlow {
state: &str,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().entity(FLOWS_NAMESPACE, state);
redis
.set_serialized(
FLOWS_NAMESPACE,
&state,
&self,
Some(expires.num_seconds()),
)
.set_serialized(&key, &self, Some(expires.num_seconds()))
.await?;
Ok(())
}
@@ -107,8 +103,9 @@ impl DBFlow {
redis: &RedisPool,
) -> Result<Option<DBFlow>, DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().entity(FLOWS_NAMESPACE, id);
redis.get_deserialized(FLOWS_NAMESPACE, id).await
redis.get_deserialized(&key).await.map_err(Into::into)
}
/// Gets the flow and removes it from the cache, but only removes if the flow was present and the predicate returned true
@@ -132,8 +129,9 @@ impl DBFlow {
redis: &RedisPool,
) -> Result<Option<()>, DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().entity(FLOWS_NAMESPACE, id);
redis.delete(FLOWS_NAMESPACE, id).await?;
redis.delete(&key).await?;
Ok(Some(()))
}
}
apps/labrinth/src/database/models/image_item.rs
+5 -4
View File
@@ -1,12 +1,12 @@
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use crate::{database::models::DatabaseError, models::images::ImageContext};
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const IMAGES_NAMESPACE: &str = "images:v1";
const IMAGES_NAMESPACE: &str = "images:v3";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DBImage {
@@ -217,7 +217,7 @@ impl DBImage {
})
.await?;
Ok(images)
Ok::<_, DatabaseError>(images)
},
).await?;
@@ -229,8 +229,9 @@ impl DBImage {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().entity(IMAGES_NAMESPACE, id.0);
redis.delete(IMAGES_NAMESPACE, id.0).await?;
redis.delete(&key).await?;
Ok(())
}
}
apps/labrinth/src/database/models/legacy_loader_fields.rs
+4 -4
View File
@@ -9,7 +9,7 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serde_json::json;
use crate::database::redis::RedisPool;
use xredis::RedisPool;
use super::{
DatabaseError, LoaderFieldEnumValueId,
@@ -220,11 +220,11 @@ impl<'a> MinecraftGameVersionBuilder<'a> {
.await?;
let mut conn = redis.connect().await?;
conn.delete(
let key = conn.key().entity(
crate::database::models::loader_fields::LOADER_FIELD_ENUM_VALUES_NAMESPACE,
game_versions_enum.id.0,
)
.await?;
);
conn.delete(&key).await?;
Ok(LoaderFieldEnumValueId(result.id))
}
apps/labrinth/src/database/models/loader_fields.rs
+37 -44
View File
@@ -4,22 +4,22 @@ use std::hash::Hasher;
use super::DatabaseError;
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use chrono::DateTime;
use chrono::Utc;
use dashmap::DashMap;
use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const GAMES_LIST_NAMESPACE: &str = "games:v1";
const LOADER_ID: &str = "loader_id:v1";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v1";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v1";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v1";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v1";
const GAMES_LIST_NAMESPACE: &str = "games:v3";
const LOADER_ID: &str = "loader_id:v3";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v3";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v3";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v3";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v3";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str =
"loader_field_enum_values:v1";
"loader_field_enum_values:v3";
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct Game {
@@ -54,9 +54,9 @@ impl Game {
{
{
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis
.get_deserialized(GAMES_LIST_NAMESPACE, "games")
.await?;
let key = redis.key().metadata(GAMES_LIST_NAMESPACE, "games");
let cached_games: Option<Vec<Game>> =
redis.get_deserialized(&key).await?;
if let Some(cached_games) = cached_games {
return Ok(cached_games);
}
@@ -79,10 +79,9 @@ impl Game {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(GAMES_LIST_NAMESPACE, "games");
redis
.set_serialized(GAMES_LIST_NAMESPACE, "games", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -109,8 +108,8 @@ impl Loader {
{
{
let mut redis = redis.connect().await?;
let cached_id: Option<i32> =
redis.get_deserialized(LOADER_ID, name).await?;
let key = redis.key().metadata(LOADER_ID, name);
let cached_id: Option<i32> = redis.get_deserialized(&key).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
}
@@ -129,9 +128,8 @@ impl Loader {
if let Some(result) = result {
let mut redis = redis.connect().await?;
redis
.set_serialized(LOADER_ID, name, &result.0, None)
.await?;
let key = redis.key().metadata(LOADER_ID, name);
redis.set_serialized(&key, &result.0, None).await?;
}
Ok(result)
@@ -146,9 +144,9 @@ impl Loader {
{
{
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized(LOADERS_LIST_NAMESPACE, "all")
.await?;
let key = redis.key().metadata(LOADERS_LIST_NAMESPACE, "all");
let cached_loaders: Option<Vec<Loader>> =
redis.get_deserialized(&key).await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
}
@@ -187,10 +185,9 @@ impl Loader {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().metadata(LOADERS_LIST_NAMESPACE, "all");
redis
.set_serialized(LOADERS_LIST_NAMESPACE, "all", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -441,7 +438,7 @@ impl LoaderField {
})
.await?;
Ok(result)
Ok::<_, DatabaseError>(result)
},
).await?;
@@ -460,10 +457,10 @@ impl LoaderField {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(LOADER_FIELDS_NAMESPACE_ALL, "");
let cached_fields: Option<Vec<LoaderField>> = redis
.get_deserialized(LOADER_FIELDS_NAMESPACE_ALL, "")
.await?;
let cached_fields: Option<Vec<LoaderField>> =
redis.get_deserialized(&key).await?;
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
@@ -494,10 +491,9 @@ impl LoaderField {
.collect();
let mut redis = redis.connect().await?;
let key = redis.key().metadata(LOADER_FIELDS_NAMESPACE_ALL, "");
redis
.set_serialized(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -513,10 +509,11 @@ impl LoaderFieldEnum {
{
{
let mut redis = redis.connect().await?;
let key = redis
.key()
.metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name);
let cached_enum = redis
.get_deserialized(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name)
.await?;
let cached_enum = redis.get_deserialized(&key).await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
}
@@ -541,15 +538,11 @@ impl LoaderFieldEnum {
});
let mut redis = redis.connect().await?;
let key = redis
.key()
.metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name);
redis
.set_serialized(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
&result,
None,
)
.await?;
redis.set_serialized(&key, &result, None).await?;
Ok(result)
}
@@ -652,7 +645,7 @@ impl LoaderFieldEnumValue {
})
.await?;
Ok(values)
Ok::<_, DatabaseError>(values)
},
).await?;
apps/labrinth/src/database/models/mod.rs
+2 -15
View File
@@ -72,25 +72,12 @@ pub enum DatabaseError {
RandomId,
#[error("Error while interacting with the cache: {0}")]
CacheError(#[from] redis::RedisError),
#[error("Redis Pool Error: {0}")]
RedisPool(#[from] deadpool_redis::PoolError),
#[error("Error while serializing with the cache: {0}")]
SerdeCacheError(#[from] serde_json::Error),
#[error("error while encoding or decoding the cache: {0}")]
PostcardCacheError(#[from] postcard::Error),
#[error(transparent)]
Redis(#[from] xredis::Error),
#[error("Schema error: {0}")]
SchemaError(String),
#[error(
"Timeout waiting on Redis cache lock ({locks_released}/{locks_waiting} released, spent {time_spent_pool_wait_ms}ms/{time_spent_total_ms}ms waiting on connections from pool)"
)]
CacheTimeout {
locks_released: usize,
locks_waiting: usize,
time_spent_pool_wait_ms: u64,
time_spent_total_ms: u64,
},
#[error(
"Timeout waiting on local cache lock ({released}/{total} released)"
)]
LocalCacheTimeout { released: usize, total: usize },
}
apps/labrinth/src/database/models/moderation_note_item.rs
+34 -44
View File
@@ -3,13 +3,13 @@ use std::collections::HashMap;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use crate::database::redis::RedisPool;
use xredis::RedisPool;
use super::{DBOrganizationId, DBUserId, DatabaseError};
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v1";
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v3";
const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str =
"moderation_notes_organizations:v1";
"moderation_notes_organizations:v3";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DBModerationNote {
@@ -32,19 +32,15 @@ impl DBModerationNote {
where
E: crate::database::Executor<'a, Database = sqlx::Postgres>,
{
let ids = user_ids
.iter()
.map(|id| id.0.to_string())
.collect::<Vec<_>>();
let cached = {
let mut redis = redis.connect().await?;
redis
.get_many_deserialized::<Self>(
MODERATION_NOTES_USERS_NAMESPACE,
&ids,
)
.await?
let keys = user_ids
.iter()
.map(|id| {
redis.key().entity(MODERATION_NOTES_USERS_NAMESPACE, id.0)
})
.collect::<Vec<_>>();
redis.get_many_deserialized::<Self>(&keys).await?
};
let mut notes = HashMap::new();
@@ -86,14 +82,10 @@ impl DBModerationNote {
};
if let Some(user_id) = note.user_id {
redis
.set_serialized(
MODERATION_NOTES_USERS_NAMESPACE,
user_id.0,
&note,
None,
)
.await?;
let key = redis
.key()
.entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0);
redis.set_serialized(&key, &note, None).await?;
notes.insert(user_id, note);
}
}
@@ -122,19 +114,17 @@ impl DBModerationNote {
where
E: crate::database::Executor<'a, Database = sqlx::Postgres>,
{
let ids = organization_ids
.iter()
.map(|id| id.0.to_string())
.collect::<Vec<_>>();
let cached = {
let mut redis = redis.connect().await?;
let keys = organization_ids
.iter()
.map(|id| {
redis
.get_many_deserialized::<Self>(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
&ids,
)
.await?
.key()
.entity(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, id.0)
})
.collect::<Vec<_>>();
redis.get_many_deserialized::<Self>(&keys).await?
};
let mut notes = HashMap::new();
@@ -176,14 +166,11 @@ impl DBModerationNote {
};
if let Some(organization_id) = note.organization_id {
redis
.set_serialized(
let key = redis.key().entity(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
organization_id.0,
&note,
None,
)
.await?;
);
redis.set_serialized(&key, &note, None).await?;
notes.insert(organization_id, note);
}
}
@@ -289,9 +276,10 @@ impl DBModerationNote {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete(MODERATION_NOTES_USERS_NAMESPACE, user_id.0)
.await
let key = redis
.key()
.entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0);
redis.delete(&key).await.map_err(Into::into)
}
pub async fn clear_organization_cache(
@@ -299,8 +287,10 @@ impl DBModerationNote {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0)
.await
let key = redis.key().entity(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
organization_id.0,
);
redis.delete(&key).await.map_err(Into::into)
}
}
apps/labrinth/src/database/models/notification_item.rs
+14 -21
View File
@@ -1,6 +1,6 @@
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::{models::DatabaseError, redis::RedisPool};
use crate::database::models::DatabaseError;
use crate::models::notifications::{
NotificationBody, NotificationChannel, NotificationDeliveryStatus,
NotificationType,
@@ -8,8 +8,9 @@ use crate::models::notifications::{
use chrono::{DateTime, Utc};
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v1";
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v3";
pub struct NotificationBuilder {
pub body: NotificationBody,
@@ -433,13 +434,11 @@ impl DBNotification {
{
{
let mut redis = redis.connect().await?;
let key =
redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0);
let cached_notifications: Option<Vec<DBNotification>> = redis
.get_deserialized(
USER_NOTIFICATIONS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let cached_notifications: Option<Vec<DBNotification>> =
redis.get_deserialized(&key).await?;
if let Some(notifications) = cached_notifications {
return Ok(notifications);
@@ -491,15 +490,9 @@ impl DBNotification {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0);
redis
.set_serialized(
USER_NOTIFICATIONS_NAMESPACE,
user_id.0,
&db_notifications,
None,
)
.await?;
redis.set_serialized(&key, &db_notifications, None).await?;
Ok(db_notifications)
}
@@ -638,12 +631,12 @@ impl DBNotification {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let keys = user_ids
.into_iter()
.map(|id| redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, id.0))
.collect::<Vec<_>>();
redis
.delete_many(user_ids.into_iter().map(|id| {
(USER_NOTIFICATIONS_NAMESPACE, Some(id.0.to_string()))
}))
.await?;
redis.delete_many(&keys).await?;
Ok(())
}
apps/labrinth/src/database/models/notifications_template_item.rs
+23 -34
View File
@@ -1,14 +1,14 @@
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::v3::notifications::{NotificationChannel, NotificationType};
use crate::routes::ApiError;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v1";
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v3";
const TEMPLATES_HTML_DATA_NAMESPACE: &str =
"notifications_templates_html_data:v1";
"notifications_templates_html_data:v3";
const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str =
"notifications_templates_dynamic_html:v1";
"notifications_templates_dynamic_html:v3";
const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes
const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes
@@ -55,10 +55,10 @@ impl NotificationTemplate {
) -> Result<Vec<NotificationTemplate>, DatabaseError> {
{
let mut redis = redis.connect().await?;
let key =
redis.key().metadata(TEMPLATES_NAMESPACE, channel.as_str());
let maybe_cached_templates = redis
.get_deserialized(TEMPLATES_NAMESPACE, channel.as_str())
.await?;
let maybe_cached_templates = redis.get_deserialized(&key).await?;
if let Some(cached) = maybe_cached_templates {
return Ok(cached);
@@ -78,14 +78,10 @@ impl NotificationTemplate {
let templates = results.into_iter().map(Into::into).collect();
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TEMPLATES_NAMESPACE, channel.as_str());
redis
.set_serialized(
TEMPLATES_NAMESPACE,
channel.as_str(),
&templates,
Some(TEMPLATES_CACHE_EXPIRY),
)
.set_serialized(&key, &templates, Some(TEMPLATES_CACHE_EXPIRY))
.await?;
Ok(templates)
@@ -96,12 +92,8 @@ impl NotificationTemplate {
redis: &RedisPool,
) -> Result<Option<String>, DatabaseError> {
let mut redis = redis.connect().await?;
redis
.get_deserialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
)
.await
let key = redis.key().metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id);
redis.get_deserialized(&key).await.map_err(Into::into)
}
pub async fn set_cached_html_data(
@@ -110,14 +102,11 @@ impl NotificationTemplate {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let key = redis.key().metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id);
redis
.set_serialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
&data,
Some(HTML_DATA_CACHE_EXPIRY),
)
.set_serialized(&key, &data, Some(HTML_DATA_CACHE_EXPIRY))
.await
.map_err(Into::into)
}
}
@@ -135,9 +124,11 @@ where
}
let mut redis_conn = redis.connect().await?;
if let Some(body) = redis_conn
.get_deserialized::<HtmlBody>(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key)
.await?
let redis_key = redis_conn
.key()
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
if let Some(body) =
redis_conn.get_deserialized::<HtmlBody>(&redis_key).await?
{
return Ok(body.html);
}
@@ -146,14 +137,12 @@ where
let cached = HtmlBody { html: get().await? };
let mut redis_conn = redis.connect().await?;
let redis_key = redis_conn
.key()
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
redis_conn
.set_serialized(
TEMPLATES_DYNAMIC_HTML_NAMESPACE,
key,
&cached,
Some(HTML_DATA_CACHE_EXPIRY),
)
.set_serialized(&redis_key, &cached, Some(HTML_DATA_CACHE_EXPIRY))
.await?;
Ok(cached.html)
apps/labrinth/src/database/models/notifications_type_item.rs
+6 -8
View File
@@ -1,9 +1,9 @@
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::v3::notifications::NotificationType;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v1";
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v3";
#[derive(Serialize, Deserialize)]
pub struct NotificationTypeItem {
@@ -41,10 +41,9 @@ impl NotificationTypeItem {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(NOTIFICATION_TYPES_NAMESPACE, "all");
let cached_types = redis
.get_deserialized(NOTIFICATION_TYPES_NAMESPACE, "all")
.await?;
let cached_types = redis.get_deserialized(&key).await?;
if let Some(types) = cached_types {
return Ok(types);
@@ -61,10 +60,9 @@ impl NotificationTypeItem {
let types = results.into_iter().map(Into::into).collect();
let mut redis = redis.connect().await?;
let key = redis.key().metadata(NOTIFICATION_TYPES_NAMESPACE, "all");
redis
.set_serialized(NOTIFICATION_TYPES_NAMESPACE, "all", &types, None)
.await?;
redis.set_serialized(&key, &types, None).await?;
Ok(types)
}
apps/labrinth/src/database/models/organization_item.rs
+15 -12
View File
@@ -1,16 +1,16 @@
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use ariadne::ids::base62_impl::parse_base62;
use dashmap::DashMap;
use futures::TryStreamExt;
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
use super::{DBTeamMember, ids::*};
use serde::{Deserialize, Serialize};
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v1";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v1";
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v3";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v3";
#[derive(Deserialize, Serialize, Clone, Debug)]
/// An organization of users who together control one or more projects and organizations.
@@ -159,7 +159,9 @@ impl DBOrganization {
})
.await?;
Ok(organizations)
Ok::<_, crate::database::models::DatabaseError>(
organizations,
)
},
)
.await?;
@@ -256,16 +258,17 @@ impl DBOrganization {
redis: &RedisPool,
) -> Result<(), super::DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete_many([
(ORGANIZATIONS_NAMESPACE, Some(id.0.to_string())),
(
let mut keys = vec![redis.key().entity(ORGANIZATIONS_NAMESPACE, id.0)];
if let Some(slug) = slug {
keys.push(
redis.key().entity(
ORGANIZATIONS_TITLES_NAMESPACE,
slug.map(|x| x.to_lowercase()),
slug.to_lowercase(),
),
])
.await?;
);
}
redis.delete_many(&keys).await?;
Ok(())
}
}
apps/labrinth/src/database/models/pat_item.rs
+24 -26
View File
@@ -1,7 +1,6 @@
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use ariadne::ids::base62_impl::parse_base62;
use chrono::{DateTime, Utc};
@@ -10,10 +9,11 @@ use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const PATS_NAMESPACE: &str = "pats:v1";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v1";
const PATS_USERS_NAMESPACE: &str = "pats_users:v1";
const PATS_NAMESPACE: &str = "pats:v3";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v3";
const PATS_USERS_NAMESPACE: &str = "pats_users:v3";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBPersonalAccessToken {
@@ -141,7 +141,7 @@ impl DBPersonalAccessToken {
async move { Ok(acc) }
})
.await?;
Ok(pats)
Ok::<_, DatabaseError>(pats)
},
)
.await?;
@@ -159,13 +159,9 @@ impl DBPersonalAccessToken {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().entity(PATS_USERS_NAMESPACE, user_id.0);
let res = redis
.get_deserialized::<Vec<i64>>(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let res = redis.get_deserialized::<Vec<i64>>(&key).await?;
if let Some(res) = res {
return Ok(res.into_iter().map(DBPatId).collect());
@@ -187,10 +183,9 @@ impl DBPersonalAccessToken {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().entity(PATS_USERS_NAMESPACE, user_id.0);
redis
.set_serialized(PATS_USERS_NAMESPACE, user_id.0, &db_pats, None)
.await?;
redis.set_serialized(&key, &db_pats, None).await?;
Ok(db_pats)
}
@@ -204,20 +199,23 @@ impl DBPersonalAccessToken {
return Ok(());
}
redis
.delete_many(clear_pats.into_iter().flat_map(
|(id, token, user_id)| {
let keys = clear_pats
.into_iter()
.flat_map(|(id, token, user_id)| {
[
(PATS_NAMESPACE, id.map(|i| i.0.to_string())),
(PATS_TOKENS_NAMESPACE, token),
(
PATS_USERS_NAMESPACE,
user_id.map(|i| i.0.to_string()),
),
id.map(|id| redis.key().entity(PATS_NAMESPACE, id.0)),
token.map(|token| {
redis.key().entity(PATS_TOKENS_NAMESPACE, token)
}),
user_id.map(|user_id| {
redis.key().entity(PATS_USERS_NAMESPACE, user_id.0)
}),
]
},
))
.await?;
.into_iter()
.flatten()
})
.collect::<Vec<_>>();
redis.delete_many(&keys).await?;
Ok(())
}
apps/labrinth/src/database/models/product_item.rs
+6 -6
View File
@@ -1,15 +1,15 @@
use crate::database::models::{
DBProductId, DBProductPriceId, DatabaseError, product_item,
};
use crate::database::redis::RedisPool;
use crate::models::billing::{Price, ProductMetadata};
use dashmap::DashMap;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::convert::TryFrom;
use std::convert::TryInto;
use xredis::RedisPool;
const PRODUCTS_NAMESPACE: &str = "products:v1";
const PRODUCTS_NAMESPACE: &str = "products:v3";
pub struct DBProduct {
pub id: DBProductId,
@@ -152,9 +152,10 @@ impl QueryProductWithPrices {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().metadata(PRODUCTS_NAMESPACE, "all");
let res: Option<Vec<QueryProductWithPrices>> =
redis.get_deserialized(PRODUCTS_NAMESPACE, "all").await?;
redis.get_deserialized(&key).await?;
if let Some(res) = res {
return Ok(res);
@@ -193,10 +194,9 @@ impl QueryProductWithPrices {
.collect::<Vec<_>>();
let mut redis = redis.connect().await?;
let key = redis.key().metadata(PRODUCTS_NAMESPACE, "all");
redis
.set_serialized(PRODUCTS_NAMESPACE, "all", &products, None)
.await?;
redis.set_serialized(&key, &products, None).await?;
Ok(products)
}
apps/labrinth/src/database/models/project_item.rs
+23 -32
View File
@@ -4,7 +4,6 @@ use super::loader_fields::{
};
use super::{DBUser, ids::*};
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::database::{PgTransaction, models};
use crate::file_hosting::FileHost;
use crate::models::exp;
@@ -22,10 +21,11 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
pub const PROJECTS_NAMESPACE: &str = "projects:v1";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v1";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v1";
pub const PROJECTS_NAMESPACE: &str = "projects:v3";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v3";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v3";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LinkUrl {
@@ -942,7 +942,7 @@ impl DBProject {
})
?;
Ok(projects)
Ok::<_, DatabaseError>(projects)
},
)
.await
@@ -974,13 +974,10 @@ impl DBProject {
{
let mut redis = redis.connect().await?;
let key = redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0);
let dependencies = redis
.get_deserialized::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
.await?;
let dependencies =
redis.get_deserialized::<Dependencies>(&key).await?;
if let Some(dependencies) = dependencies {
return Ok(dependencies);
}
@@ -1012,15 +1009,9 @@ impl DBProject {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0);
redis
.set_serialized(
PROJECTS_DEPENDENCIES_NAMESPACE,
id.0,
&dependencies,
None,
)
.await?;
redis.set_serialized(&key, &dependencies, None).await?;
Ok(dependencies)
}
@@ -1031,21 +1022,21 @@ impl DBProject {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let mut keys = vec![redis.key().entity(PROJECTS_NAMESPACE, id.0)];
if let Some(slug) = slug {
keys.push(
redis
.delete_many([
(PROJECTS_NAMESPACE, Some(id.0.to_string())),
(PROJECTS_SLUGS_NAMESPACE, slug.map(|x| x.to_lowercase())),
(
PROJECTS_DEPENDENCIES_NAMESPACE,
.key()
.entity(PROJECTS_SLUGS_NAMESPACE, slug.to_lowercase()),
);
}
if clear_dependencies.unwrap_or(false) {
Some(id.0.to_string())
} else {
None
},
),
])
.await?;
keys.push(
redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0),
);
}
redis.delete_many(&keys).await?;
Ok(())
}
}
apps/labrinth/src/database/models/session_item.rs
+24 -31
View File
@@ -1,7 +1,6 @@
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use ariadne::ids::base62_impl::parse_base62;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
@@ -9,10 +8,11 @@ use futures_util::TryStreamExt;
use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const SESSIONS_NAMESPACE: &str = "sessions:v1";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v1";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v1";
const SESSIONS_NAMESPACE: &str = "sessions:v3";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v3";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v3";
pub struct SessionBuilder {
pub session: String,
@@ -208,7 +208,7 @@ impl DBSession {
})
.await?;
Ok(db_sessions)
Ok::<_, DatabaseError>(db_sessions)
}).await?;
Ok(val)
@@ -224,13 +224,9 @@ impl DBSession {
{
{
let mut redis = redis.connect().await?;
let key = redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0);
let res = redis
.get_deserialized::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let res = redis.get_deserialized::<Vec<i64>>(&key).await?;
if let Some(res) = res {
return Ok(res.into_iter().map(DBSessionId).collect());
@@ -253,15 +249,9 @@ impl DBSession {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0);
redis
.set_serialized(
SESSIONS_USERS_NAMESPACE,
user_id.0,
&db_sessions,
None,
)
.await?;
redis.set_serialized(&key, &db_sessions, None).await?;
Ok(db_sessions)
}
@@ -280,20 +270,23 @@ impl DBSession {
return Ok(());
}
redis
.delete_many(clear_sessions.into_iter().flat_map(
|(id, session, user_id)| {
let keys = clear_sessions
.into_iter()
.flat_map(|(id, session, user_id)| {
[
(SESSIONS_NAMESPACE, id.map(|i| i.0.to_string())),
(SESSIONS_IDS_NAMESPACE, session),
(
SESSIONS_USERS_NAMESPACE,
user_id.map(|i| i.0.to_string()),
),
id.map(|id| redis.key().entity(SESSIONS_NAMESPACE, id.0)),
session.map(|session| {
redis.key().entity(SESSIONS_IDS_NAMESPACE, session)
}),
user_id.map(|user_id| {
redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0)
}),
]
},
))
.await?;
.into_iter()
.flatten()
})
.collect::<Vec<_>>();
redis.delete_many(&keys).await?;
Ok(())
}
apps/labrinth/src/database/models/team_item.rs
+6 -4
View File
@@ -1,6 +1,6 @@
use super::{DBOrganization, DBProject, ids::*};
use crate::{
database::{PgTransaction, redis::RedisPool},
database::PgTransaction,
models::teams::{OrganizationPermissions, ProjectPermissions},
};
use dashmap::DashMap;
@@ -8,8 +8,9 @@ use futures::TryStreamExt;
use itertools::Itertools;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const TEAMS_NAMESPACE: &str = "teams:v1";
const TEAMS_NAMESPACE: &str = "teams:v3";
pub struct TeamBuilder {
pub members: Vec<TeamMemberBuilder>,
@@ -253,7 +254,7 @@ impl DBTeamMember {
})
.await?;
Ok(teams)
Ok::<_, crate::database::models::DatabaseError>(teams)
},
).await?;
@@ -265,7 +266,8 @@ impl DBTeamMember {
redis: &RedisPool,
) -> Result<(), super::DatabaseError> {
let mut redis = redis.connect().await?;
redis.delete(TEAMS_NAMESPACE, id.0).await?;
let key = redis.key().entity(TEAMS_NAMESPACE, id.0);
redis.delete(&key).await?;
Ok(())
}
apps/labrinth/src/database/models/user_item.rs
+30 -35
View File
@@ -3,7 +3,6 @@ use super::{DBCollectionId, DBReportId, DBThreadId};
use crate::database::models::charge_item::DBCharge;
use crate::database::models::user_subscription_item::DBUserSubscription;
use crate::database::models::{DBOrganizationId, DatabaseError};
use crate::database::redis::RedisPool;
use crate::database::{PgTransaction, models};
use crate::models::billing::ChargeStatus;
use crate::models::users::Badges;
@@ -15,10 +14,11 @@ use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const USERS_NAMESPACE: &str = "users:v1";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v1";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v1";
const USERS_NAMESPACE: &str = "users:v3";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v3";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v3";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBUser {
@@ -273,7 +273,7 @@ impl DBUser {
})
.await?;
Ok(users)
Ok::<_, DatabaseError>(users)
}).await?;
Ok(val)
}
@@ -389,13 +389,10 @@ impl DBUser {
{
let mut redis = redis.connect().await?;
let key = redis.key().entity(USERS_PROJECTS_NAMESPACE, user_id.0);
let cached_projects = redis
.get_deserialized::<Vec<DBProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let cached_projects =
redis.get_deserialized::<Vec<DBProjectId>>(&key).await?;
if let Some(projects) = cached_projects {
return Ok(projects);
@@ -417,15 +414,9 @@ impl DBUser {
.await?;
let mut redis = redis.connect().await?;
let key = redis.key().entity(USERS_PROJECTS_NAMESPACE, user_id.0);
redis
.set_serialized(
USERS_PROJECTS_NAMESPACE,
user_id.0,
&db_projects,
None,
)
.await?;
redis.set_serialized(&key, &db_projects, None).await?;
Ok(db_projects)
}
@@ -556,18 +547,24 @@ impl DBUser {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete_many(user_ids.iter().flat_map(|(id, username)| {
let keys = user_ids
.iter()
.flat_map(|(id, username)| {
[
(USERS_NAMESPACE, Some(id.0.to_string())),
(
Some(redis.key().entity(USERS_NAMESPACE, id.0)),
username.as_ref().map(|username| {
redis.key().entity(
USER_USERNAMES_NAMESPACE,
username.clone().map(|i| i.to_lowercase()),
),
username.to_lowercase(),
)
}),
]
}))
.await?;
.into_iter()
.flatten()
})
.collect::<Vec<_>>();
redis.delete_many(&keys).await?;
Ok(())
}
@@ -576,14 +573,12 @@ impl DBUser {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
let keys = user_ids
.iter()
.map(|id| redis.key().entity(USERS_PROJECTS_NAMESPACE, id.0))
.collect::<Vec<_>>();
redis
.delete_many(
user_ids.iter().map(|id| {
(USERS_PROJECTS_NAMESPACE, Some(id.0.to_string()))
}),
)
.await?;
redis.delete_many(&keys).await?;
Ok(())
}
apps/labrinth/src/database/models/version_item.rs
+19 -29
View File
@@ -5,9 +5,9 @@ use crate::database::PgTransaction;
use crate::database::models::loader_fields::{
QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
};
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::exp;
use xredis::RedisPool;
use crate::models::projects::{FileType, VersionStatus};
use crate::queue::file_scan::scan_file;
@@ -19,11 +19,10 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::collections::HashMap;
use std::iter;
use tracing::error;
pub const VERSIONS_NAMESPACE: &str = "versions:v1";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v1";
pub const VERSIONS_NAMESPACE: &str = "versions:v3";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v3";
pub async fn cleanup_unused_attribution_files_and_groups(
transaction: &mut PgTransaction<'_>,
@@ -948,7 +947,7 @@ impl DBVersion {
})
.await?;
Ok(res)
Ok::<_, DatabaseError>(res)
},
).await?;
@@ -1039,7 +1038,7 @@ impl DBVersion {
})
.await?;
Ok(files)
Ok::<_, DatabaseError>(files)
}
).await?;
@@ -1051,25 +1050,18 @@ impl DBVersion {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete_many(
iter::once((
VERSIONS_NAMESPACE,
Some(version.inner.id.0.to_string()),
))
.chain(version.files.iter().flat_map(
|file| {
file.hashes.iter().map(|(algo, hash)| {
(
let mut keys =
vec![redis.key().entity(VERSIONS_NAMESPACE, version.inner.id.0)];
keys.extend(version.files.iter().flat_map(|file| {
file.hashes.iter().map(|(algorithm, hash)| {
redis.key().entity(
VERSION_FILES_NAMESPACE,
Some(format!("{algo}_{hash}")),
format!("{algorithm}_{hash}"),
)
})
},
)),
)
.await?;
}));
redis.delete_many(&keys).await?;
Ok(())
}
@@ -1078,14 +1070,12 @@ impl DBVersion {
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.delete_many(
version_ids
let keys = version_ids
.iter()
.map(|id| (VERSIONS_NAMESPACE, Some(id.0.to_string()))),
)
.await?;
.map(|id| redis.key().entity(VERSIONS_NAMESPACE, id.0))
.collect::<Vec<_>>();
redis.delete_many(&keys).await?;
Ok(())
}
}
apps/labrinth/src/database/redis.rs
+56
View File
@@ -0,0 +1,56 @@
use std::sync::Arc;
use crate::env::ENV;
struct RedisConfig {
inner: xredis::RedisConfig,
cache_settings: xredis::CacheSettings,
}
impl RedisConfig {
fn from_env() -> Result<Self, xredis::RedisConfigError> {
let inner = xredis::RedisConfig::new(
ENV.REDIS_TOPOLOGY,
ENV.REDIS_CONNECTION_TYPE,
&ENV.REDIS_URL,
ENV.REDIS_WAIT_TIMEOUT_MS,
(
ENV.REDIS_MAX_CONNECTIONS as usize,
ENV.REDIS_MIN_CONNECTIONS,
),
(
ENV.REDIS_CLUSTER_MAX_CONNECTIONS as usize,
ENV.REDIS_CLUSTER_MIN_CONNECTIONS,
),
(ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0),
ENV.REDIS_CACHE_LOCKING_STRATEGY,
ENV.REDIS_READ_REPLICA_STRATEGY,
)?;
let cache_settings = xredis::CacheSettings {
default_expiry: ENV.REDIS_DEFAULT_EXPIRY,
actual_expiry: ENV.REDIS_ACTUAL_EXPIRY,
version_default_expiry: ENV.REDIS_VERSION_DEFAULT_EXPIRY,
version_actual_expiry: ENV.REDIS_VERSION_ACTUAL_EXPIRY,
encoding_format: ENV.REDIS_ENCODING_FORMAT,
compression_algorithm: ENV.REDIS_COMPRESSION_ALGORITHM,
compression_level: ENV.REDIS_COMPRESSION_LEVEL,
compression_threshold_bytes: ENV.REDIS_COMPRESSION_THRESHOLD_BYTES,
compression_min_savings_ratio: ENV
.REDIS_COMPRESSION_MIN_SAVINGS_RATIO,
};
Ok(Self {
inner,
cache_settings,
})
}
}
pub async fn from_env(
meta_namespace: impl Into<Arc<str>>,
) -> xredis::RedisPool {
let config = RedisConfig::from_env().expect("invalid Redis configuration");
xredis::RedisPool::new(meta_namespace, config.inner, config.cache_settings)
.await
.expect("failed to initialize Redis connections")
}
apps/labrinth/src/database/redis/mod.rs
-957
View File
@@ -1,957 +0,0 @@
use crate::env::ENV;
use super::models::DatabaseError;
use ariadne::ids::base62_impl::{parse_base62, to_base62};
use chrono::{TimeZone, Utc};
use dashmap::DashMap;
use deadpool_redis::{Config, Runtime};
use futures::TryStreamExt;
use futures::future::Either;
use futures::stream::{FuturesUnordered, StreamExt};
use prometheus::{IntGauge, Registry};
use redis::ToRedisArgs;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::{Debug, Display};
use std::future::Future;
use std::hash::Hash;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
use tracing::{Instrument, info, info_span};
use util::{cmd, redis_pipe};
pub mod util;
// Bound how many commands we send in a single Redis pipeline. The multiplexed
// connection's BytesMut write buffer keeps its peak capacity for the life of
// the connection, so larger pipelines cause higher steady-state RSS.
const PIPELINE_CHUNK_SIZE: usize = 25;
// Bound how many keys we send in a single MGET. Each MGET response must fit
// into the connection's read buffer, which also retains its peak capacity. At
// ~1 MB per cached value, 32 keys caps any single response at ~32 MB.
const MGET_CHUNK_SIZE: usize = 32;
// How long a pooled Redis connection lives before being recycled, regardless
// of activity. Forced recycling is the only way to release the per-connection
// BytesMut peak capacity that builds up under steady load.
const REDIS_MAX_CONN_AGE: Duration = Duration::from_secs(120);
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Codec {
Raw = 0,
Lz4 = 1,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncodingFormat {
Json,
Postcard,
}
#[derive(Debug, Error)]
#[error("invalid redis codec")]
pub struct InvalidCodec;
#[derive(Debug, Error)]
#[error("invalid redis encoding format")]
pub struct InvalidEncodingFormat;
impl TryFrom<u8> for Codec {
type Error = InvalidCodec;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(Self::Raw),
1 => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for Codec {
type Err = InvalidCodec;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"lz4" => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for EncodingFormat {
type Err = InvalidEncodingFormat;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"json" => Ok(Self::Json),
"postcard" => Ok(Self::Postcard),
_ => Err(InvalidEncodingFormat),
}
}
}
fn encode_value<T: Serialize>(value: &T) -> Result<Vec<u8>, DatabaseError> {
let mut value = match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::to_vec(value)?,
EncodingFormat::Postcard => postcard::to_allocvec(value)?,
};
if ENV.REDIS_COMPRESSION_LEVEL > 0
&& ENV.REDIS_COMPRESSION_ALGORITHM == Codec::Lz4
&& value.len() >= ENV.REDIS_COMPRESSION_THRESHOLD_BYTES
{
let compressed = lz4_flex::block::compress_prepend_size(&value);
let savings_ratio = value.len().saturating_sub(compressed.len()) as f64
/ value.len().max(1) as f64
* 100.0;
if savings_ratio >= ENV.REDIS_COMPRESSION_MIN_SAVINGS_RATIO {
let mut encoded = Vec::with_capacity(compressed.len() + 1);
encoded.push(Codec::Lz4 as u8);
encoded.extend(compressed);
return Ok(encoded);
}
}
let mut encoded = Vec::with_capacity(value.len() + 1);
encoded.push(Codec::Raw as u8);
encoded.append(&mut value);
Ok(encoded)
}
fn decode_value<T>(value: &[u8]) -> Option<T>
where
T: for<'a> Deserialize<'a>,
{
let (codec, value) = value.split_first()?;
let value = match Codec::try_from(*codec).ok()? {
Codec::Raw => Cow::Borrowed(value),
Codec::Lz4 => {
Cow::Owned(lz4_flex::block::decompress_size_prepended(value).ok()?)
}
};
match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::from_slice(&value).ok(),
EncodingFormat::Postcard => postcard::from_bytes(&value).ok(),
}
}
fn cache_expiries(namespace: &str) -> (i64, i64) {
// Namespaces may embed a version suffix like `:v1`, so split it out.
match namespace.split_once(':').map(|t| t.0).unwrap_or(namespace) {
"versions" | "versions_files" => (
ENV.REDIS_VERSION_DEFAULT_EXPIRY,
ENV.REDIS_VERSION_ACTUAL_EXPIRY,
),
_ => (ENV.REDIS_DEFAULT_EXPIRY, ENV.REDIS_ACTUAL_EXPIRY),
}
}
#[derive(Clone)]
pub struct RedisPool {
pub url: String,
pub pool: deadpool_redis::Pool,
cache_list: Arc<DashMap<String, util::CacheSubscriber>>,
meta_namespace: Arc<str>,
}
pub struct RedisConnection {
pub connection: deadpool_redis::Connection,
meta_namespace: Arc<str>,
}
impl RedisPool {
// initiate a new redis pool
// testing pool uses a hashmap to mimic redis behaviour for very small data sizes (ie: tests)
// PANICS: production pool will panic if redis url is not set
pub fn new(meta_namespace: impl Into<Arc<str>>) -> Self {
let wait_timeout = Duration::from_millis(ENV.REDIS_WAIT_TIMEOUT_MS);
let url = &ENV.REDIS_URL;
let pool = Config::from_url(url.clone())
.builder()
.expect("Error building Redis pool")
.max_size(ENV.REDIS_MAX_CONNECTIONS as usize)
.wait_timeout(Some(wait_timeout))
.runtime(Runtime::Tokio1)
.build()
.expect("Redis connection failed");
let pool = RedisPool {
url: url.clone(),
pool,
cache_list: Arc::new(DashMap::with_capacity(2048)),
meta_namespace: meta_namespace.into(),
};
let redis_min_connections = ENV.REDIS_MIN_CONNECTIONS;
let spawn_min_connections = (0..redis_min_connections)
.map(|_| {
let pool = pool.clone();
tokio::spawn(async move { pool.pool.get().await })
})
.collect::<FuturesUnordered<_>>();
tokio::spawn({
let pool = pool.clone();
async move {
// collect the connections into a buffer while we're spawning them,
// to make sure that we're not `get`ing any connections we previously took
let _connections =
spawn_min_connections.try_collect::<Vec<_>>().await;
info!(
pool_status = ?pool.pool.status(),
"Finished getting {redis_min_connections} initial Redis connections"
);
}
});
let interval = Duration::from_secs(30);
let max_idle = Duration::from_secs(5 * 60); // 5 minutes
let pool_ref = pool.clone();
tokio::spawn(async move {
loop {
tokio::time::sleep(interval).await;
pool_ref.pool.retain(|_, metrics| {
// Drop connections that have been idle too long, OR that
// are older than REDIS_MAX_CONN_AGE regardless of use.
// The age-based recycle is what releases the per-connection
// BytesMut peak capacity under steady traffic.
metrics.last_used() < max_idle
&& metrics.created.elapsed() < REDIS_MAX_CONN_AGE
});
}
});
pool
}
pub async fn register_and_set_metrics(
&self,
registry: &Registry,
) -> Result<(), prometheus::Error> {
let redis_max_size = IntGauge::new(
"labrinth_redis_pool_max_size",
"Maximum size of Redis pool",
)?;
let redis_size = IntGauge::new(
"labrinth_redis_pool_size",
"Current size of Redis pool",
)?;
let redis_available = IntGauge::new(
"labrinth_redis_pool_available",
"Available connections in Redis pool",
)?;
let redis_waiting = IntGauge::new(
"labrinth_redis_pool_waiting",
"Number of futures waiting for a Redis connection",
)?;
registry.register(Box::new(redis_max_size.clone()))?;
registry.register(Box::new(redis_size.clone()))?;
registry.register(Box::new(redis_available.clone()))?;
registry.register(Box::new(redis_waiting.clone()))?;
let redis_pool_ref = self.pool.clone();
tokio::spawn(async move {
loop {
let status = redis_pool_ref.status();
redis_max_size.set(status.max_size as i64);
redis_size.set(status.size as i64);
redis_available.set(status.available as i64);
redis_waiting.set(status.waiting as i64);
tokio::time::sleep(Duration::from_secs(5)).await;
}
});
Ok(())
}
#[tracing::instrument(skip(self))]
pub async fn connect(&self) -> Result<RedisConnection, DatabaseError> {
Ok(RedisConnection {
connection: self.pool.get().await?,
meta_namespace: self.meta_namespace.clone(),
})
}
#[tracing::instrument(skip(self, closure))]
pub async fn get_cached_keys<F, Fut, T, K>(
&self,
namespace: &str,
keys: &[K],
closure: F,
) -> Result<Vec<T>, DatabaseError>
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, DatabaseError>>,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
+ Eq
+ PartialEq
+ Clone
+ DeserializeOwned
+ Serialize
+ Debug,
{
Ok(self
.get_cached_keys_raw(namespace, keys, closure)
.await?
.into_iter()
.map(|x| x.1)
.collect())
}
#[tracing::instrument(skip(self, closure))]
pub async fn get_cached_keys_raw<F, Fut, T, K>(
&self,
namespace: &str,
keys: &[K],
closure: F,
) -> Result<HashMap<K, T>, DatabaseError>
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, DatabaseError>>,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
+ Eq
+ PartialEq
+ Clone
+ DeserializeOwned
+ Serialize
+ Debug,
{
self.get_cached_keys_raw_with_slug(
namespace,
None,
false,
keys,
|ids| async move {
Ok(closure(ids)
.await?
.into_iter()
.map(|(key, val)| (key, (None::<String>, val)))
.collect())
},
)
.await
}
#[tracing::instrument(skip(self, closure))]
pub async fn get_cached_keys_with_slug<F, Fut, T, I, K, S>(
&self,
namespace: &str,
slug_namespace: &str,
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<Vec<T>, DatabaseError>
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, DatabaseError>>,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
+ Hash
+ Eq
+ PartialEq
+ Clone
+ DeserializeOwned
+ Serialize,
S: Display + Clone + DeserializeOwned + Serialize + Debug,
{
Ok(self
.get_cached_keys_raw_with_slug(
namespace,
Some(slug_namespace),
case_sensitive,
keys,
closure,
)
.await?
.into_iter()
.map(|x| x.1)
.collect())
}
#[tracing::instrument(skip(self, closure))]
pub async fn get_cached_keys_raw_with_slug<F, Fut, T, I, K, S>(
&self,
namespace: &str,
slug_namespace: Option<&str>,
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<HashMap<K, T>, DatabaseError>
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, DatabaseError>>,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
+ Hash
+ Eq
+ PartialEq
+ Clone
+ DeserializeOwned
+ Serialize,
S: Display + Clone + DeserializeOwned + Serialize + Debug,
{
let ids = keys
.iter()
.map(|x| (x.to_string(), x.clone()))
.collect::<DashMap<String, I>>();
if ids.is_empty() {
return Ok(HashMap::new());
}
let get_cached_values = |ids: DashMap<String, I>| {
async move {
let slug_ids = if let Some(slug_namespace) = slug_namespace {
async {
let mut connection = self.pool.get().await?;
let args = ids
.iter()
.map(|x| {
format!(
"{}_{slug_namespace}:{}",
self.meta_namespace,
if case_sensitive {
x.value().to_string()
} else {
x.value().to_string().to_lowercase()
}
)
})
.collect::<Vec<_>>();
let mut v = Vec::new();
for chunk in args.chunks(MGET_CHUNK_SIZE) {
let part = cmd("MGET")
.arg(chunk)
.query_async::<Vec<Option<String>>>(
&mut connection,
)
.await?;
v.extend(part.into_iter().flatten());
}
Ok::<_, DatabaseError>(v)
}
.instrument(info_span!("get slug ids"))
.await?
} else {
Vec::new()
};
let mut connection = self.pool.get().await?;
let args = ids
.iter()
.map(|x| x.value().to_string())
.chain(ids.iter().filter_map(|x| {
parse_base62(&x.value().to_string())
.ok()
.map(|x| x.to_string())
}))
.chain(slug_ids)
.map(|x| format!("{}_{namespace}:{x}", self.meta_namespace))
.collect::<Vec<_>>();
let mut cached_values = HashMap::new();
for chunk in args.chunks(MGET_CHUNK_SIZE) {
let part = cmd("MGET")
.arg(chunk)
.query_async::<Vec<Option<Vec<u8>>>>(&mut connection)
.await?;
cached_values.extend(part.into_iter().filter_map(|x| {
x.and_then(|val| {
decode_value::<RedisValue<T, K, S>>(&val)
})
.map(|val| (val.key.clone(), val))
}));
}
Ok::<_, DatabaseError>((cached_values, ids))
}
.instrument(info_span!("get cached values"))
};
let (default_expiry, actual_expiry) = cache_expiries(namespace);
let current_time = Utc::now();
let mut expired_values = HashMap::new();
let (cached_values_raw, ids) = get_cached_values(ids).await?;
let mut cached_values = cached_values_raw
.into_iter()
.filter_map(|(key, val)| {
if Utc.timestamp_opt(val.iat + actual_expiry, 0).unwrap()
< current_time
{
expired_values.insert(val.key.to_string(), val);
None
} else {
let key_str = val.key.to_string();
ids.remove(&key_str);
if let Ok(value) = key_str.parse::<u64>() {
let base62 = to_base62(value);
ids.remove(&base62);
}
if let Some(ref alias) = val.alias {
ids.remove(&alias.to_string());
}
Some((key, val))
}
})
.collect::<HashMap<_, _>>();
let subscribe_ids = DashMap::new();
let mut cache_writers = HashMap::new();
if !ids.is_empty() {
let fetch_ids =
ids.iter().map(|x| x.key().clone()).collect::<Vec<_>>();
fetch_ids.into_iter().for_each(|key| {
let ns_key_value = if case_sensitive {
key.to_lowercase()
} else {
key.clone()
};
let namespaced_key = format!(
"{}_{namespace}:{ns_key_value}",
self.meta_namespace,
);
let either = self.acquire_lock(namespaced_key);
match either {
Either::Left(sentinel) => {
cache_writers.insert(key, sentinel);
}
Either::Right(subscriber) => {
if let Some((key, raw_key)) = ids.remove(&key) {
if let Some(val) = expired_values.remove(&key) {
if let Some(ref alias) = val.alias {
ids.remove(&alias.to_string());
}
if let Ok(value) =
val.key.to_string().parse::<u64>()
{
let base62 = to_base62(value);
ids.remove(&base62);
}
cached_values.insert(val.key.clone(), val);
} else {
subscribe_ids.insert(raw_key, subscriber);
}
}
}
}
});
}
let mut fetch_tasks = Vec::new();
if !ids.is_empty() {
fetch_tasks.push(Either::Left(async {
let fetch_ids =
ids.iter().map(|x| x.value().clone()).collect::<Vec<_>>();
let vals = closure(fetch_ids).await?;
let mut return_values = HashMap::new();
let mut pipe = redis_pipe();
let mut pipe_cmds: usize = 0;
let mut connection = self.pool.get().await?;
// Doesn't need to be atomic
if !vals.is_empty() {
for (key, (slug, value)) in vals {
let value = RedisValue {
key: key.clone(),
iat: Utc::now().timestamp(),
val: value,
alias: slug.clone(),
};
pipe.set_ex(
format!(
"{}_{namespace}:{key}",
self.meta_namespace
),
encode_value(&value)?,
default_expiry as u64,
);
pipe_cmds += 1;
if let Some(slug) = slug {
ids.remove(&slug.to_string());
if let Some(slug_namespace) = slug_namespace {
let actual_slug = if case_sensitive {
slug.to_string()
} else {
slug.to_string().to_lowercase()
};
pipe.set_ex(
format!(
"{}_{slug_namespace}:{}",
self.meta_namespace, actual_slug
),
key.to_string(),
default_expiry as u64,
);
pipe_cmds += 1;
}
}
let key_str = key.to_string();
ids.remove(&key_str);
if let Ok(value) = key_str.parse::<u64>() {
let base62 = to_base62(value);
ids.remove(&base62);
}
return_values.insert(key, value);
if pipe_cmds >= PIPELINE_CHUNK_SIZE {
pipe.query_async::<()>(&mut connection).await?;
pipe = redis_pipe();
pipe_cmds = 0;
}
}
}
if pipe_cmds > 0 {
pipe.query_async::<()>(&mut connection).await?;
}
drop(cache_writers);
Result::<_, DatabaseError>::Ok(return_values)
}));
}
if !subscribe_ids.is_empty() {
fetch_tasks.push(Either::Right(async move {
let mut futures = FuturesUnordered::new();
let len = subscribe_ids.len();
for (key, subscriber) in subscribe_ids {
futures.push(async move {
(
key,
subscriber
.wait_timeout(Duration::from_secs(5))
.await,
)
});
}
let fetch_ids = DashMap::with_capacity(len);
while let Some((key, result)) = futures.next().await {
result?;
fetch_ids.insert(key.to_string(), key);
}
let (return_values, _) = get_cached_values(fetch_ids).await?;
Ok(return_values)
}));
}
if !fetch_tasks.is_empty() {
for map in futures::future::try_join_all(fetch_tasks).await? {
for (key, value) in map {
cached_values.insert(key, value);
}
}
}
Ok(cached_values.into_iter().map(|x| (x.0, x.1.val)).collect())
}
/// Acquire or create a cache lock onto the given key.
fn acquire_lock(
&self,
key: String,
) -> Either<LockSentinel<'_>, util::CacheSubscriber> {
let mut out_writer = None;
let subscriber =
self.cache_list.entry(key.clone()).or_insert_with(|| {
let (writer, subscriber) = util::cache();
out_writer = Some(writer);
subscriber
});
match out_writer {
Some(writer) => Either::Left(LockSentinel {
pool: self,
key,
writer,
}),
None => Either::Right(subscriber.clone()),
}
}
}
struct LockSentinel<'a> {
pool: &'a RedisPool,
key: String,
writer: util::CacheWriter,
}
impl<'a> Drop for LockSentinel<'a> {
fn drop(&mut self) {
self.writer.write();
self.pool.cache_list.remove(&self.key);
}
}
impl RedisConnection {
#[tracing::instrument(skip(self))]
pub async fn set<D>(
&mut self,
namespace: &str,
id: &str,
data: D,
expiry: Option<i64>,
) -> Result<(), DatabaseError>
where
D: ToRedisArgs + Send + Sync + Debug,
{
let mut cmd = cmd("SET");
cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id))
.arg(data)
.arg("EX")
.arg(expiry.unwrap_or(ENV.REDIS_DEFAULT_EXPIRY));
redis_execute::<()>(&mut cmd, &mut self.connection).await?;
Ok(())
}
#[tracing::instrument(skip(self, id, data))]
pub async fn set_serialized<Id, D>(
&mut self,
namespace: &str,
id: Id,
data: D,
expiry: Option<i64>,
) -> Result<(), DatabaseError>
where
Id: Display,
D: serde::Serialize,
{
self.set(namespace, &id.to_string(), encode_value(&data)?, expiry)
.await
}
#[tracing::instrument(skip(self))]
pub async fn get(
&mut self,
namespace: &str,
id: &str,
) -> Result<Option<String>, DatabaseError> {
let mut cmd = cmd("GET");
redis_args(
&mut cmd,
vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
let res = redis_execute(&mut cmd, &mut self.connection).await?;
Ok(res)
}
#[tracing::instrument(skip(self))]
pub async fn get_many(
&mut self,
namespace: &str,
ids: &[String],
) -> Result<Vec<Option<Vec<u8>>>, DatabaseError> {
let mut cmd = cmd("MGET");
redis_args(
&mut cmd,
ids.iter()
.map(|x| format!("{}_{}:{}", self.meta_namespace, namespace, x))
.collect::<Vec<_>>()
.as_slice(),
);
let res = redis_execute(&mut cmd, &mut self.connection).await?;
Ok(res)
}
#[tracing::instrument(skip(self))]
pub async fn get_deserialized<R>(
&mut self,
namespace: &str,
id: &str,
) -> Result<Option<R>, DatabaseError>
where
R: for<'a> serde::Deserialize<'a>,
{
let mut cmd = cmd("GET");
redis_args(
&mut cmd,
vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
let value: Option<Vec<u8>> =
redis_execute(&mut cmd, &mut self.connection).await?;
Ok(value.and_then(|value| decode_value(&value)))
}
#[tracing::instrument(skip(self))]
pub async fn get_many_deserialized<R>(
&mut self,
namespace: &str,
ids: &[String],
) -> Result<Vec<Option<R>>, DatabaseError>
where
R: for<'a> serde::Deserialize<'a>,
{
Ok(self
.get_many(namespace, ids)
.await?
.into_iter()
.map(|value| value.and_then(|value| decode_value::<R>(&value)))
.collect())
}
#[tracing::instrument(skip(self, id))]
pub async fn delete<T1>(
&mut self,
namespace: &str,
id: T1,
) -> Result<(), DatabaseError>
where
T1: Display,
{
let mut cmd = cmd("DEL");
redis_args(
&mut cmd,
vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
redis_execute::<()>(&mut cmd, &mut self.connection).await?;
Ok(())
}
#[tracing::instrument(skip(self, iter))]
pub async fn delete_many(
&mut self,
iter: impl IntoIterator<Item = (&str, Option<String>)>,
) -> Result<(), DatabaseError> {
let mut cmd = cmd("DEL");
let mut any = false;
for (namespace, id) in iter {
if let Some(id) = id {
redis_args(
&mut cmd,
[format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
any = true;
}
}
if any {
redis_execute::<()>(&mut cmd, &mut self.connection).await?;
}
Ok(())
}
#[tracing::instrument(skip(self, value))]
pub async fn lpush(
&mut self,
namespace: &str,
key: &str,
value: impl ToRedisArgs + Send + Sync + Debug,
) -> Result<(), DatabaseError> {
let key = format!("{}_{namespace}:{key}", self.meta_namespace);
cmd("LPUSH")
.arg(key)
.arg(value)
.query_async::<()>(&mut self.connection)
.await?;
Ok(())
}
#[tracing::instrument(skip(self))]
pub async fn brpop(
&mut self,
namespace: &str,
key: &str,
timeout: Option<f64>,
) -> Result<Option<[Vec<u8>; 2]>, DatabaseError> {
let key = format!("{}_{namespace}:{key}", self.meta_namespace);
// a timeout of 0 is infinite
let timeout = timeout.unwrap_or(0.0);
let values = cmd("BRPOP")
.arg(key)
.arg(timeout)
.query_async(&mut self.connection)
.await?;
Ok(values)
}
#[tracing::instrument(skip(self))]
pub async fn incr(
&mut self,
namespace: &str,
id: &str,
) -> Result<Option<u64>, DatabaseError> {
let key = format!("{}_{namespace}:{id}", self.meta_namespace);
let value = cmd("INCR")
.arg(key)
.query_async(&mut self.connection)
.await?;
Ok(value)
}
}
#[derive(Serialize, Deserialize)]
pub struct RedisValue<T, K, S> {
key: K,
alias: Option<S>,
iat: i64,
val: T,
}
impl<T, K, S> RedisValue<T, K, S> {
pub fn value(&self) -> &T {
&self.val
}
}
pub fn redis_args(cmd: &mut util::InstrumentedCmd, args: &[String]) {
for arg in args {
cmd.arg(arg);
}
}
pub async fn redis_execute<T>(
cmd: &mut util::InstrumentedCmd,
redis: &mut deadpool_redis::Connection,
) -> Result<T, deadpool_redis::PoolError>
where
T: redis::FromRedisValue,
{
let res = cmd.query_async::<T>(redis).await?;
Ok(res)
}
apps/labrinth/src/database/redis/util.rs
-160
View File
@@ -1,160 +0,0 @@
use std::fmt::Debug;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use derive_more::{Deref, DerefMut};
use redis::{FromRedisValue, RedisResult, ToRedisArgs};
use tokio::sync::Notify;
use tokio::time::{Duration, timeout};
use tracing::{Instrument, info_span};
use crate::database::models::DatabaseError;
pub fn redis_pipe() -> InstrumentedPipeline {
InstrumentedPipeline {
inner: redis::pipe(),
}
}
#[derive(Clone, Deref, DerefMut)]
pub struct InstrumentedPipeline {
#[deref]
#[deref_mut]
inner: redis::Pipeline,
}
impl InstrumentedPipeline {
pub fn atomic(&mut self) -> &mut Self {
self.inner.atomic();
self
}
#[inline]
pub async fn query_async<T: FromRedisValue>(
&self,
con: &mut impl redis::aio::ConnectionLike,
) -> RedisResult<T> {
self.inner
.query_async(con)
.instrument(info_span!("pipeline.query_async"))
.await
}
}
pub fn cmd(name: &str) -> InstrumentedCmd {
InstrumentedCmd {
inner: redis::cmd(name),
name: name.to_string(),
args: Vec::new(),
}
}
pub struct InstrumentedCmd {
inner: redis::Cmd,
name: String,
args: Vec<String>,
}
impl InstrumentedCmd {
#[inline]
pub fn arg<T: ToRedisArgs + Debug>(&mut self, arg: T) -> &mut Self {
self.args.push(format!("{arg:?}"));
self.inner.arg(arg);
self
}
#[inline]
pub async fn query_async<T: FromRedisValue>(
&self,
con: &mut impl redis::aio::ConnectionLike,
) -> RedisResult<T> {
let span = info_span!(
"cmd.query_async",
// <https://opentelemetry.io/docs/specs/semconv/db/redis/>
db.system.name = "redis",
db.operation.name = self.name,
db.query.text = format!("{} {}", self.name, self.args.join(" ")),
);
self.inner.query_async(con).instrument(span).await
}
}
pub fn cache() -> (CacheWriter, CacheSubscriber) {
let shared = Arc::new(Shared::new());
(
CacheWriter {
shared: shared.clone(),
},
CacheSubscriber { shared },
)
}
pub struct CacheWriter {
shared: Arc<Shared>,
}
impl CacheWriter {
pub fn write(&self) {
self.shared.make_ready();
}
}
#[derive(Clone)]
pub struct CacheSubscriber {
shared: Arc<Shared>,
}
impl CacheSubscriber {
pub async fn wait_timeout(
self,
duration: Duration,
) -> Result<(), DatabaseError> {
timeout(duration, self.shared.wait()).await.map_err(|_| {
DatabaseError::LocalCacheTimeout {
released: 0,
total: 1,
}
})
}
}
struct Shared {
ready: AtomicBool,
// With this implementation's intrusive linked lists, the waiters are stored inline in the future
// so there's no heap allocation per waiter.
wakers: Notify,
}
impl Shared {
fn new() -> Self {
Self {
ready: AtomicBool::new(false),
wakers: Notify::new(),
}
}
fn make_ready(&self) {
self.ready.store(true, Ordering::Release);
self.wakers.notify_waiters();
}
async fn wait(&self) {
let ready = self.ready.load(Ordering::Acquire);
if ready {
return;
}
let notification = self.wakers.notified();
// Don't need to call `enable` as we use notify_waiters
// Prevent race where the writer set the ready bit and notified waiters between the load and registering the waiter
let ready = self.ready.load(Ordering::SeqCst);
if ready {
return;
}
notification.await;
}
}
apps/labrinth/src/env.rs
+47 -13
View File
@@ -5,6 +5,11 @@ use eyre::{Context, eyre};
use rust_decimal::Decimal;
use serde::de::DeserializeOwned;
use xredis::{
CacheLockingStrategy, ReadReplicaStrategy, RedisConnectionType,
RedisTopology,
};
macro_rules! vars {
(
$(
@@ -130,7 +135,49 @@ vars! {
LABRINTH_SUBSCRIPTIONS_KEY: String = "";
RATE_LIMIT_IGNORE_KEY: String = "";
DATABASE_URL: String = "postgresql://labrinth:labrinth@localhost/labrinth";
// Redis
REDIS_TOPOLOGY: RedisTopology = RedisTopology::Standalone;
REDIS_CONNECTION_TYPE: RedisConnectionType = RedisConnectionType::Pooled;
REDIS_CACHE_LOCKING_STRATEGY: CacheLockingStrategy = CacheLockingStrategy::Local;
// URL(s) for Redis. Use comma-separated values for multiple URLs in Cluster topology.
REDIS_URL: String = "redis://localhost";
// Configures the waiting timeout for Redis connection *pools*.
// This doesn't affect the bulk of Redis work in Multiplexed connection type.
REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64;
// Minimum and maximum number of connections when Redis is in Standalone topology.
REDIS_MAX_CONNECTIONS: u32 = 2048u32;
REDIS_MIN_CONNECTIONS: usize = 0usize;
REDIS_DEFAULT_EXPIRY: i64 = 60 * 60 * 12;
REDIS_ACTUAL_EXPIRY: i64 = 60 * 30;
REDIS_VERSION_DEFAULT_EXPIRY: i64 = 60 * 60 * 12;
REDIS_VERSION_ACTUAL_EXPIRY: i64 = 60 * 30;
// Minimum and maximum number of connections when Redis is in Cluster topology, Pooled connection type.
REDIS_CLUSTER_MAX_CONNECTIONS: u32 = 16u32;
REDIS_CLUSTER_MIN_CONNECTIONS: usize = 0usize;
// The maximum number of connections of the Redis blocking pool. There's a blocking pool regardless of topology
// and main connection type.
REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 256u32;
// The encoding format used for Redis cache values.
REDIS_ENCODING_FORMAT: xredis::EncodingFormat = xredis::EncodingFormat::Json;
// The level of LZ4 compression used for Redis cache values. A value of 0 disables compression (supports 1-12)
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
// The compression algorithm used for Redis cache values. Currently only LZ4 is supported.
REDIS_COMPRESSION_ALGORITHM: xredis::Codec = xredis::Codec::Lz4;
// The minimum number of bytes required to trigger compression for Redis cache values.
REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize;
// The minimum savings ratio required to trigger compression for Redis cache values. If the savings ratio is lower than this,
// the compressed payload is discarded and the original payload is stored as-is.
REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64;
REDIS_READ_REPLICA_STRATEGY: ReadReplicaStrategy = ReadReplicaStrategy::Primary;
KAFKA_BOOTSTRAP_SERVERS: StringCsv = StringCsv(vec!["localhost:19092".into()]);
KAFKA_CLIENT_ID: String = "labrinth";
BIND_ADDR: String = "";
@@ -292,19 +339,6 @@ vars! {
READONLY_DATABASE_MIN_CONNECTIONS: u32 = 0u32;
READONLY_DATABASE_MAX_CONNECTIONS: u32 = 1u32;
REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64;
REDIS_MAX_CONNECTIONS: u32 = 10000u32;
REDIS_MIN_CONNECTIONS: usize = 0usize;
REDIS_DEFAULT_EXPIRY: i64 = 60 * 60 * 12;
REDIS_ACTUAL_EXPIRY: i64 = 60 * 30;
REDIS_VERSION_DEFAULT_EXPIRY: i64 = 60 * 60 * 12;
REDIS_VERSION_ACTUAL_EXPIRY: i64 = 60 * 30;
REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Json;
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4;
REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize;
REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64;
SEARCH_OPERATION_TIMEOUT: u64 = 300000u64;
SMTP_REPLY_TO_NAME: String = "";
apps/labrinth/src/lib.rs
+4 -5
View File
@@ -4,12 +4,12 @@ use std::sync::Arc;
use std::time::Duration;
use actix_web::web;
use database::redis::RedisPool;
use queue::{
analytics::AnalyticsQueue, email::EmailQueue, payouts::PayoutsQueue,
session::AuthQueue, socket::ActiveSockets,
};
use tracing::{debug, info, warn};
use xredis::RedisPool;
extern crate clickhouse as clickhouse_crate;
use clickhouse_crate::Client;
@@ -26,7 +26,7 @@ use crate::util::archon::ArchonClient;
use crate::util::http::HttpClient;
use crate::util::ratelimit::{AsyncRateLimiter, GCRAParameters};
use crate::util::tiltify::TiltifyClient;
use sync::friends::handle_pubsub;
use sync::friends::{FRIENDS_CHANNEL_NAME, handle_pubsub};
use url::Url;
use webauthn_rs::{Webauthn, WebauthnBuilder};
@@ -290,11 +290,10 @@ pub fn app_setup(
{
let pool = pool.clone();
let redis_client = redis::Client::open(redis_pool.url.clone()).unwrap();
let pubsub_messages = redis_pool.subscribe(FRIENDS_CHANNEL_NAME);
let sockets = active_sockets.clone();
actix_rt::spawn(async move {
let pubsub = redis_client.get_async_pubsub().await.unwrap();
handle_pubsub(pubsub, pool, sockets).await;
handle_pubsub(pubsub_messages, pool, sockets).await;
});
}
apps/labrinth/src/main.rs
+2 -2
View File
@@ -7,7 +7,7 @@ use actix_web_prom::PrometheusMetricsBuilder;
use clap::Parser;
use labrinth::background_task::BackgroundTask;
use labrinth::database::redis::RedisPool;
use labrinth::database::redis;
use labrinth::env::ENV;
use labrinth::file_hosting::{FileHost, FileHostKind, S3BucketConfig, S3Host};
use labrinth::queue::email::EmailQueue;
@@ -110,7 +110,7 @@ async fn app() -> std::io::Result<()> {
.expect("Database connection failed");
// Redis connector
let redis_pool = RedisPool::new("");
let redis_pool = redis::from_env("").await;
let storage_backend = ENV.STORAGE_BACKEND;
let file_host: Arc<dyn FileHost> = match storage_backend {
apps/labrinth/src/models/exp/project.rs
+16 -18
View File
@@ -2,12 +2,10 @@ use eyre::Result;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use validator::Validate;
use xredis::RedisPool;
use crate::{
database::{
models::{DBProjectId, DBVersionId},
redis::RedisPool,
},
database::models::{DBProjectId, DBVersionId},
models::{
exp::{
component::{
@@ -257,14 +255,14 @@ pub async fn fetch_query_context(
{
HashMap::new()
} else {
redis
.get_many_deserialized::<minecraft::JavaServerPing>(
server_ping::REDIS_NAMESPACE,
&minecraft_java_server_pings
let ping_keys = minecraft_java_server_pings
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>(),
)
.map(|project_id| {
redis.key().entity(server_ping::REDIS_NAMESPACE, project_id)
})
.collect::<Vec<_>>();
redis
.get_many_deserialized::<minecraft::JavaServerPing>(&ping_keys)
.await?
.into_iter()
.enumerate()
@@ -280,14 +278,14 @@ pub async fn fetch_query_context(
let minecraft_server_analytics = if minecraft_server_analytics.is_empty() {
HashMap::new()
} else {
redis
.get_many_deserialized::<MinecraftServerAnalytics>(
MINECRAFT_SERVER_ANALYTICS,
&minecraft_server_analytics
let analytics_keys = minecraft_server_analytics
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>(),
)
.map(|project_id| {
redis.key().entity(MINECRAFT_SERVER_ANALYTICS, project_id)
})
.collect::<Vec<_>>();
redis
.get_many_deserialized::<MinecraftServerAnalytics>(&analytics_keys)
.await?
.into_iter()
.enumerate()
apps/labrinth/src/models/v2/projects.rs
+1 -1
View File
@@ -4,7 +4,6 @@ use std::collections::HashMap;
use super::super::ids::OrganizationId;
use crate::database::models::{DatabaseError, version_item};
use crate::database::redis::RedisPool;
use crate::models::ids::{ProjectId, TeamId, ThreadId, VersionId};
use crate::models::projects::{
Dependency, License, Link, Loader, ModeratorMessage, MonetizationStatus,
@@ -16,6 +15,7 @@ use chrono::{DateTime, Utc};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use validator::Validate;
use xredis::RedisPool;
/// A project returned from the API
#[derive(Serialize, Deserialize, Clone, utoipa::ToSchema)]
apps/labrinth/src/queue/analytics/cache.rs
+6 -11
View File
@@ -1,4 +1,5 @@
use std::collections::HashMap;
use xredis::RedisPool;
use const_format::formatcp;
use eyre::{Result, eyre};
@@ -7,13 +8,11 @@ use sqlx::PgPool;
use tracing::{debug, info};
use crate::{
database::{DBProject, redis::RedisPool},
models::ids::ProjectId,
routes::analytics::MINECRAFT_SERVER_PLAYS,
util::error::Context,
database::DBProject, models::ids::ProjectId,
routes::analytics::MINECRAFT_SERVER_PLAYS, util::error::Context,
};
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v1";
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v3";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MinecraftServerAnalytics {
@@ -116,13 +115,9 @@ pub async fn cache_analytics(
};
debug!("Caching analytics for {project_id}: {analytics:?}");
let key = redis.key().entity(MINECRAFT_SERVER_ANALYTICS, project_id);
redis
.set_serialized(
MINECRAFT_SERVER_ANALYTICS,
project_id.to_string(),
analytics,
None,
)
.set_serialized(&key, analytics, None)
.await
.wrap_err_with(|| {
eyre!("failed to set analytics for project '{project_id}'")
apps/labrinth/src/queue/analytics/mod.rs
+54 -80
View File
@@ -1,21 +1,19 @@
use crate::database::PgPool;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::analytics::{
AffiliateCodeClick, Download, MinecraftServerPlay, PageView, Playtime,
};
use crate::routes::ApiError;
use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
use dashmap::{DashMap, DashSet};
use redis::cmd;
use std::collections::HashMap;
use tracing::trace;
use xredis::RedisPool;
pub mod cache;
const DOWNLOADS_NAMESPACE: &str = "downloads:v1";
const VIEWS_NAMESPACE: &str = "views:v1";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v1";
const DOWNLOADS_NAMESPACE: &str = "downloads:v3";
const VIEWS_NAMESPACE: &str = "views:v3";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v3";
const MINECRAFT_SERVER_PLAYS_EXPIRY: u64 = 86_400; // 24 hours
const MINECRAFT_SERVER_PLAYS_LIMIT: u32 = 5;
@@ -133,29 +131,22 @@ impl AnalyticsQueue {
raw_plays.insert(index, play);
}
let mut redis =
redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let results = cmd("MGET")
.arg(
plays_keys
let redis_keys = plays_keys
.iter()
.map(|x| {
format!(
"{}:{}-{}",
MINECRAFT_SERVER_PLAYS_NAMESPACE, x.0, x.1
.map(|key| {
let logical_key = format!("{}-{}", key.0, key.1);
redis.key().with_slot(
MINECRAFT_SERVER_PLAYS_NAMESPACE,
&logical_key,
&logical_key,
)
})
.collect::<Vec<_>>(),
)
.query_async::<Vec<Option<u32>>>(&mut redis)
.await
.map_err(DatabaseError::CacheError)?;
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut pipe = redis::pipe();
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
for (idx, count) in results.into_iter().enumerate() {
let key = &plays_keys[idx];
let new_count = if let Some(count) = count {
if count >= MINECRAFT_SERVER_PLAYS_LIMIT {
raw_plays.remove(&idx);
@@ -166,18 +157,15 @@ impl AnalyticsQueue {
1
};
pipe.atomic().set_ex(
format!(
"{}:{}-{}",
MINECRAFT_SERVER_PLAYS_NAMESPACE, key.0, key.1
),
let key = &redis_keys[idx];
redis_connection
.set(
key,
new_count,
MINECRAFT_SERVER_PLAYS_EXPIRY,
);
Some(MINECRAFT_SERVER_PLAYS_EXPIRY as i64),
)
.await?;
}
pipe.query_async::<()>(&mut *redis)
.await
.map_err(DatabaseError::CacheError)?;
let mut plays = client
.insert::<MinecraftServerPlay>(MINECRAFT_SERVER_PLAYS)
@@ -199,24 +187,22 @@ impl AnalyticsQueue {
raw_views.push((views, true));
}
let mut redis =
redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let results = cmd("MGET")
.arg(
views_keys
let redis_keys = views_keys
.iter()
.map(|x| format!("{}:{}-{}", VIEWS_NAMESPACE, x.0, x.1))
.collect::<Vec<_>>(),
.map(|key| {
let logical_key = format!("{}-{}", key.0, key.1);
redis.key().with_slot(
VIEWS_NAMESPACE,
&logical_key,
&logical_key,
)
.query_async::<Vec<Option<u32>>>(&mut redis)
.await
.map_err(DatabaseError::CacheError)?;
})
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut pipe = redis::pipe();
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
for (idx, count) in results.into_iter().enumerate() {
let key = &views_keys[idx];
let new_count =
if let Some((views, monetized)) = raw_views.get_mut(idx) {
if let Some(count) = count {
@@ -237,15 +223,11 @@ impl AnalyticsQueue {
1
};
pipe.atomic().set_ex(
format!("{}:{}-{}", VIEWS_NAMESPACE, key.0, key.1),
new_count,
6 * 60 * 60,
);
let key = &redis_keys[idx];
redis_connection
.set(key, new_count, Some(6 * 60 * 60))
.await?;
}
pipe.query_async::<()>(&mut *redis)
.await
.map_err(DatabaseError::CacheError)?;
let mut views = client.insert::<PageView>("views").await?;
@@ -274,26 +256,22 @@ impl AnalyticsQueue {
raw_downloads.insert(index, download);
}
let mut redis =
redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let results = cmd("MGET")
.arg(
downloads_keys
let redis_keys = downloads_keys
.iter()
.map(|x| {
format!("{}:{}-{}", DOWNLOADS_NAMESPACE, x.0, x.1)
})
.collect::<Vec<_>>(),
.map(|key| {
let logical_key = format!("{}-{}", key.0, key.1);
redis.key().with_slot(
DOWNLOADS_NAMESPACE,
&logical_key,
&logical_key,
)
.query_async::<Vec<Option<u32>>>(&mut redis)
.await
.map_err(DatabaseError::CacheError)?;
})
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut pipe = redis::pipe();
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
for (idx, count) in results.into_iter().enumerate() {
let key = &downloads_keys[idx];
let new_count = if let Some(count) = count {
if count > 5 {
raw_downloads.remove(&idx);
@@ -305,15 +283,11 @@ impl AnalyticsQueue {
1
};
pipe.atomic().set_ex(
format!("{}:{}-{}", DOWNLOADS_NAMESPACE, key.0, key.1),
new_count,
6 * 60 * 60,
);
let key = &redis_keys[idx];
redis_connection
.set(key, new_count, Some(6 * 60 * 60))
.await?;
}
pipe.query_async::<()>(&mut *redis)
.await
.map_err(DatabaseError::CacheError)?;
let mut transaction = pool.begin().await?;
let mut downloads = client.insert::<Download>("downloads").await?;
apps/labrinth/src/queue/billing.rs
+1 -1
View File
@@ -10,7 +10,6 @@ use crate::database::models::{DatabaseError, ids::*};
use crate::database::models::{
product_item, user_subscription_item, users_redeemals,
};
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::models::billing::{
@@ -35,6 +34,7 @@ use std::str::FromStr;
use std::time::Instant;
use stripe::{self, Currency};
use tracing::{debug, error, info, warn};
use xredis::RedisPool;
/// Updates charges which need to have their tax amount updated. This is done within a timer to avoid reaching
/// Anrok API limits.
apps/labrinth/src/queue/email.rs
+1 -1
View File
@@ -3,7 +3,6 @@ use crate::database::models::notification_item::DBNotification;
use crate::database::models::notifications_deliveries_item::DBNotificationDelivery;
use crate::database::models::notifications_template_item::NotificationTemplate;
use crate::database::models::user_item::DBUser;
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::models::notifications::{NotificationBody, NotificationType};
@@ -23,6 +22,7 @@ use thiserror::Error;
use tokio::sync::Mutex as TokioMutex;
use tokio::sync::Semaphore;
use tracing::{error, info, instrument, warn};
use xredis::RedisPool;
const EMAIL_RETRY_DELAY_SECONDS: i64 = 10;
apps/labrinth/src/queue/email/templates.rs
+1 -1
View File
@@ -7,7 +7,6 @@ use crate::database::models::notifications_template_item::{
use crate::database::models::{
DBOrganization, DBProject, DBUser, DatabaseError,
};
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::v3::notifications::NotificationBody;
use crate::routes::ApiError;
@@ -20,6 +19,7 @@ use sqlx::query;
use std::collections::HashMap;
use std::time::Duration;
use tracing::{error, warn};
use xredis::RedisPool;
const USER_NAME: &str = "user.name";
const USER_EMAIL: &str = "user.email";
apps/labrinth/src/queue/file_scan.rs
+2 -1
View File
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::io::{Cursor, Read};
use std::sync::Arc;
use xredis::RedisPool;
use chrono::Utc;
use eyre::{Result, eyre};
@@ -17,7 +18,7 @@ use crate::database::models::ids::{
};
use crate::database::models::moderation_external_item::ExternalLicense;
use crate::database::models::{DBFileId, DBUserId, DBVersion};
use crate::database::{PgPool, PgTransaction, redis::RedisPool};
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::file_hosting::{FileHost, FileHostPublicity};
use crate::models::error::ApiError;
apps/labrinth/src/queue/payouts/mod.rs
+1 -1
View File
@@ -1,6 +1,5 @@
use crate::database::models::notification_item::NotificationBuilder;
use crate::database::models::payouts_values_notifications;
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::models::payouts::{
@@ -31,6 +30,7 @@ use sqlx::postgres::PgQueryResult;
use std::collections::HashMap;
use tokio::sync::RwLock;
use tracing::{error, info, warn};
use xredis::RedisPool;
mod affiliate;
pub mod flow;
apps/labrinth/src/queue/server_ping.rs
+21 -17
View File
@@ -1,6 +1,5 @@
use crate::database::DBProject;
use crate::database::models::DBProjectId;
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::exp;
use crate::models::ids::ProjectId;
@@ -18,6 +17,7 @@ use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use tokio::task::JoinSet;
use tracing::{Instrument, info, info_span, trace, warn};
use xredis::RedisPool;
pub struct ServerPingQueue {
pub db: PgPool,
@@ -26,9 +26,9 @@ pub struct ServerPingQueue {
pub incremental_search_queue: IncrementalSearchQueue,
}
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v1";
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v3";
pub const REDIS_FAILURE_NAMESPACE: &str =
"minecraft_java_server_ping_failures:v1";
"minecraft_java_server_ping_failures:v3";
pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings";
impl ServerPingQueue {
@@ -111,6 +111,12 @@ impl ServerPingQueue {
for (project_id, ping) in &pings {
let data = ping.data.as_ref();
let ping_key =
self.redis.key().entity(REDIS_NAMESPACE, project_id);
let failure_key = self
.redis
.key()
.entity(REDIS_FAILURE_NAMESPACE, project_id);
let row = ServerPingRecord {
recorded: ping.when.timestamp_nanos_opt().unwrap()
@@ -134,13 +140,13 @@ impl ServerPingQueue {
// ping succeeded; immediately update its online status in redis
redis
.set_serialized(REDIS_NAMESPACE, project_id, ping, None)
.set_serialized(&ping_key, ping, None)
.await
.wrap_err("failed to set redis key")?;
updated_project = true;
redis
.delete(REDIS_FAILURE_NAMESPACE, project_id)
.delete(&failure_key)
.await
.wrap_err("failed to delete failure count")?;
} else {
@@ -148,7 +154,7 @@ impl ServerPingQueue {
// otherwise, just add to the fail counter
let failure_count = redis
.incr(REDIS_FAILURE_NAMESPACE, &project_id.to_string())
.incr(&failure_key)
.await
.wrap_err("failed to increment failure count")?;
@@ -156,12 +162,7 @@ impl ServerPingQueue {
&& count >= ENV.SERVER_PING_MAX_FAIL_COUNT
{
redis
.set_serialized(
REDIS_NAMESPACE,
project_id,
ping,
None,
)
.set_serialized(&ping_key, ping, None)
.await
.wrap_err(
"failed to set failed ping record in redis",
@@ -246,14 +247,17 @@ impl ServerPingQueue {
// and if we do miss an entry that we shouldn't, we just ping it again
let all_project_ids = all_server_projects
.iter()
.map(|row| ProjectId::from(DBProjectId(row.id)).to_string())
.map(|row| ProjectId::from(DBProjectId(row.id)))
.collect::<Vec<_>>();
let ping_keys = all_project_ids
.iter()
.map(|project_id| {
self.redis.key().entity(REDIS_NAMESPACE, project_id)
})
.collect::<Vec<_>>();
let all_server_last_pings = redis
.get_many_deserialized::<exp::minecraft::JavaServerPing>(
REDIS_NAMESPACE,
&all_project_ids,
)
.get_many_deserialized::<exp::minecraft::JavaServerPing>(&ping_keys)
.await
.wrap_err("failed to fetch server project last pings")?;
apps/labrinth/src/queue/session.rs
+1 -1
View File
@@ -3,13 +3,13 @@ use crate::database::models::session_item::DBSession;
use crate::database::models::{
DBOAuthAccessTokenId, DBPatId, DBSessionId, DBUserId, DatabaseError,
};
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::routes::internal::session::SessionMetadata;
use chrono::Utc;
use itertools::Itertools;
use std::collections::{HashMap, HashSet};
use tokio::sync::Mutex;
use xredis::RedisPool;
pub struct AuthQueue {
session_queue: Mutex<HashMap<DBSessionId, SessionMetadata>>,
apps/labrinth/src/routes/analytics.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use crate::auth::get_user_from_headers;
use crate::database::PgPool;
use crate::database::models::DBProject;
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::analytics::{MinecraftServerPlay, PageView, Playtime};
use crate::models::ids::ProjectId;
@@ -22,6 +21,7 @@ use std::sync::Arc;
use tracing::trace;
use url::Url;
use uuid::Uuid;
use xredis::RedisPool;
pub const FILTERED_HEADERS: &[&str] = &[
"authorization",
apps/labrinth/src/routes/internal/admin.rs
+1 -1
View File
@@ -1,6 +1,5 @@
use crate::auth::validate::get_user_record_from_bearer_token;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::analytics::{Download, DownloadReason};
use crate::models::ids::{ProjectId, VersionId};
use crate::models::pats::Scopes;
@@ -22,6 +21,7 @@ use std::net::Ipv4Addr;
use std::str::FromStr;
use std::sync::Arc;
use tracing::trace;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/internal/affiliate.rs
+2 -4
View File
@@ -1,13 +1,11 @@
use std::{collections::HashMap, net::Ipv4Addr, sync::Arc};
use xredis::RedisPool;
use crate::database::PgPool;
use crate::env::ENV;
use crate::{
auth::get_user_from_headers,
database::{
models::{DBAffiliateCode, DBAffiliateCodeId, DBUser, DBUserId},
redis::RedisPool,
},
database::models::{DBAffiliateCode, DBAffiliateCodeId, DBUser, DBUserId},
models::{
analytics::AffiliateCodeClick, ids::AffiliateCodeId, pats::Scopes,
users::Badges, v3::affiliate_code::AffiliateCode,
apps/labrinth/src/routes/internal/attribution.rs
+1 -1
View File
@@ -12,7 +12,6 @@ use crate::database::models::{
generate_attribution_group_id,
},
};
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::ids::{FileId, ProjectId, VersionId};
use crate::models::pats::Scopes;
@@ -27,6 +26,7 @@ use crate::queue::moderation::ApprovalType;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::util::error::Context;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(list)
apps/labrinth/src/routes/internal/billing.rs
+1 -1
View File
@@ -11,7 +11,6 @@ use crate::database::models::{
DBAffiliateCodeId, charge_item, generate_charge_id, product_item,
user_subscription_item,
};
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::models::billing::{
@@ -42,6 +41,7 @@ use stripe::{
Webhook,
};
use tracing::warn;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/internal/billing/payments.rs
+1 -1
View File
@@ -3,12 +3,12 @@ use crate::database::models::{
generate_charge_id, generate_user_subscription_id, product_item,
products_tax_identifier_item, user_subscription_item,
};
use crate::database::redis::RedisPool;
use crate::models::ids::*;
use crate::models::v3::billing::SubscriptionStatus;
use crate::models::v3::users::User;
use crate::routes::ApiError;
use crate::util::anrok;
use xredis::RedisPool;
use crate::database::PgPool;
use ariadne::ids::base62_impl::to_base62;
apps/labrinth/src/routes/internal/campaign.rs
+7 -8
View File
@@ -10,6 +10,7 @@ use sha2::Sha256;
use std::collections::HashSet;
use tracing::{debug, info, warn};
use uuid::Uuid;
use xredis::RedisPool;
use crate::{
database::{
@@ -18,7 +19,6 @@ use crate::{
DBCampaignDonationId, DBUser, DBUserId,
generate_campaign_donation_id,
},
redis::RedisPool,
},
env::ENV,
models::payouts::TremendousForexResponse,
@@ -68,7 +68,7 @@ pub struct CampaignInfo {
cached_at: DateTime<Utc>,
}
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v1";
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v3";
const CAMPAIGN_INFO_CACHE_STALE_SECONDS: i64 = 15 * 60;
const CAMPAIGN_INFO_CACHE_TTL_SECONDS: i64 = 24 * 60 * 60;
@@ -324,12 +324,12 @@ pub async fn pride_26(
.connect()
.await
.wrap_internal_err("connecting to redis")?;
let cache_key = redis
.key()
.entity(CAMPAIGN_INFO_CACHE_NAMESPACE, campaign_id);
let cached = redis_connection
.get_deserialized::<CampaignInfo>(
CAMPAIGN_INFO_CACHE_NAMESPACE,
campaign_id,
)
.get_deserialized::<CampaignInfo>(&cache_key)
.await
.wrap_internal_err("getting cached campaign info")?;
@@ -383,8 +383,7 @@ pub async fn pride_26(
redis_connection
.set_serialized(
CAMPAIGN_INFO_CACHE_NAMESPACE,
campaign_id,
&cache_key,
&campaign_info,
Some(CAMPAIGN_INFO_CACHE_TTL_SECONDS),
)
apps/labrinth/src/routes/internal/delphi/mod.rs
+4 -6
View File
@@ -1,4 +1,5 @@
use std::{collections::HashMap, fmt::Write, time::Instant};
use xredis::RedisPool;
use crate::env::ENV;
use crate::{database::PgPool, util::http::HttpClient};
@@ -11,17 +12,14 @@ use tracing::info;
use crate::{
auth::check_is_moderator_from_headers,
database::{
models::{
database::models::{
DBFileId, DBProjectId, DelphiReportId, DelphiReportIssueDetailsId,
DelphiReportIssueId,
delphi_report_item::{
DBDelphiReport, DBDelphiReportIssue, DelphiSeverity,
DelphiStatus, ReportIssueDetail,
DBDelphiReport, DBDelphiReportIssue, DelphiSeverity, DelphiStatus,
ReportIssueDetail,
},
},
redis::RedisPool,
},
models::{
ids::{ProjectId, VersionId},
pats::Scopes,
apps/labrinth/src/routes/internal/external_notifications.rs
+1 -1
View File
@@ -6,7 +6,6 @@ use crate::database::models::ids::{DBNotificationId, DBUserId};
use crate::database::models::notification_item::DBNotification;
use crate::database::models::notification_item::NotificationBuilder;
use crate::database::models::user_item::DBUser;
use crate::database::redis::RedisPool;
use crate::models::notifications::NotificationDeliveryStatus;
use crate::models::users::Role;
use crate::models::v3::notifications::{Notification, NotificationBody};
@@ -27,6 +26,7 @@ use ariadne::ids::UserId;
use eyre::eyre;
use lettre::message::Mailbox;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(create)
apps/labrinth/src/routes/internal/flows.rs
+8 -14
View File
@@ -8,7 +8,6 @@ use crate::database::models::flow_item::DBFlow;
use crate::database::models::notification_item::NotificationBuilder;
use crate::database::models::session_item::DBSession;
use crate::database::models::{DBPasskey, DBPasskeyId, DBUser, DBUserId};
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::file_hosting::{FileHost, FileHostPublicity};
use crate::models::error::ApiError as ApiErrorResponse;
@@ -59,6 +58,7 @@ use webauthn_rs::prelude::{
PublicKeyCredential, RegisterPublicKeyCredential, RequestChallengeResponse,
Webauthn, WebauthnError,
};
use xredis::RedisPool;
use zxcvbn::Score;
/// Sourced from <https://github.com/disposable-email-domains/disposable-email-domains>.
@@ -2213,15 +2213,15 @@ async fn validate_2fa_code(
)
.map_err(|_| AuthenticationError::InvalidCredentials)?;
const TOTP_NAMESPACE: &str = "used_totp:v1";
const TOTP_NAMESPACE: &str = "used_totp:v3";
let mut conn = redis.connect().await?;
let logical_key = format!("{}-{}", input, user_id.0);
let key = redis
.key()
.with_slot(TOTP_NAMESPACE, &logical_key, &logical_key);
// Check if TOTP has already been used
if conn
.get(TOTP_NAMESPACE, &format!("{}-{}", input, user_id.0))
.await?
.is_some()
{
if conn.get(&key).await?.is_some() {
return Err(AuthenticationError::InvalidCredentials);
}
@@ -2229,13 +2229,7 @@ async fn validate_2fa_code(
.check_current(input.as_str())
.map_err(|_| AuthenticationError::InvalidCredentials)?
{
conn.set(
TOTP_NAMESPACE,
&format!("{}-{}", input, user_id.0),
"",
Some(60),
)
.await?;
conn.set(&key, "", Some(60)).await?;
Ok(true)
} else if allow_backup {
apps/labrinth/src/routes/internal/gdpr.rs
+1 -1
View File
@@ -1,10 +1,10 @@
use crate::auth::get_user_from_headers;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use actix_web::{HttpRequest, HttpResponse, post, web};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(web::scope("/gdpr").service(export));
apps/labrinth/src/routes/internal/gotenberg.rs
+6 -12
View File
@@ -7,15 +7,15 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::trace;
use crate::database::redis::RedisPool;
use crate::models::ids::PayoutId;
use crate::routes::ApiError;
use crate::util::error::Context;
use crate::util::gotenberg::{
GeneratedPdfType, MODRINTH_GENERATED_PDF_TYPE, MODRINTH_PAYMENT_ID,
PAYMENT_STATEMENTS_NAMESPACE,
payment_statement_key,
};
use crate::util::guards::internal_network_guard;
use xredis::RedisPool;
/// Document generated by Gotenberg and returned to us.
#[derive(Debug, Serialize, Deserialize)]
@@ -63,6 +63,7 @@ pub async fn success_callback(
.0
.0;
let response_key = payment_statement_key(redis.get_ref(), &payout_id);
let mut redis = redis
.connect()
.await
@@ -77,11 +78,7 @@ pub async fn success_callback(
.wrap_internal_err("failed to serialize Redis document response")?;
redis
.lpush(
PAYMENT_STATEMENTS_NAMESPACE,
&payout_id.to_string(),
&redis_msg,
)
.lpush(&response_key, &redis_msg)
.await
.wrap_internal_err("failed to send document over Redis")?;
@@ -134,6 +131,7 @@ pub async fn error_callback(
.wrap_request_err("no payout ID for document")?
.0
.0;
let response_key = payment_statement_key(redis.get_ref(), &payout_id);
let mut redis = redis
.connect()
.await
@@ -146,11 +144,7 @@ pub async fn error_callback(
.wrap_internal_err("failed to serialize Redis error response")?;
redis
.lpush(
PAYMENT_STATEMENTS_NAMESPACE,
&payout_id.to_string(),
&redis_msg,
)
.lpush(&response_key, &redis_msg)
.await
.wrap_internal_err("failed to send error over Redis")?;
apps/labrinth/src/routes/internal/medal.rs
+1 -1
View File
@@ -8,10 +8,10 @@ use tracing::warn;
use crate::database::models::users_redeemals::{
Offer, RedeemalLookupFields, Status, UserRedeemal,
};
use crate::database::redis::RedisPool;
use crate::queue::billing::try_process_user_redeemal;
use crate::routes::ApiError;
use crate::util::guards::medal_key_guard;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(web::scope("/medal").service(verify).service(redeem));
apps/labrinth/src/routes/internal/moderation/external_license.rs
+1 -1
View File
@@ -7,11 +7,11 @@ use serde::{Deserialize, Serialize};
use crate::database::PgPool;
use crate::database::models::ids::DBUserId;
use crate::database::models::moderation_external_item::ExternalLicense;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::queue::moderation::ApprovalType;
use crate::routes::ApiError;
use crate::{auth::check_is_moderator_from_headers, queue::session::AuthQueue};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(search)
apps/labrinth/src/routes/internal/moderation/mod.rs
+1 -1
View File
@@ -4,7 +4,6 @@ use crate::database;
use crate::database::PgPool;
use crate::database::models::DBModerationLock;
use crate::database::models::moderation_external_item;
use crate::database::redis::RedisPool;
use crate::models::ids::{OrganizationId, ProjectId};
use crate::models::projects::{ProjectStatus, VersionStatus};
use crate::queue::moderation::{ApprovalType, IdentifiedFile, MissingMetadata};
@@ -20,6 +19,7 @@ use chrono::{DateTime, Utc};
use eyre::eyre;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use xredis::RedisPool;
pub mod external_license;
mod ownership;
apps/labrinth/src/routes/internal/moderation/ownership.rs
+1 -1
View File
@@ -1,11 +1,11 @@
use crate::database::PgPool;
use crate::database::models::{DBOrganization, DBTeamId, DBTeamMember, DBUser};
use crate::database::redis::RedisPool;
use crate::models::ids::OrganizationId;
use crate::routes::internal::moderation::Ownership;
use crate::util::error::Context;
use ariadne::ids::UserId;
use eyre::eyre;
use xredis::RedisPool;
/// Fetches ownership information for multiple projects efficiently
pub async fn get_projects_ownership(
apps/labrinth/src/routes/internal/moderation/tech_review.rs
+1 -1
View File
@@ -1,4 +1,5 @@
use std::{collections::HashMap, fmt};
use xredis::RedisPool;
use crate::database::PgPool;
use actix_web::{HttpRequest, get, patch, post, put, web};
@@ -22,7 +23,6 @@ use crate::{
thread_item::ThreadMessageBuilder,
version_item::VersionQueryResult,
},
redis::RedisPool,
},
models::{
ids::{FileId, ProjectId, ThreadId, VersionId},
apps/labrinth/src/routes/internal/moderation/tech_review/global.rs
+1 -1
View File
@@ -1,6 +1,7 @@
use actix_web::{HttpRequest, post, web};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
use crate::{
auth::check_is_moderator_from_headers,
@@ -11,7 +12,6 @@ use crate::{
DelphiReportIssueId,
delphi_report_item::{DelphiSeverity, DelphiStatus},
},
redis::RedisPool,
},
models::{
ids::{FileId, ProjectId, VersionId},
apps/labrinth/src/routes/internal/pats.rs
+1 -1
View File
@@ -4,7 +4,6 @@ use crate::database::models::generate_pat_id;
use crate::auth::get_user_from_headers;
use crate::routes::ApiError;
use crate::database::redis::RedisPool;
use actix_web::web::{self, Data};
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post};
use chrono::{DateTime, Utc};
@@ -12,6 +11,7 @@ use rand::Rng;
use rand::distributions::Alphanumeric;
use rand_chacha::ChaCha20Rng;
use rand_chacha::rand_core::SeedableRng;
use xredis::RedisPool;
use crate::database::PgPool;
use crate::database::models::notification_item::NotificationBuilder;
apps/labrinth/src/routes/internal/server_ping.rs
+2 -1
View File
@@ -1,11 +1,12 @@
use std::time::Duration;
use xredis::RedisPool;
use actix_web::{HttpRequest, post, web};
use serde::{Deserialize, Serialize};
use crate::{
auth::get_user_from_headers,
database::{PgPool, redis::RedisPool},
database::PgPool,
models::pats::Scopes,
queue::{server_ping, session::AuthQueue},
routes::ApiError,
apps/labrinth/src/routes/internal/session.rs
+1 -1
View File
@@ -3,7 +3,6 @@ use crate::auth::{AuthenticationError, get_user_from_headers};
use crate::database::models::DBUserId;
use crate::database::models::session_item::DBSession;
use crate::database::models::session_item::SessionBuilder;
use crate::database::redis::RedisPool;
use crate::database::{PgPool, PgTransaction};
use crate::env::ENV;
use crate::models::pats::Scopes;
@@ -18,6 +17,7 @@ use rand::distributions::Alphanumeric;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use woothee::parser::Parser;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/internal/statuses.rs
+4 -8
View File
@@ -3,7 +3,6 @@ use crate::auth::validate::get_user_record_from_bearer_token;
use crate::database::PgPool;
use crate::database::models::friend_item::DBFriend;
use crate::database::models::notification_item::DBNotification;
use crate::database::redis::RedisPool;
use crate::models::notifications::{Notification, NotificationBody};
use crate::models::pats::Scopes;
use crate::models::users::User;
@@ -28,12 +27,12 @@ use chrono::Utc;
use either::Either;
use futures_util::future::select;
use futures_util::{StreamExt, TryStreamExt};
use redis::AsyncCommands;
use serde::Deserialize;
use std::pin::pin;
use std::sync::atomic::Ordering;
use tokio::sync::oneshot::error::TryRecvError;
use tokio::time::{Duration, sleep};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(ws_init);
@@ -393,13 +392,10 @@ pub async fn broadcast_friends_message(
redis: &RedisPool,
message: RedisFriendsMessage,
) -> Result<(), crate::database::models::DatabaseError> {
let _: () = redis
.pool
.get()
.await?
redis
.publish(FRIENDS_CHANNEL_NAME, message)
.await?;
Ok(())
.await
.map_err(Into::into)
}
pub async fn broadcast_to_local_friends(
apps/labrinth/src/routes/maven.rs
+1 -1
View File
@@ -6,7 +6,6 @@ use crate::database::models::project_item::ProjectQueryResult;
use crate::database::models::version_item::{
FileQueryResult, VersionQueryResult,
};
use crate::database::redis::RedisPool;
use crate::models::ids::{ProjectId, VersionId};
use crate::models::pats::Scopes;
use crate::models::projects::FileType;
@@ -16,6 +15,7 @@ use crate::{auth::get_user_from_headers, database};
use actix_web::{HttpRequest, HttpResponse, get, route, web};
use quick_xml::escape::escape;
use std::collections::HashSet;
use xredis::RedisPool;
use yaserde::YaSerialize;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
apps/labrinth/src/routes/mod.rs
+6
View File
@@ -267,6 +267,12 @@ pub enum ApiError {
},
}
impl From<xredis::Error> for ApiError {
fn from(error: xredis::Error) -> Self {
Self::Database(error.into())
}
}
impl ApiError {
pub fn delphi(err: impl Into<eyre::Error>) -> Self {
Self::Delphi(err.into())
apps/labrinth/src/routes/updates.rs
+1 -1
View File
@@ -10,10 +10,10 @@ use crate::auth::checks::{filter_visible_versions, is_visible_project};
use crate::auth::get_user_from_headers;
use crate::database;
use crate::database::models::legacy_loader_fields::MinecraftGameVersion;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::models::projects::VersionType;
use crate::queue::session::AuthQueue;
use xredis::RedisPool;
use super::ApiError;
apps/labrinth/src/routes/v2/notifications.rs
+1 -1
View File
@@ -1,5 +1,4 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::ids::NotificationId;
use crate::models::notifications::Notification;
use crate::models::v2::notifications::LegacyNotification;
@@ -9,6 +8,7 @@ use crate::routes::v2_reroute;
use crate::routes::v3;
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(notifications_get);
apps/labrinth/src/routes/v2/project_creation.rs
+1 -1
View File
@@ -1,6 +1,5 @@
use crate::database::PgPool;
use crate::database::models::version_item;
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models;
use crate::models::ids::ImageId;
@@ -19,6 +18,7 @@ use actix_web::web::Data;
use actix_web::{HttpRequest, HttpResponse, post};
use serde::{Deserialize, Serialize};
use serde_json::json;
use xredis::RedisPool;
use std::collections::HashMap;
use validator::Validate;
apps/labrinth/src/routes/v2/projects.rs
+1 -1
View File
@@ -1,6 +1,5 @@
use crate::database::models::categories::LinkPlatform;
use crate::database::models::{project_item, version_item};
use crate::database::redis::RedisPool;
use crate::database::{PgPool, ReadOnlyPgPool};
use crate::file_hosting::FileHost;
use crate::models::projects::{
@@ -18,6 +17,7 @@ use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(project_search);
apps/labrinth/src/routes/v2/reports.rs
+1 -1
View File
@@ -1,5 +1,4 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::reports::Report;
use crate::models::v2::reports::LegacyReport;
use crate::queue::session::AuthQueue;
@@ -7,6 +6,7 @@ use crate::routes::{ApiError, v2_reroute, v3};
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
use serde::Deserialize;
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(reports_get);
apps/labrinth/src/routes/v2/tags.rs
+1 -1
View File
@@ -3,7 +3,6 @@ use std::collections::HashMap;
use super::ApiError;
use crate::database::PgPool;
use crate::database::models::loader_fields::LoaderFieldEnumValue;
use crate::database::redis::RedisPool;
use crate::models::v2::projects::LegacySideType;
use crate::routes::v2_reroute::capitalize_first;
use crate::routes::v3::tags::{LinkPlatformQueryData, LoaderFieldsEnumQuery};
@@ -11,6 +10,7 @@ use crate::routes::{v2_reroute, v3};
use actix_web::{HttpResponse, get, web};
use chrono::{DateTime, Utc};
use itertools::Itertools;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/v2/teams.rs
+1 -1
View File
@@ -1,5 +1,4 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::ids::TeamId;
use crate::models::teams::{
OrganizationPermissions, ProjectPermissions, TeamMember,
@@ -11,6 +10,7 @@ use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
use ariadne::ids::UserId;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(teams_get);
apps/labrinth/src/routes/v2/threads.rs
+1 -1
View File
@@ -1,5 +1,4 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::ids::{ThreadId, ThreadMessageId};
use crate::models::threads::{MessageBody, Thread};
@@ -8,6 +7,7 @@ use crate::queue::session::AuthQueue;
use crate::routes::{ApiError, v2_reroute, v3};
use actix_web::{HttpRequest, HttpResponse, delete, get, post, web};
use serde::Deserialize;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/v2/users.rs
+1 -1
View File
@@ -1,5 +1,4 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::notifications::Notification;
use crate::models::projects::Project;
@@ -12,6 +11,7 @@ use crate::routes::{ApiError, v2_reroute, v3};
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
use serde::{Deserialize, Serialize};
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(user_auth_get);
apps/labrinth/src/routes/v2/version_creation.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use crate::database::PgPool;
use crate::database::models::loader_fields::VersionField;
use crate::database::models::{project_item, version_item};
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::ids::{ImageId, ProjectId, VersionId};
use crate::models::projects::{
@@ -22,6 +21,7 @@ use serde::{Deserialize, Serialize};
use serde_json::json;
use std::collections::HashMap;
use validator::Validate;
use xredis::RedisPool;
pub fn default_requested_status() -> VersionStatus {
VersionStatus::Listed
apps/labrinth/src/routes/v2/version_file.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use super::ApiError;
use crate::database::PgPool;
use crate::database::ReadOnlyPgPool;
use crate::database::redis::RedisPool;
use crate::models::projects::{Project, Version, VersionType};
use crate::models::v2::projects::{LegacyProject, LegacyVersion};
use crate::queue::session::AuthQueue;
@@ -11,6 +10,7 @@ use crate::routes::{FileHash, v2_reroute, v3};
use actix_web::{HttpRequest, HttpResponse, delete, get, post, web};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(
apps/labrinth/src/routes/v2/versions.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use std::collections::HashMap;
use super::ApiError;
use crate::database::redis::RedisPool;
use crate::database::{PgPool, ReadOnlyPgPool};
use crate::models;
use crate::models::ids::VersionId;
@@ -15,6 +14,7 @@ use crate::search::{SearchBackend, SearchState};
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
use serde::{Deserialize, Serialize};
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(versions_get);
apps/labrinth/src/routes/v3/analytics_event.rs
+1 -1
View File
@@ -2,6 +2,7 @@ use actix_web::{HttpRequest, delete, get, patch, post, web};
use chrono::{DateTime, Utc};
use eyre::eyre;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
use crate::{
auth::get_user_from_headers,
@@ -10,7 +11,6 @@ use crate::{
models::{
DBAnalyticsEvent, DBAnalyticsEventId, generate_analytics_event_id,
},
redis::RedisPool,
},
models::{
ids::AnalyticsEventId,
apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs
+2 -3
View File
@@ -4,11 +4,10 @@ use super::{
ProjectViewsFacets,
};
use crate::{
database::{PgPool, redis::RedisPool},
models::v3::analytics::DownloadReason,
routes::ApiError,
database::PgPool, models::v3::analytics::DownloadReason, routes::ApiError,
util::tags::valid_download_tags,
};
use xredis::RedisPool;
pub async fn fetch(
pool: &PgPool,
apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs
+2 -3
View File
@@ -1,3 +1,4 @@
use xredis::RedisPool;
mod fixed;
use actix_web::{HttpRequest, post, web};
@@ -8,9 +9,7 @@ use crate::models::{
ids::VersionId, pats::Scopes, v3::analytics::DownloadReason,
};
use crate::{
auth::get_user_from_headers,
database::{PgPool, redis::RedisPool},
queue::session::AuthQueue,
auth::get_user_from_headers, database::PgPool, queue::session::AuthQueue,
routes::ApiError,
};
apps/labrinth/src/routes/v3/analytics_get/mod.rs
+2 -1
View File
@@ -7,6 +7,8 @@
//! requests, you have to zip together M arrays of N elements
//! - this makes it inconvenient to have separate endpoints
use xredis::RedisPool;
pub mod facets;
mod metrics;
pub mod old;
@@ -36,7 +38,6 @@ use crate::{
DBAffiliateCode, DBAffiliateCodeId, DBProjectId, DBUser, DBVersion,
DBVersionId,
},
redis::RedisPool,
},
models::{
ids::{AffiliateCodeId, ProjectId, VersionId},
apps/labrinth/src/routes/v3/analytics_get/old.rs
+1 -1
View File
@@ -3,7 +3,6 @@
use super::ApiError;
use crate::database;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::teams::ProjectPermissions;
use crate::{
auth::get_user_from_headers,
@@ -20,6 +19,7 @@ use sqlx::postgres::types::PgInterval;
use std::collections::HashMap;
use std::convert::TryInto;
use std::num::NonZeroU32;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(playtimes_get)
apps/labrinth/src/routes/v3/collections.rs
+1 -1
View File
@@ -6,7 +6,6 @@ use crate::database::PgPool;
use crate::database::models::{
collection_item, generate_collection_id, project_item,
};
use crate::database::redis::RedisPool;
use crate::file_hosting::{FileHost, FileHostPublicity};
use crate::models::collections::{Collection, CollectionStatus};
use crate::models::ids::{CollectionId, ProjectId};
@@ -28,6 +27,7 @@ use eyre::eyre;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(collections_get)
apps/labrinth/src/routes/v3/content/mod.rs
+24 -20
View File
@@ -6,7 +6,7 @@ use crate::auth::get_user_from_headers;
use crate::database::models::ids::DBVersionId;
use crate::database::models::version_item::VersionQueryResult;
use crate::database::models::{DBProject, DBVersion};
use crate::database::{PgPool, ReadOnlyPgPool, redis::RedisPool};
use crate::database::{PgPool, ReadOnlyPgPool};
use crate::models::pats::Scopes;
use crate::models::projects::{DependencyType, Version};
use crate::models::users::User;
@@ -21,10 +21,11 @@ use modrinth_content_management::{
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use xredis::RedisPool;
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v1";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v1";
const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v1";
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v3";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v3";
const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v3";
const CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS: i64 = 60 * 60 * 24;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
@@ -295,11 +296,13 @@ async fn increment_content_resolve_cache_heat(
return None;
}
};
let key = redis.key().with_slot(
CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE,
heat_key,
heat_key,
);
let count = match redis
.incr(CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE, heat_key)
.await
{
let count = match redis.incr(&key).await {
Ok(Some(count)) => count,
Ok(None) => 1,
Err(error) => {
@@ -312,8 +315,7 @@ async fn increment_content_resolve_cache_heat(
if let Err(error) = redis
.set(
CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE,
heat_key,
&key,
&count.to_string(),
Some(CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS),
)
@@ -338,11 +340,13 @@ async fn get_cached_resolve_content_plan(
return None;
}
};
let key = redis.key().with_slot(
CONTENT_RESOLVE_CACHE_NAMESPACE,
cache_key,
cache_key,
);
match redis
.get_deserialized(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key)
.await
{
match redis.get_deserialized(&key).await {
Ok(cached) => cached,
Err(error) => {
tracing::warn!("failed to read content resolve cache: {error}");
@@ -366,14 +370,14 @@ async fn set_cached_resolve_content_plan(
return;
}
};
if let Err(error) = redis
.set_serialized(
let key = redis.key().with_slot(
CONTENT_RESOLVE_CACHE_NAMESPACE,
cache_key,
cached,
Some(expiry_seconds),
)
cache_key,
);
if let Err(error) = redis
.set_serialized(&key, cached, Some(expiry_seconds))
.await
{
tracing::warn!("failed to write content resolve cache: {error}");
apps/labrinth/src/routes/v3/friends.rs
+1 -1
View File
@@ -2,7 +2,6 @@ use crate::auth::get_user_from_headers;
use crate::database::PgPool;
use crate::database::models::friend_item::DBFriend;
use crate::database::models::{DBUser, DBUserId};
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::models::users::UserFriend;
use crate::queue::session::AuthQueue;
@@ -16,6 +15,7 @@ use crate::sync::status::get_user_status;
use actix_web::{HttpRequest, HttpResponse, delete, get, post, web};
use ariadne::networking::message::ServerToClientMessage;
use chrono::Utc;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(add_friend);
apps/labrinth/src/routes/v3/images.rs
+1 -1
View File
@@ -6,7 +6,6 @@ use crate::database::PgPool;
use crate::database::models::{
project_item, report_item, thread_item, version_item,
};
use crate::database::redis::RedisPool;
use crate::file_hosting::{FileHost, FileHostPublicity};
use crate::models::ids::{ReportId, ThreadMessageId, VersionId};
use crate::models::images::{Image, ImageContext};
@@ -16,6 +15,7 @@ use crate::util::img::upload_image_optimized;
use crate::util::routes::read_limited_from_payload;
use actix_web::{HttpRequest, HttpResponse, post, web};
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(images_add);
apps/labrinth/src/routes/v3/limits.rs
+1 -1
View File
@@ -1,12 +1,12 @@
use crate::database::PgPool;
use crate::{
auth::get_user_from_headers,
database::redis::RedisPool,
models::{pats::Scopes, v3::user_limits::UserLimits},
queue::session::AuthQueue,
routes::ApiError,
};
use actix_web::{HttpRequest, get, web};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(get_project_limits)
apps/labrinth/src/routes/v3/notifications.rs
+1 -1
View File
@@ -1,7 +1,6 @@
use crate::auth::get_user_from_headers;
use crate::database;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::models::ids::NotificationId;
use crate::models::notifications::Notification;
use crate::models::pats::Scopes;
@@ -9,6 +8,7 @@ use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(notifications_get_route)
apps/labrinth/src/routes/v3/oauth_clients.rs
+2 -4
View File
@@ -1,4 +1,5 @@
use std::{collections::HashSet, fmt::Display};
use xredis::RedisPool;
use super::ApiError;
use crate::database::{PgPool, PgTransaction};
@@ -7,15 +8,12 @@ use crate::models::ids::OAuthClientId;
use crate::util::img::{delete_old_images, upload_image_optimized};
use crate::{
auth::{checks::ValidateAuthorized, get_user_from_headers},
database::{
models::{
database::models::{
DBOAuthClientId, DBUser, DatabaseError, generate_oauth_client_id,
generate_oauth_redirect_id,
oauth_client_authorization_item::DBOAuthClientAuthorization,
oauth_client_item::{DBOAuthClient, DBOAuthRedirectUri},
},
redis::RedisPool,
},
models::{
self,
oauth_clients::{GetOAuthClientsRequest, OAuthClientCreationResult},
apps/labrinth/src/routes/v3/organizations.rs
+1 -1
View File
@@ -10,7 +10,6 @@ use crate::database::models::team_item::DBTeamMember;
use crate::database::models::{
DBModerationNote, DBOrganization, generate_organization_id, team_item,
};
use crate::database::redis::RedisPool;
use crate::file_hosting::{FileHost, FileHostPublicity};
use crate::models::ids::OrganizationId;
use crate::models::pats::Scopes;
@@ -29,6 +28,7 @@ use futures::TryStreamExt;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use validator::Validate;
use xredis::RedisPool;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(organizations_get)
apps/labrinth/src/routes/v3/payouts.rs
+1 -1
View File
@@ -3,7 +3,6 @@ use crate::auth::{AuthenticationError, get_user_from_headers};
use crate::database::PgPool;
use crate::database::models::DBUserId;
use crate::database::models::{generate_payout_id, users_compliance};
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::ids::PayoutId;
use crate::models::pats::Scopes;
@@ -29,6 +28,7 @@ use sha2::Sha256;
use std::collections::HashMap;
use tokio_stream::StreamExt;
use tracing::error;
use xredis::RedisPool;
const COMPLIANCE_CHECK_DEBOUNCE: chrono::Duration =
chrono::Duration::seconds(15);
apps/labrinth/src/routes/v3/project_creation.rs
+1 -1
View File
@@ -9,7 +9,6 @@ use crate::database::models::loader_fields::{
};
use crate::database::models::thread_item::ThreadBuilder;
use crate::database::models::{self, DBUser, image_item};
use crate::database::redis::RedisPool;
use crate::file_hosting::{FileHost, FileHostPublicity, FileHostingError};
use crate::models::error::ApiError;
use crate::models::exp;
@@ -45,6 +44,7 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use thiserror::Error;
use validator::Validate;
use xredis::RedisPool;
pub mod new;

Some files were not shown because too many files have changed in this diff Show More