sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-08-24 16:44:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: use postcard for redis serde (#6956)

Browse Source 
* redo error handling in xredis

* give proper types to metadata fields

* add round-trip tests

* inline loader enum metadata fields

* postcard roundtrips

* prepare

* bump redis key version

* serde-binhum

* clippy

* fix

* fix frontend checking existence of component fields rather than non-null-ness

* prepare
This commit is contained in:
aecsocket
2026-08-05 10:36:05 +00:00
committed by GitHub
parent cc53d9a6f8
commit 5148e8ec35
70 changed files with 1713 additions and 646 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+13 -1
View File
@@ -5515,6 +5515,7 @@ dependencies = [
"scalar_api_reference",
"sentry",
"serde",
"serde-binhum",
"serde_json",
"serde_with",
"sha1 0.10.6",
@@ -9379,6 +9380,16 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-binhum"
version = "0.1.0"
dependencies = [
"darling 0.23.0",
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "serde-untagged"
version = "0.1.9"
@@ -13443,12 +13454,13 @@ dependencies = [
"chrono",
"dashmap",
"deadpool-redis",
"eyre",
"futures",
"lz4_flex",
"postcard",
"prometheus",
"redis",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tracing",
Cargo.toml
+2
View File
@@ -15,6 +15,7 @@ members = [
"packages/modrinth-util",
"packages/neverbounce",
"packages/path-util",
"packages/serde-binhum",
"packages/xredis",
]
@@ -183,6 +184,7 @@ sentry = { version = "0.45.0", default-features = false, features = [
"rustls",
] }
serde = "1.0.228"
serde-binhum = { path = "packages/serde-binhum" }
serde_bytes = "0.11.19"
serde_cbor = "0.11.2"
serde_ini = "0.2.0"
apps/frontend/src/pages/[type]/[project].vue
+2 -4
View File
@@ -2014,9 +2014,7 @@ const navLinks = computed(() => {
label: formatMessage(messages.changelogTab),
href: withInstallContextQuery(`${projectUrl}/changelog`),
shown:
hasVersions.value &&
projectV3Loaded.value &&
projectV3.value?.minecraft_server === undefined,
hasVersions.value && projectV3Loaded.value && projectV3.value?.minecraft_server == null,
onHover: loadVersions,
},
{
@@ -2025,7 +2023,7 @@ const navLinks = computed(() => {
shown:
(hasVersions.value || !!currentMember.value) &&
projectV3Loaded.value &&
projectV3.value?.minecraft_server === undefined,
projectV3.value?.minecraft_server == null,
subpages: [`${projectUrl}/version/`],
onHover: loadVersions,
},
apps/labrinth/.sqlx/query-04c04958c71c4fab903c46c9185286e7460a6ff7b03cbc90939ac6c7cb526433.json → apps/labrinth/.sqlx/query-214cc9257db904fd2cc68b8aa8d61d32cab210cfaea2205c95d9ca3bbd0ec1cd.json
Generated
+11 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, enum_id, value, ordering, metadata, created FROM loader_field_enum_values\n WHERE enum_id = ANY($1)\n ORDER BY enum_id, ordering, created DESC\n ",
"query": "\n SELECT id, enum_id, value, ordering,\n metadata->>'type' AS \"ty?\",\n (metadata->>'major')::boolean AS \"major?\",\n created FROM loader_field_enum_values\n WHERE enum_id = ANY($1)\n ORDER BY enum_id, ordering, created DESC\n ",
"describe": {
"columns": [
{
@@ -25,11 +25,16 @@
},
{
"ordinal": 4,
"name": "metadata",
"type_info": "Jsonb"
"name": "ty?",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "major?",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "created",
"type_info": "Timestamptz"
}
@@ -44,9 +49,10 @@
false,
false,
true,
true,
null,
null,
false
]
},
"hash": "04c04958c71c4fab903c46c9185286e7460a6ff7b03cbc90939ac6c7cb526433"
"hash": "214cc9257db904fd2cc68b8aa8d61d32cab210cfaea2205c95d9ca3bbd0ec1cd"
}
apps/labrinth/.sqlx/query-d9c4d536ce0bea290f445c3bccb56b4743f2f3a9ce4b170fb439e0e135ca9d51.json → apps/labrinth/.sqlx/query-83b7543e426ae348e589b591f249358c2b9eac5323ae0795581358ed29e5ba65.json
Generated
+11 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created, metadata\n FROM loader_field_enum_values lfev\n WHERE id = ANY($1)\n ORDER BY enum_id, ordering, created ASC\n ",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created,\n metadata->>'type' AS \"ty?\",\n (metadata->>'major')::boolean AS \"major?\"\n FROM loader_field_enum_values lfev\n WHERE id = ANY($1)\n ORDER BY enum_id, ordering, created DESC\n ",
"describe": {
"columns": [
{
@@ -30,8 +30,13 @@
},
{
"ordinal": 5,
"name": "metadata",
"type_info": "Jsonb"
"name": "ty?",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "major?",
"type_info": "Bool"
}
],
"parameters": {
@@ -45,8 +50,9 @@
false,
true,
false,
true
null,
null
]
},
"hash": "d9c4d536ce0bea290f445c3bccb56b4743f2f3a9ce4b170fb439e0e135ca9d51"
"hash": "83b7543e426ae348e589b591f249358c2b9eac5323ae0795581358ed29e5ba65"
}
apps/labrinth/.sqlx/query-fe34673ce6d7bcb616a5ab2e8900d7dfb4e0fa2ee640128d29d6e4beafe60f4c.json → apps/labrinth/.sqlx/query-a8569122a309057326be0d49630119f81d17197673d831d210e351c149e41fc8.json
Generated
+11 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created, metadata\n FROM loader_field_enum_values lfev\n ORDER BY enum_id, ordering, created DESC\n ",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created,\n metadata->>'type' AS \"ty?\",\n (metadata->>'major')::boolean AS \"major?\"\n FROM loader_field_enum_values lfev\n ORDER BY enum_id, ordering, created DESC\n ",
"describe": {
"columns": [
{
@@ -30,8 +30,13 @@
},
{
"ordinal": 5,
"name": "metadata",
"type_info": "Jsonb"
"name": "ty?",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "major?",
"type_info": "Bool"
}
],
"parameters": {
@@ -43,8 +48,9 @@
false,
true,
false,
true
null,
null
]
},
"hash": "fe34673ce6d7bcb616a5ab2e8900d7dfb4e0fa2ee640128d29d6e4beafe60f4c"
"hash": "a8569122a309057326be0d49630119f81d17197673d831d210e351c149e41fc8"
}
apps/labrinth/.sqlx/query-ab9b4b383ce8431214eb26abde734d36eff6902ae55b7ed4d89a10c4d69506a9.json
Generated
-50
View File
@@ -1,50 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT l.id id, l.loader loader, l.icon icon, l.metadata metadata,\n ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,\n ARRAY_AGG(DISTINCT g.slug) filter (where g.slug is not null) games\n FROM loaders l\n LEFT OUTER JOIN loaders_project_types lpt ON joining_loader_id = l.id\n LEFT OUTER JOIN project_types pt ON lpt.joining_project_type_id = pt.id\n LEFT OUTER JOIN loaders_project_types_games lptg ON lptg.loader_id = lpt.joining_loader_id AND lptg.project_type_id = lpt.joining_project_type_id\n LEFT OUTER JOIN games g ON lptg.game_id = g.id\n GROUP BY l.id;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "loader",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "icon",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "metadata",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "project_types",
"type_info": "VarcharArray"
},
{
"ordinal": 5,
"name": "games",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
null,
null
]
},
"hash": "ab9b4b383ce8431214eb26abde734d36eff6902ae55b7ed4d89a10c4d69506a9"
}
apps/labrinth/.sqlx/query-43d4eafdbcb449a56551d3d6edeba0d6e196fa6539e3f9df107c23a74ba962af.json → apps/labrinth/.sqlx/query-ec9abad348739217eb887e2ac84901ba8dca63912e9d7054115052ad89ab4718.json
Generated
+11 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created, metadata\n FROM loader_field_enum_values lfev\n WHERE id = ANY($1)\n ORDER BY enum_id, ordering, created DESC\n ",
"query": "\n SELECT DISTINCT id, enum_id, value, ordering, created,\n metadata->>'type' AS \"ty?\",\n (metadata->>'major')::boolean AS \"major?\"\n FROM loader_field_enum_values lfev\n WHERE id = ANY($1)\n ORDER BY enum_id, ordering, created ASC\n ",
"describe": {
"columns": [
{
@@ -30,8 +30,13 @@
},
{
"ordinal": 5,
"name": "metadata",
"type_info": "Jsonb"
"name": "ty?",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "major?",
"type_info": "Bool"
}
],
"parameters": {
@@ -45,8 +50,9 @@
false,
true,
false,
true
null,
null
]
},
"hash": "43d4eafdbcb449a56551d3d6edeba0d6e196fa6539e3f9df107c23a74ba962af"
"hash": "ec9abad348739217eb887e2ac84901ba8dca63912e9d7054115052ad89ab4718"
}
apps/labrinth/.sqlx/query-f21bfe5bcc157d442180bf49c97654bd8650dcdd16cb9f74148bef6e932e2837.json
Generated
+50
View File
@@ -0,0 +1,50 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT l.id id, l.loader loader, l.icon icon,\n (l.metadata->>'platform')::boolean AS platform,\n ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,\n ARRAY_AGG(DISTINCT g.slug) filter (where g.slug is not null) games\n FROM loaders l\n LEFT OUTER JOIN loaders_project_types lpt ON joining_loader_id = l.id\n LEFT OUTER JOIN project_types pt ON lpt.joining_project_type_id = pt.id\n LEFT OUTER JOIN loaders_project_types_games lptg ON lptg.loader_id = lpt.joining_loader_id AND lptg.project_type_id = lpt.joining_project_type_id\n LEFT OUTER JOIN games g ON lptg.game_id = g.id\n GROUP BY l.id;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "loader",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "icon",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "platform",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "project_types",
"type_info": "VarcharArray"
},
{
"ordinal": 5,
"name": "games",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
null,
null,
null
]
},
"hash": "f21bfe5bcc157d442180bf49c97654bd8650dcdd16cb9f74148bef6e932e2837"
}
apps/labrinth/AGENTS.md
+3 -1
View File
@@ -12,8 +12,10 @@
- no trailing punctuation
- wrap code items e.g. type names in backticks
- Prefer `wrap_internal_err`, `wrap_request_err` when attaching context to an existing error (like Anyhow `context` or Eyre `wrap_err`)
- Prefer importing `eyre::Result` and using `Result<T>` instead of `eyre::Result<T>`
- Prefer `eyre::Ok(value)` instead of `Ok::<_, eyre::Report>(value)` when an explicit Eyre result type is needed
- All operations should ideally have some context attached
- Database operations can have a message like `.wrap_internal_err("failed to fetch XYZ")`
- Database operations can have a message like `.wrap_internal_err("fetching XYZ")`
- You can perform real-time queries against the databases in the Docker Compose
- `docker exec labrinth-postgres psql -c "select 1"`
- `docker exec labrinth-redis redis-cli flushall`
apps/labrinth/Cargo.toml
+1
View File
@@ -111,6 +111,7 @@ rusty-money = { workspace = true }
scalar_api_reference = { workspace = true, features = ["actix-web"] }
sentry = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde-binhum = { workspace = true }
serde_json = { workspace = true }
serde_with = { workspace = true }
sha1 = { workspace = true }
apps/labrinth/src/auth/mod.rs
-6
View File
@@ -59,12 +59,6 @@ pub enum AuthenticationError {
Url,
}
impl From<xredis::Error> for AuthenticationError {
fn from(error: xredis::Error) -> Self {
Self::Database(error.into())
}
}
impl actix_web::ResponseError for AuthenticationError {
fn status_code(&self) -> StatusCode {
match self {
apps/labrinth/src/database/models/analytics_event_item.rs
+1 -1
View File
@@ -9,7 +9,7 @@ use crate::{
};
use serde::{Deserialize, Serialize};
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v3";
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v4";
const ANALYTICS_EVENTS_ALL_KEY: &str = "all";
#[derive(Debug, Clone, Serialize, Deserialize)]
apps/labrinth/src/database/models/categories.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use super::ids::*;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const TAGS_NAMESPACE: &str = "tags:v3";
const TAGS_NAMESPACE: &str = "tags:v4";
pub struct ProjectType {
pub id: ProjectTypeId,
apps/labrinth/src/database/models/collection_item.rs
+1 -1
View File
@@ -8,7 +8,7 @@ use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const COLLECTIONS_NAMESPACE: &str = "collections:v3";
const COLLECTIONS_NAMESPACE: &str = "collections:v4";
#[derive(Clone)]
pub struct CollectionBuilder {
apps/labrinth/src/database/models/flow_item.rs
+29 -3
View File
@@ -8,14 +8,14 @@ use rand::Rng;
use rand::distributions::Alphanumeric;
use rand_chacha::ChaCha20Rng;
use rand_chacha::rand_core::SeedableRng;
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use url::Url;
use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration};
use xredis::RedisPool;
const FLOWS_NAMESPACE: &str = "flows:v3";
const FLOWS_NAMESPACE: &str = "flows:v4";
#[derive(Deserialize, Serialize)]
#[serde_binhum]
pub enum DBFlow {
OAuth {
user_id: Option<DBUserId>,
@@ -60,13 +60,39 @@ pub enum DBFlow {
},
RegisterPasskey {
user_id: DBUserId,
#[serde_binhum(binary(with = "json_string"))]
state: PasskeyRegistration,
},
AuthenticatePasskey {
#[serde_binhum(binary(with = "json_string"))]
state: DiscoverableAuthentication,
},
}
mod json_string {
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub fn serialize<T, S>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
where
T: Serialize,
S: Serializer,
{
let value =
serde_json::to_string(value).map_err(serde::ser::Error::custom)?;
value.serialize(serializer)
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
where
T: DeserializeOwned,
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
serde_json::from_str(&value).map_err(serde::de::Error::custom)
}
}
impl DBFlow {
pub async fn insert_with_state(
&self,
apps/labrinth/src/database/models/image_item.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const IMAGES_NAMESPACE: &str = "images:v3";
const IMAGES_NAMESPACE: &str = "images:v4";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DBImage {
apps/labrinth/src/database/models/legacy_loader_fields.rs
+2 -11
View File
@@ -117,17 +117,8 @@ impl MinecraftGameVersion {
id: loader_field_enum_value.id,
version: loader_field_enum_value.value,
created: loader_field_enum_value.created,
type_: loader_field_enum_value
.metadata
.get("type")
.and_then(|x| x.as_str())
.map(|x| x.to_string())
.unwrap_or_default(),
major: loader_field_enum_value
.metadata
.get("major")
.and_then(|x| x.as_bool())
.unwrap_or_default(),
type_: loader_field_enum_value.ty.unwrap_or_default(),
major: loader_field_enum_value.major.unwrap_or_default(),
}
}
}
apps/labrinth/src/database/models/loader_fields.rs
+73 -58
View File
@@ -12,14 +12,14 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const GAMES_LIST_NAMESPACE: &str = "games:v3";
const LOADER_ID: &str = "loader_id:v3";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v3";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v3";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v3";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v3";
const GAMES_LIST_NAMESPACE: &str = "games:v4";
const LOADER_ID: &str = "loader_id:v4";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v4";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v4";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v4";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v4";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str =
"loader_field_enum_values:v3";
"loader_field_enum_values:v4";
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct Game {
@@ -87,6 +87,11 @@ impl Game {
}
}
#[derive(Clone, Serialize, Deserialize)]
pub struct LoaderMetadata {
pub platform: Option<bool>,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct Loader {
pub id: LoaderId,
@@ -94,7 +99,7 @@ pub struct Loader {
pub icon: String,
pub supported_project_types: Vec<String>,
pub supported_games: Vec<String>, // slugs
pub metadata: serde_json::Value,
pub metadata: LoaderMetadata,
}
impl Loader {
@@ -154,7 +159,8 @@ impl Loader {
let result = sqlx::query!(
"
SELECT l.id id, l.loader loader, l.icon icon, l.metadata metadata,
SELECT l.id id, l.loader loader, l.icon icon,
(l.metadata->>'platform')::boolean AS platform,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,
ARRAY_AGG(DISTINCT g.slug) filter (where g.slug is not null) games
FROM loaders l
@@ -179,7 +185,9 @@ impl Loader {
supported_games: x
.games
.unwrap_or_default(),
metadata: x.metadata
metadata: LoaderMetadata {
platform: x.platform,
},
})
.try_collect::<Vec<_>>()
.await?;
@@ -277,8 +285,9 @@ pub struct LoaderFieldEnumValue {
pub value: String,
pub ordering: Option<i32>,
pub created: DateTime<Utc>,
#[serde(flatten)]
pub metadata: serde_json::Value,
#[serde(rename = "type")]
pub ty: Option<String>,
pub major: Option<bool>,
}
impl std::hash::Hash for LoaderFieldEnumValue {
@@ -357,7 +366,8 @@ pub struct QueryLoaderFieldEnumValue {
pub value: String,
pub ordering: Option<i32>,
pub created: DateTime<Utc>,
pub metadata: Option<serde_json::Value>,
pub ty: Option<String>,
pub major: Option<bool>,
}
impl LoaderField {
@@ -612,42 +622,50 @@ impl LoaderFieldEnumValue {
where
E: crate::database::Executor<'a, Database = sqlx::Postgres>,
{
let val = redis.get_cached_keys_raw(
LOADER_FIELD_ENUM_VALUES_NAMESPACE,
&loader_field_enum_ids.iter().map(|x| x.0).collect::<Vec<_>>(),
|loader_field_enum_ids| async move {
let values = sqlx::query!(
"
SELECT id, enum_id, value, ordering, metadata, created FROM loader_field_enum_values
let val = redis
.get_cached_keys_raw(
LOADER_FIELD_ENUM_VALUES_NAMESPACE,
&loader_field_enum_ids
.iter()
.map(|x| x.0)
.collect::<Vec<_>>(),
|loader_field_enum_ids| async move {
let values = sqlx::query!(
r#"
SELECT id, enum_id, value, ordering,
metadata->>'type' AS "ty?",
(metadata->>'major')::boolean AS "major?",
created FROM loader_field_enum_values
WHERE enum_id = ANY($1)
ORDER BY enum_id, ordering, created DESC
",
&loader_field_enum_ids
)
"#,
&loader_field_enum_ids
)
.fetch(exec)
.try_fold(DashMap::new(), |acc: DashMap<i32, Vec<LoaderFieldEnumValue>>, c| {
let value = LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(c.id),
enum_id: LoaderFieldEnumId(c.enum_id),
value: c.value,
ordering: c.ordering,
created: c.created,
metadata: c.metadata.unwrap_or_default(),
};
.try_fold(
DashMap::new(),
|acc: DashMap<i32, Vec<LoaderFieldEnumValue>>, c| {
let value = LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(c.id),
enum_id: LoaderFieldEnumId(c.enum_id),
value: c.value,
ordering: c.ordering,
created: c.created,
ty: c.ty,
major: c.major,
};
acc.entry(c.enum_id)
.or_default()
.push(value);
acc.entry(c.enum_id).or_default().push(value);
async move {
Ok(acc)
}
})
async move { Ok(acc) }
},
)
.await?;
Ok::<_, DatabaseError>(values)
},
).await?;
Ok::<_, DatabaseError>(values)
},
)
.await?;
Ok(val
.into_iter()
@@ -669,15 +687,16 @@ impl LoaderFieldEnumValue {
.await?
.into_iter()
.filter(|x| {
let mut bool = true;
for (key, value) in &filter {
if let Some(metadata_value) = x.metadata.get(key) {
bool &= metadata_value == value;
} else {
bool = false;
filter.iter().all(|(key, value)| match key.as_str() {
"type" => {
x.ty.as_deref()
.is_some_and(|type_| value.as_str() == Some(type_))
}
}
bool
"major" => x
.major
.is_some_and(|major| value.as_bool() == Some(major)),
_ => false,
})
})
.collect();
@@ -1170,10 +1189,8 @@ impl VersionFieldValue {
value: lfev.value.clone(),
ordering: lfev.ordering,
created: lfev.created,
metadata: lfev
.metadata
.clone()
.unwrap_or_default(),
ty: lfev.ty.clone(),
major: lfev.major,
}
}),
))
@@ -1249,10 +1266,8 @@ impl VersionFieldValue {
value: lfev.value.clone(),
ordering: lfev.ordering,
created: lfev.created,
metadata: lfev
.metadata
.clone()
.unwrap_or_default(),
ty: lfev.ty.clone(),
major: lfev.major,
})
})
.collect::<Result<_, _>>()?,
apps/labrinth/src/database/models/mod.rs
-2
View File
@@ -77,8 +77,6 @@ pub enum DatabaseError {
SerdeCacheError(#[from] serde_json::Error),
#[error("error while encoding or decoding the cache: {0}")]
PostcardCacheError(#[from] postcard::Error),
#[error(transparent)]
Redis(#[from] xredis::Error),
#[error("Schema error: {0}")]
SchemaError(String),
}
apps/labrinth/src/database/models/moderation_note_item.rs
+2 -2
View File
@@ -7,9 +7,9 @@ use xredis::RedisPool;
use super::{DBOrganizationId, DBUserId, DatabaseError};
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v3";
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v4";
const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str =
"moderation_notes_organizations:v3";
"moderation_notes_organizations:v4";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DBModerationNote {
apps/labrinth/src/database/models/notification_item.rs
+1 -1
View File
@@ -10,7 +10,7 @@ use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v3";
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v4";
pub struct NotificationBuilder {
pub body: NotificationBody,
apps/labrinth/src/database/models/notifications_template_item.rs
+16 -8
View File
@@ -1,14 +1,15 @@
use crate::database::models::DatabaseError;
use crate::models::v3::notifications::{NotificationChannel, NotificationType};
use crate::routes::ApiError;
use crate::util::error::Context;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v3";
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v4";
const TEMPLATES_HTML_DATA_NAMESPACE: &str =
"notifications_templates_html_data:v3";
"notifications_templates_html_data:v4";
const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str =
"notifications_templates_dynamic_html:v3";
"notifications_templates_dynamic_html:v4";
const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes
const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes
@@ -123,12 +124,16 @@ where
html: String,
}
let mut redis_conn = redis.connect().await?;
let mut redis_conn = redis.connect().await.wrap_internal_err(
"connecting to redis for dynamic notification html",
)?;
let redis_key = redis_conn
.key()
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
if let Some(body) =
redis_conn.get_deserialized::<HtmlBody>(&redis_key).await?
if let Some(body) = redis_conn
.get_deserialized::<HtmlBody>(&redis_key)
.await
.wrap_internal_err("fetching dynamic notification html from redis")?
{
return Ok(body.html);
}
@@ -136,14 +141,17 @@ where
drop(redis_conn);
let cached = HtmlBody { html: get().await? };
let mut redis_conn = redis.connect().await?;
let mut redis_conn = redis.connect().await.wrap_internal_err(
"connecting to redis for dynamic notification html",
)?;
let redis_key = redis_conn
.key()
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
redis_conn
.set_serialized(&redis_key, &cached, Some(HTML_DATA_CACHE_EXPIRY))
.await?;
.await
.wrap_internal_err("writing dynamic notification html to redis")?;
Ok(cached.html)
}
apps/labrinth/src/database/models/notifications_type_item.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use crate::models::v3::notifications::NotificationType;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v3";
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v4";
#[derive(Serialize, Deserialize)]
pub struct NotificationTypeItem {
apps/labrinth/src/database/models/organization_item.rs
+2 -2
View File
@@ -9,8 +9,8 @@ use xredis::RedisPool;
use super::{DBTeamMember, ids::*};
use serde::{Deserialize, Serialize};
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v3";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v3";
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v4";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v4";
#[derive(Deserialize, Serialize, Clone, Debug)]
/// An organization of users who together control one or more projects and organizations.
apps/labrinth/src/database/models/pat_item.rs
+3 -3
View File
@@ -11,9 +11,9 @@ use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const PATS_NAMESPACE: &str = "pats:v3";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v3";
const PATS_USERS_NAMESPACE: &str = "pats_users:v3";
const PATS_NAMESPACE: &str = "pats:v4";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v4";
const PATS_USERS_NAMESPACE: &str = "pats_users:v4";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBPersonalAccessToken {
apps/labrinth/src/database/models/product_item.rs
+2 -2
View File
@@ -9,7 +9,7 @@ use std::convert::TryFrom;
use std::convert::TryInto;
use xredis::RedisPool;
const PRODUCTS_NAMESPACE: &str = "products:v3";
const PRODUCTS_NAMESPACE: &str = "products:v4";
pub struct DBProduct {
pub id: DBProductId,
@@ -136,7 +136,7 @@ pub struct QueryProductWithPrices {
pub id: DBProductId,
pub metadata: ProductMetadata,
pub unitary: bool,
#[serde(skip_serializing_if = "Option::is_none", default)]
#[serde(default)]
pub name: Option<String>,
pub prices: Vec<DBProductPrice>,
}
apps/labrinth/src/database/models/project_item.rs
+15 -10
View File
@@ -19,13 +19,14 @@ use dashmap::{DashMap, DashSet};
use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
pub const PROJECTS_NAMESPACE: &str = "projects:v3";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v3";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v3";
pub const PROJECTS_NAMESPACE: &str = "projects:v4";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v4";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v4";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LinkUrl {
@@ -657,12 +658,14 @@ impl DBProject {
.await?;
let loader_field_enum_values: Vec<QueryLoaderFieldEnumValue> = sqlx::query!(
"
SELECT DISTINCT id, enum_id, value, ordering, created, metadata
r#"
SELECT DISTINCT id, enum_id, value, ordering, created,
metadata->>'type' AS "ty?",
(metadata->>'major')::boolean AS "major?"
FROM loader_field_enum_values lfev
WHERE id = ANY($1)
ORDER BY enum_id, ordering, created DESC
",
"#,
&loader_field_enum_value_ids
.iter()
.map(|x| x.0)
@@ -675,7 +678,8 @@ impl DBProject {
value: m.value,
ordering: m.ordering,
created: m.created,
metadata: m.metadata,
ty: m.ty,
major: m.major,
})
.try_collect()
.await?;
@@ -946,7 +950,7 @@ impl DBProject {
},
)
.await
.wrap_internal_err("failed to fetch cached projects")?;
.wrap_internal_err("fetching cached projects")?;
Ok(val)
}
@@ -1041,8 +1045,10 @@ impl DBProject {
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde_binhum]
#[derive(Clone, Debug)]
pub struct ProjectQueryResult {
#[serde(flatten)]
pub inner: DBProject,
pub categories: Vec<String>,
pub additional_categories: Vec<String>,
@@ -1053,6 +1059,5 @@ pub struct ProjectQueryResult {
pub gallery_items: Vec<DBGalleryItem>,
pub thread_id: DBThreadId,
pub aggregate_version_fields: Vec<VersionField>,
#[serde(flatten)]
pub components: exp::ProjectQuery,
}
apps/labrinth/src/database/models/session_item.rs
+3 -3
View File
@@ -10,9 +10,9 @@ use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const SESSIONS_NAMESPACE: &str = "sessions:v3";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v3";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v3";
const SESSIONS_NAMESPACE: &str = "sessions:v4";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v4";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v4";
pub struct SessionBuilder {
pub session: String,
apps/labrinth/src/database/models/team_item.rs
+1 -1
View File
@@ -10,7 +10,7 @@ use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use xredis::RedisPool;
const TEAMS_NAMESPACE: &str = "teams:v3";
const TEAMS_NAMESPACE: &str = "teams:v4";
pub struct TeamBuilder {
pub members: Vec<TeamMemberBuilder>,
apps/labrinth/src/database/models/user_item.rs
+3 -3
View File
@@ -16,9 +16,9 @@ use std::fmt::{Debug, Display};
use std::hash::Hash;
use xredis::RedisPool;
const USERS_NAMESPACE: &str = "users:v3";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v3";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v3";
const USERS_NAMESPACE: &str = "users:v4";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v4";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v4";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBUser {
apps/labrinth/src/database/models/version_item.rs
+13 -8
View File
@@ -17,12 +17,13 @@ use dashmap::{DashMap, DashSet};
use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use std::cmp::Ordering;
use std::collections::HashMap;
use tracing::error;
pub const VERSIONS_NAMESPACE: &str = "versions:v3";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v3";
pub const VERSIONS_NAMESPACE: &str = "versions:v4";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v4";
pub async fn cleanup_unused_attribution_files_and_groups(
transaction: &mut PgTransaction<'_>,
@@ -704,12 +705,14 @@ impl DBVersion {
.await?;
let loader_field_enum_values: Vec<QueryLoaderFieldEnumValue> = sqlx::query!(
"
SELECT DISTINCT id, enum_id, value, ordering, created, metadata
r#"
SELECT DISTINCT id, enum_id, value, ordering, created,
metadata->>'type' AS "ty?",
(metadata->>'major')::boolean AS "major?"
FROM loader_field_enum_values lfev
WHERE id = ANY($1)
ORDER BY enum_id, ordering, created ASC
",
"#,
&loader_field_enum_value_ids
.iter()
.map(|x| x.0)
@@ -722,7 +725,8 @@ impl DBVersion {
value: m.value,
ordering: m.ordering,
created: m.created,
metadata: m.metadata,
ty: m.ty,
major: m.major,
})
.try_collect()
.await?;
@@ -1080,8 +1084,10 @@ impl DBVersion {
}
}
#[derive(Clone, Deserialize, Serialize)]
#[serde_binhum]
#[derive(Clone)]
pub struct VersionQueryResult {
#[serde(flatten)]
pub inner: DBVersion,
pub files: Vec<FileQueryResult>,
@@ -1090,7 +1096,6 @@ pub struct VersionQueryResult {
pub project_types: Vec<String>,
pub games: Vec<String>,
pub dependencies: Vec<DependencyQueryResult>,
#[serde(flatten)]
pub components: exp::VersionQuery,
}
apps/labrinth/src/database/redis.rs
-56
View File
@@ -1,56 +0,0 @@
use std::sync::Arc;
use crate::env::ENV;
struct RedisConfig {
inner: xredis::RedisConfig,
cache_settings: xredis::CacheSettings,
}
impl RedisConfig {
fn from_env() -> Result<Self, xredis::RedisConfigError> {
let inner = xredis::RedisConfig::new(
ENV.REDIS_TOPOLOGY,
ENV.REDIS_CONNECTION_TYPE,
&ENV.REDIS_URL,
ENV.REDIS_WAIT_TIMEOUT_MS,
(
ENV.REDIS_MAX_CONNECTIONS as usize,
ENV.REDIS_MIN_CONNECTIONS,
),
(
ENV.REDIS_CLUSTER_MAX_CONNECTIONS as usize,
ENV.REDIS_CLUSTER_MIN_CONNECTIONS,
),
(ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0),
ENV.REDIS_CACHE_LOCKING_STRATEGY,
ENV.REDIS_READ_REPLICA_STRATEGY,
)?;
let cache_settings = xredis::CacheSettings {
default_expiry: ENV.REDIS_DEFAULT_EXPIRY,
actual_expiry: ENV.REDIS_ACTUAL_EXPIRY,
version_default_expiry: ENV.REDIS_VERSION_DEFAULT_EXPIRY,
version_actual_expiry: ENV.REDIS_VERSION_ACTUAL_EXPIRY,
encoding_format: ENV.REDIS_ENCODING_FORMAT,
compression_algorithm: ENV.REDIS_COMPRESSION_ALGORITHM,
compression_level: ENV.REDIS_COMPRESSION_LEVEL,
compression_threshold_bytes: ENV.REDIS_COMPRESSION_THRESHOLD_BYTES,
compression_min_savings_ratio: ENV
.REDIS_COMPRESSION_MIN_SAVINGS_RATIO,
};
Ok(Self {
inner,
cache_settings,
})
}
}
pub async fn from_env(
meta_namespace: impl Into<Arc<str>>,
) -> xredis::RedisPool {
let config = RedisConfig::from_env().expect("invalid Redis configuration");
xredis::RedisPool::new(meta_namespace, config.inner, config.cache_settings)
.await
.expect("failed to initialize Redis connections")
}
apps/labrinth/src/database/redis/mod.rs
+411
View File
@@ -0,0 +1,411 @@
use std::sync::Arc;
use crate::env::ENV;
use eyre::{Result, WrapErr};
struct RedisConfig {
inner: xredis::RedisConfig,
cache_settings: xredis::CacheSettings,
}
impl RedisConfig {
fn from_env() -> Result<Self> {
let inner = xredis::RedisConfig::new(
ENV.REDIS_TOPOLOGY,
ENV.REDIS_CONNECTION_TYPE,
&ENV.REDIS_URL,
ENV.REDIS_WAIT_TIMEOUT_MS,
(
ENV.REDIS_MAX_CONNECTIONS as usize,
ENV.REDIS_MIN_CONNECTIONS,
),
(
ENV.REDIS_CLUSTER_MAX_CONNECTIONS as usize,
ENV.REDIS_CLUSTER_MIN_CONNECTIONS,
),
(ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0),
ENV.REDIS_CACHE_LOCKING_STRATEGY,
ENV.REDIS_READ_REPLICA_STRATEGY,
)
.wrap_err("loading Redis configuration from environment")?;
let cache_settings = xredis::CacheSettings {
default_expiry: ENV.REDIS_DEFAULT_EXPIRY,
actual_expiry: ENV.REDIS_ACTUAL_EXPIRY,
version_default_expiry: ENV.REDIS_VERSION_DEFAULT_EXPIRY,
version_actual_expiry: ENV.REDIS_VERSION_ACTUAL_EXPIRY,
encoding_format: ENV.REDIS_ENCODING_FORMAT,
compression_algorithm: ENV.REDIS_COMPRESSION_ALGORITHM,
compression_level: ENV.REDIS_COMPRESSION_LEVEL,
compression_threshold_bytes: ENV.REDIS_COMPRESSION_THRESHOLD_BYTES,
compression_min_savings_ratio: ENV
.REDIS_COMPRESSION_MIN_SAVINGS_RATIO,
};
Ok(Self {
inner,
cache_settings,
})
}
}
pub async fn from_env(
meta_namespace: impl Into<Arc<str>>,
) -> xredis::RedisPool {
let config = RedisConfig::from_env().expect("invalid Redis configuration");
xredis::RedisPool::new(meta_namespace, config.inner, config.cache_settings)
.await
.expect("failed to initialize Redis connections")
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use ::serde::{Serialize, de::DeserializeOwned};
use chrono::Utc;
use url::Url;
use uuid::Uuid;
use webauthn_rs::WebauthnBuilder;
use crate::database::models::flow_item::DBFlow;
use crate::database::models::ids::{
DBNotificationId, DBProjectId, DBTeamId, DBThreadId, DBUserId,
DBVersionId, LoaderFieldEnumId, LoaderFieldEnumValueId, LoaderFieldId,
LoaderId,
};
use crate::database::models::loader_fields::{
Loader, LoaderFieldEnumValue, LoaderMetadata, VersionField,
VersionFieldValue,
};
use crate::database::models::notification_item::DBNotification;
use crate::database::models::project_item::{
DBProject, ProjectQueryResult,
};
use crate::database::models::version_item::{
DBVersion, VersionQueryResult,
};
use crate::models::billing::{Price, ProductMetadata};
use crate::models::exp::{self, minecraft};
use crate::models::notifications::NotificationBody;
use crate::models::projects::{
MonetizationStatus, ProjectStatus, SideTypesMigrationReviewStatus,
VersionStatus,
};
fn postcard_round_trip<T>(value: &T) -> T
where
T: Serialize + DeserializeOwned,
{
let serialized =
postcard::to_allocvec(value).expect("serializing with postcard");
postcard::from_bytes(&serialized).expect("deserializing with postcard")
}
fn loader_field_enum_value(ty: &str, major: bool) -> LoaderFieldEnumValue {
LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(1),
enum_id: LoaderFieldEnumId(2),
value: "1.21.8".to_string(),
ordering: None,
created: Utc::now(),
ty: Some(ty.to_string()),
major: Some(major),
}
}
fn db_project() -> DBProject {
let now = Utc::now();
DBProject {
id: DBProjectId(1),
team_id: DBTeamId(2),
organization_id: None,
name: "project".to_string(),
summary: "summary".to_string(),
description: "description".to_string(),
published: now,
updated: now,
approved: Some(now),
queued: None,
status: ProjectStatus::Approved,
requested_status: None,
downloads: 3,
follows: 4,
icon_url: None,
raw_icon_url: None,
license_url: None,
license: "MIT".to_string(),
slug: Some("project".to_string()),
moderation_message: None,
moderation_message_body: None,
webhook_sent: false,
color: None,
monetization_status: MonetizationStatus::Monetized,
side_types_migration_review_status:
SideTypesMigrationReviewStatus::Reviewed,
loaders: vec!["fabric".to_string()],
components: exp::ProjectSerial::default(),
}
}
fn db_version() -> DBVersion {
DBVersion {
id: DBVersionId(1),
project_id: DBProjectId(2),
author_id: DBUserId(3),
name: "version".to_string(),
version_number: "1.0.0".to_string(),
changelog: "changelog".to_string(),
date_published: Utc::now(),
downloads: 4,
version_type: "release".to_string(),
featured: true,
status: VersionStatus::Listed,
requested_status: None,
ordering: None,
components: exp::VersionSerial::default(),
}
}
#[test]
fn loader_metadata_round_trips_with_postcard() {
for platform in [None, Some(false), Some(true)] {
let metadata = LoaderMetadata { platform };
let round_tripped = postcard_round_trip(&metadata);
assert_eq!(round_tripped.platform, platform);
}
}
#[test]
fn loader_round_trips_with_postcard() {
for platform in [None, Some(false), Some(true)] {
let loader = Loader {
id: LoaderId(1),
loader: "paper".to_string(),
icon: "icon".to_string(),
supported_project_types: vec!["plugin".to_string()],
supported_games: vec!["minecraft-java".to_string()],
metadata: LoaderMetadata { platform },
};
let round_tripped = postcard_round_trip(&loader);
assert_eq!(round_tripped.id, loader.id);
assert_eq!(round_tripped.loader, loader.loader);
assert_eq!(round_tripped.icon, loader.icon);
assert_eq!(
round_tripped.supported_project_types,
loader.supported_project_types
);
assert_eq!(round_tripped.supported_games, loader.supported_games);
assert_eq!(round_tripped.metadata.platform, platform);
}
}
#[test]
fn loader_field_enum_values_round_trip_with_postcard() {
let metadata_values = [
(None, None),
(Some("snapshot"), Some(false)),
(Some("alpha"), Some(false)),
(Some("beta"), Some(true)),
(Some("release"), Some(true)),
(Some("beta"), Some(false)),
(Some("release"), Some(false)),
];
for (ty, major) in metadata_values {
let enum_value = LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(1),
enum_id: LoaderFieldEnumId(2),
value: "1.21.8".to_string(),
ordering: None,
created: Utc::now(),
ty: ty.map(str::to_string),
major,
};
assert_eq!(postcard_round_trip(&enum_value), enum_value);
}
}
#[test]
fn loader_field_enum_value_keeps_flattened_json_layout() {
let enum_value = loader_field_enum_value("release", true);
let json = serde_json::to_value(&enum_value)
.expect("serializing loader field enum value as JSON");
assert_eq!(json.get("type"), Some(&serde_json::json!("release")));
assert_eq!(json.get("major"), Some(&serde_json::json!(true)));
assert!(json.get("metadata").is_none());
assert_eq!(
serde_json::from_value::<LoaderFieldEnumValue>(json)
.expect("deserializing loader field enum value from JSON"),
enum_value
);
}
#[test]
fn non_enum_version_fields_round_trip_with_postcard() {
let values = [
VersionFieldValue::Integer(5),
VersionFieldValue::Text("value".to_string()),
VersionFieldValue::Boolean(true),
VersionFieldValue::ArrayInteger(vec![1, 2]),
VersionFieldValue::ArrayText(vec!["one".to_string()]),
VersionFieldValue::ArrayBoolean(vec![true, false]),
];
for value in values {
let field = VersionField {
version_id: DBVersionId(1),
field_id: LoaderFieldId(2),
field_name: "field".to_string(),
value,
};
assert_eq!(postcard_round_trip(&field), field);
}
}
#[test]
fn flattened_cache_values_round_trip_with_postcard() {
let enum_value = loader_field_enum_value("release", true);
postcard_round_trip(&enum_value);
let version = VersionQueryResult {
inner: db_version(),
files: Vec::new(),
version_fields: vec![VersionField {
version_id: DBVersionId(1),
field_id: LoaderFieldId(2),
field_name: "game_versions".to_string(),
value: VersionFieldValue::Enum(
LoaderFieldEnumId(2),
enum_value,
),
}],
loaders: vec!["fabric".to_string()],
project_types: vec!["mod".to_string()],
games: vec!["minecraft-java".to_string()],
dependencies: Vec::new(),
components: exp::VersionQuery::default(),
};
postcard_round_trip(&version);
let project = ProjectQueryResult {
inner: db_project(),
categories: Vec::new(),
additional_categories: Vec::new(),
versions: vec![DBVersionId(1)],
project_types: vec!["mod".to_string()],
games: vec!["minecraft-java".to_string()],
urls: Vec::new(),
gallery_items: Vec::new(),
thread_id: DBThreadId(1),
aggregate_version_fields: Vec::new(),
components: exp::ProjectQuery::default(),
};
postcard_round_trip(&project);
}
#[test]
fn skipped_cache_fields_round_trip_with_postcard() {
postcard_round_trip(&exp::ProjectSerial::default());
postcard_round_trip(&exp::ProjectQuery::default());
postcard_round_trip(&db_project());
}
#[test]
fn internally_tagged_cache_values_round_trip_with_postcard() {
for metadata in [
ProductMetadata::Midas,
ProductMetadata::Pyro {
cpu: 1,
ram: 2,
swap: 3,
storage: 4,
},
ProductMetadata::Medal {
cpu: 1,
ram: 2,
swap: 3,
storage: 4,
region: "us-east".to_string(),
},
] {
postcard_round_trip(&metadata);
}
for price in [
Price::OneTime { price: 500 },
Price::Recurring {
intervals: HashMap::new(),
},
] {
postcard_round_trip(&price);
}
postcard_round_trip(&NotificationBody::TwoFactorEnabled);
postcard_round_trip(&DBNotification {
id: DBNotificationId(1),
user_id: DBUserId(2),
body: NotificationBody::TwoFactorEnabled,
read: false,
created: Utc::now(),
});
postcard_round_trip(&minecraft::ServerContent::Vanilla {
supported_game_versions: vec!["1.21.8".to_string()],
recommended_game_version: Some("1.21.8".to_string()),
});
postcard_round_trip(&minecraft::ServerContentQuery::Vanilla {
supported_game_versions: vec!["1.21.8".to_string()],
recommended_game_version: Some("1.21.8".to_string()),
});
}
#[test]
fn simple_flow_variants_round_trip_with_postcard() {
assert!(matches!(
postcard_round_trip(&DBFlow::MinecraftAuth),
DBFlow::MinecraftAuth
));
let flow = postcard_round_trip(&DBFlow::Login2FA {
user_id: DBUserId(1),
});
assert!(matches!(
flow,
DBFlow::Login2FA {
user_id: DBUserId(1)
}
));
}
#[test]
fn passkey_flow_variants_round_trip_with_postcard() {
let origin = Url::parse("https://example.com").unwrap();
let webauthn = WebauthnBuilder::new("example.com", &origin)
.unwrap()
.build()
.unwrap();
let (_, registration) = webauthn
.start_passkey_registration(
Uuid::from_u128(1),
"user@example.com",
"user",
None,
)
.unwrap();
postcard_round_trip(&DBFlow::RegisterPasskey {
user_id: DBUserId(1),
state: registration,
});
let (_, authentication) =
webauthn.start_discoverable_authentication().unwrap();
postcard_round_trip(&DBFlow::AuthenticatePasskey {
state: authentication,
});
}
}
apps/labrinth/src/env.rs
+1 -1
View File
@@ -165,7 +165,7 @@ vars! {
REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 256u32;
// The encoding format used for Redis cache values.
REDIS_ENCODING_FORMAT: xredis::EncodingFormat = xredis::EncodingFormat::Json;
REDIS_ENCODING_FORMAT: xredis::EncodingFormat = xredis::EncodingFormat::Postcard;
// The level of LZ4 compression used for Redis cache values. A value of 0 disables compression (supports 1-12)
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
// The compression algorithm used for Redis cache values. Currently only LZ4 is supported.
apps/labrinth/src/models/exp/minecraft.rs
+5 -2
View File
@@ -3,6 +3,7 @@ use std::time::Duration;
use chrono::{DateTime, Utc};
use eyre::Result;
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use tracing::warn;
use validator::Validate;
@@ -324,7 +325,8 @@ impl ComponentEdit for JavaServerProjectEdit {
}
/// What game content a [`JavaServerProject`] is using.
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(Debug, Clone)]
#[serde_binhum(schema)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ServerContent {
/// Server runs modded content with a modpack found on the Modrinth platform.
@@ -346,7 +348,8 @@ pub enum ServerContent {
}
/// What game content a [`JavaServerProject`] is using.
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(Debug, Clone)]
#[serde_binhum(schema)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ServerContentQuery {
/// Server runs modded content with a modpack found on the Modrinth platform.
apps/labrinth/src/models/exp/project.rs
+1 -2
View File
@@ -54,7 +54,7 @@ macro_rules! define_project_components {
pub struct ProjectSerial {
$(
#[validate(nested)]
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub $field_name: Option<$ty>,
)*
}
@@ -114,7 +114,6 @@ macro_rules! define_project_components {
#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)]
pub struct ProjectQuery {
$(
#[serde(skip_serializing_if = "Option::is_none")]
pub $field_name: Option<Query<$ty>>,
)*
}
apps/labrinth/src/models/v3/billing.rs
+3 -2
View File
@@ -4,6 +4,7 @@ use crate::models::ids::{
use ariadne::ids::UserId;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use std::collections::HashMap;
#[derive(Serialize, Deserialize)]
@@ -14,7 +15,7 @@ pub struct Product {
pub unitary: bool,
}
#[derive(Serialize, Deserialize)]
#[serde_binhum]
#[serde(tag = "type", rename_all = "kebab-case")]
pub enum ProductMetadata {
Midas,
@@ -55,7 +56,7 @@ pub struct ProductPrice {
pub currency_code: String,
}
#[derive(Serialize, Deserialize)]
#[serde_binhum]
#[serde(tag = "type", rename_all = "kebab-case")]
pub enum Price {
OneTime {
apps/labrinth/src/models/v3/notifications.rs
+3 -1
View File
@@ -12,6 +12,7 @@ use crate::routes::ApiError;
use ariadne::ids::UserId;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_binhum::serde_binhum;
use uuid::Uuid;
#[derive(Serialize, Deserialize)]
@@ -151,7 +152,8 @@ impl NotificationType {
}
}
#[derive(Serialize, Deserialize, Clone)]
#[derive(Clone)]
#[serde_binhum]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum NotificationBody {
ProjectUpdate {
apps/labrinth/src/queue/analytics/cache.rs
+1 -1
View File
@@ -12,7 +12,7 @@ use crate::{
routes::analytics::MINECRAFT_SERVER_PLAYS, util::error::Context,
};
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v3";
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v4";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MinecraftServerAnalytics {
apps/labrinth/src/queue/analytics/mod.rs
+34 -15
View File
@@ -4,6 +4,7 @@ use crate::models::analytics::{
};
use crate::routes::ApiError;
use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
use crate::util::error::Context;
use dashmap::{DashMap, DashSet};
use std::collections::HashMap;
use tracing::trace;
@@ -11,9 +12,9 @@ use xredis::RedisPool;
pub mod cache;
const DOWNLOADS_NAMESPACE: &str = "downloads:v3";
const VIEWS_NAMESPACE: &str = "views:v3";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v3";
const DOWNLOADS_NAMESPACE: &str = "downloads:v4";
const VIEWS_NAMESPACE: &str = "views:v4";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v4";
const MINECRAFT_SERVER_PLAYS_EXPIRY: u64 = 86_400; // 24 hours
const MINECRAFT_SERVER_PLAYS_LIMIT: u32 = 5;
@@ -142,10 +143,15 @@ impl AnalyticsQueue {
)
})
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut redis_connection =
redis.connect().await.wrap_internal_err(
"connecting to redis for server play counts",
)?;
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
let results = redis_connection
.get_many_typed::<u32>(&redis_keys)
.await
.wrap_internal_err("fetching server play counts from redis")?;
for (idx, count) in results.into_iter().enumerate() {
let new_count = if let Some(count) = count {
if count >= MINECRAFT_SERVER_PLAYS_LIMIT {
@@ -164,7 +170,8 @@ impl AnalyticsQueue {
new_count,
Some(MINECRAFT_SERVER_PLAYS_EXPIRY as i64),
)
.await?;
.await
.wrap_internal_err("writing server play count to redis")?;
}
let mut plays = client
@@ -198,10 +205,15 @@ impl AnalyticsQueue {
)
})
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut redis_connection = redis
.connect()
.await
.wrap_internal_err("connecting to redis for view counts")?;
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
let results = redis_connection
.get_many_typed::<u32>(&redis_keys)
.await
.wrap_internal_err("fetching view counts from redis")?;
for (idx, count) in results.into_iter().enumerate() {
let new_count =
if let Some((views, monetized)) = raw_views.get_mut(idx) {
@@ -226,7 +238,8 @@ impl AnalyticsQueue {
let key = &redis_keys[idx];
redis_connection
.set(key, new_count, Some(6 * 60 * 60))
.await?;
.await
.wrap_internal_err("writing view count to redis")?;
}
let mut views = client.insert::<PageView>("views").await?;
@@ -267,10 +280,15 @@ impl AnalyticsQueue {
)
})
.collect::<Vec<_>>();
let mut redis_connection = redis.connect().await?;
let mut redis_connection = redis
.connect()
.await
.wrap_internal_err("connecting to redis for download counts")?;
let results =
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
let results = redis_connection
.get_many_typed::<u32>(&redis_keys)
.await
.wrap_internal_err("fetching download counts from redis")?;
for (idx, count) in results.into_iter().enumerate() {
let new_count = if let Some(count) = count {
if count > 5 {
@@ -286,7 +304,8 @@ impl AnalyticsQueue {
let key = &redis_keys[idx];
redis_connection
.set(key, new_count, Some(6 * 60 * 60))
.await?;
.await
.wrap_internal_err("writing download count to redis")?;
}
let mut transaction = pool.begin().await?;
apps/labrinth/src/queue/server_ping.rs
+2 -2
View File
@@ -26,9 +26,9 @@ pub struct ServerPingQueue {
pub incremental_search_queue: IncrementalSearchQueue,
}
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v3";
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v4";
pub const REDIS_FAILURE_NAMESPACE: &str =
"minecraft_java_server_ping_failures:v3";
"minecraft_java_server_ping_failures:v4";
pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings";
impl ServerPingQueue {
apps/labrinth/src/routes/internal/campaign.rs
+1 -1
View File
@@ -68,7 +68,7 @@ pub struct CampaignInfo {
cached_at: DateTime<Utc>,
}
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v3";
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v4";
const CAMPAIGN_INFO_CACHE_STALE_SECONDS: i64 = 15 * 60;
const CAMPAIGN_INFO_CACHE_TTL_SECONDS: i64 = 24 * 60 * 60;
apps/labrinth/src/routes/internal/flows.rs
+1 -1
View File
@@ -2213,7 +2213,7 @@ async fn validate_2fa_code(
)
.map_err(|_| AuthenticationError::InvalidCredentials)?;
const TOTP_NAMESPACE: &str = "used_totp:v3";
const TOTP_NAMESPACE: &str = "used_totp:v4";
let mut conn = redis.connect().await?;
let logical_key = format!("{}-{}", input, user_id.0);
let key = redis
apps/labrinth/src/routes/mod.rs
-6
View File
@@ -267,12 +267,6 @@ pub enum ApiError {
},
}
impl From<xredis::Error> for ApiError {
fn from(error: xredis::Error) -> Self {
Self::Database(error.into())
}
}
impl ApiError {
pub fn delphi(err: impl Into<eyre::Error>) -> Self {
Self::Delphi(err.into())
apps/labrinth/src/routes/v2/tags.rs
+2 -11
View File
@@ -210,18 +210,9 @@ pub async fn game_version_list(
.into_iter()
.map(|f| GameVersionQueryData {
version: f.value,
version_type: f
.metadata
.get("type")
.and_then(|m| m.as_str())
.unwrap_or_default()
.to_string(),
version_type: f.ty.unwrap_or_default(),
date: f.created,
major: f
.metadata
.get("major")
.and_then(|m| m.as_bool())
.unwrap_or_default(),
major: f.major.unwrap_or_default(),
})
.collect::<Vec<_>>();
HttpResponse::Ok().json(fields)
apps/labrinth/src/routes/v3/content/mod.rs
+2 -2
View File
@@ -23,8 +23,8 @@ use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use xredis::RedisPool;
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v3";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v3";
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v4";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v4";
const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v3";
const CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS: i64 = 60 * 60 * 24;
apps/labrinth/src/routes/v3/tags.rs
+2 -1
View File
@@ -6,6 +6,7 @@ use crate::database::models::categories::{
};
use crate::database::models::loader_fields::{
Game, Loader, LoaderField, LoaderFieldEnumValue, LoaderFieldType,
LoaderMetadata,
};
use actix_web::{HttpResponse, get, web};
use xredis::RedisPool;
@@ -103,7 +104,7 @@ pub struct LoaderData {
pub supported_project_types: Vec<String>,
pub supported_games: Vec<String>,
pub supported_fields: Vec<String>, // Available loader fields for this loader
pub metadata: Value,
pub metadata: LoaderMetadata,
}
#[utoipa::path(tag = "tags", responses((status = OK)))]
apps/labrinth/src/search/indexing.rs
+7 -4
View File
@@ -394,11 +394,13 @@ async fn build_search_documents(
let loader_field_enum_values: Vec<QueryLoaderFieldEnumValue> =
sqlx::query!(
"
SELECT DISTINCT id, enum_id, value, ordering, created, metadata
r#"
SELECT DISTINCT id, enum_id, value, ordering, created,
metadata->>'type' AS "ty?",
(metadata->>'major')::boolean AS "major?"
FROM loader_field_enum_values lfev
ORDER BY enum_id, ordering, created DESC
"
"#
)
.fetch(pool)
.map_ok(|m| QueryLoaderFieldEnumValue {
@@ -407,7 +409,8 @@ async fn build_search_documents(
value: m.value,
ordering: m.ordering,
created: m.created,
metadata: m.metadata,
ty: m.ty,
major: m.major,
})
.try_collect()
.await?;
apps/labrinth/src/sync/friends.rs
+1 -1
View File
@@ -14,7 +14,7 @@ use redis::{RedisWrite, ToRedisArgs, ToSingleRedisArg};
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc;
pub const FRIENDS_CHANNEL_NAME: &str = "friends:v3";
pub const FRIENDS_CHANNEL_NAME: &str = "friends:v4";
#[derive(Serialize, Deserialize)]
pub enum RedisFriendsMessage {
apps/labrinth/src/sync/status.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use redis::AsyncCommands;
use xredis::RedisPool;
const EXPIRY_TIME_SECONDS: i64 = 60;
const USER_STATUS_NAMESPACE: &str = "user_status:v3";
const USER_STATUS_NAMESPACE: &str = "user_status:v4";
pub async fn get_user_status(
user: UserId,
apps/labrinth/src/util/gotenberg.rs
+1 -1
View File
@@ -14,7 +14,7 @@ pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName =
HeaderName::from_static("modrinth-generated-pdf-type");
pub const MODRINTH_PAYMENT_ID: HeaderName =
HeaderName::from_static("modrinth-payment-id");
pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v3";
pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v4";
const REDIS_TIMEOUT_MARGIN_MS: u64 = 250;
pub(crate) fn payment_statement_key(
apps/labrinth/src/util/ratelimit.rs
+1 -1
View File
@@ -12,7 +12,7 @@ use std::str::FromStr;
use std::sync::Arc;
use xredis::RedisPool;
const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v3";
const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v4";
const RATE_LIMIT_EXPIRY: i64 = 300; // 5 minutes
const MINUTE_IN_NANOS: i64 = 60_000_000_000;
apps/labrinth/tests/loader_fields.rs
+2 -12
View File
@@ -574,12 +574,7 @@ async fn minecraft_game_version_update() {
// A couple specific checks- in the dummy data, all game versions are marked as major=false except 1.20.5
let name_to_major = game_versions
.iter()
.map(|x| {
(
x.value.clone(),
x.metadata.get("major").unwrap().as_bool().unwrap(),
)
})
.map(|x| (x.value.clone(), x.major.unwrap()))
.collect::<std::collections::HashMap<_, _>>();
for (name, major) in name_to_major {
if name == "1.20.5" {
@@ -612,12 +607,7 @@ async fn minecraft_game_version_update() {
let name_to_major = game_versions
.iter()
.map(|x| {
(
x.value.clone(),
x.metadata.get("major").unwrap().as_bool().unwrap(),
)
})
.map(|x| (x.value.clone(), x.major.unwrap()))
.collect::<std::collections::HashMap<_, _>>();
// Confirm that the new version is there
assert!(name_to_major.contains_key("1.20.6"));
apps/labrinth/tests/redis.rs
+20 -20
View File
@@ -22,7 +22,7 @@ use serde_json::json;
use tokio::sync::{Barrier, Notify};
use tokio::time::timeout;
use uuid::Uuid;
use xredis::{KeyBuilder, RedisPool, RedisTopology};
use xredis::{KeyBuilder, RedisPool, RedisTopology, RedisValue};
pub mod common;
@@ -250,7 +250,7 @@ async fn cache_lock_coalesces_concurrent_misses_for_one_key() {
tasks.push(tokio::spawn(async move {
barrier.wait().await;
pool.get_cached_keys_raw(
"single_flight:v3",
"single_flight:v4",
&["shared".to_string()],
move |keys| async move {
fetch_count.fetch_add(1, Ordering::SeqCst);
@@ -292,7 +292,7 @@ async fn cache_lock_coalesces_only_overlapping_keys() {
tasks.push(tokio::spawn(async move {
barrier.wait().await;
pool.get_cached_keys_raw(
"overlapping_locks:v3",
"overlapping_locks:v4",
&requested,
move |keys| async move {
tokio::time::sleep(Duration::from_millis(75)).await;
@@ -333,7 +333,7 @@ async fn cache_lock_does_not_block_independent_keys() {
let slow = tokio::spawn(async move {
slow_pool
.get_cached_keys_raw(
"independent_locks:v3",
"independent_locks:v4",
&["slow".to_string()],
move |keys| async move {
slow_started.notify_one();
@@ -350,7 +350,7 @@ async fn cache_lock_does_not_block_independent_keys() {
let fast = timeout(
Duration::from_secs(1),
pool.get_cached_keys_raw(
"independent_locks:v3",
"independent_locks:v4",
&["fast".to_string()],
|keys| async move {
let values = DashMap::new();
@@ -379,7 +379,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() {
let failed = pool
.get_cached_keys_raw(
"error_recovery:v3",
"error_recovery:v4",
&["key".to_string()],
|_| async {
Err::<DashMap<String, String>, _>(DatabaseError::Internal(
@@ -393,7 +393,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() {
let recovered = timeout(
Duration::from_secs(1),
pool.get_cached_keys_raw(
"error_recovery:v3",
"error_recovery:v4",
&["key".to_string()],
|keys| async move {
let values = DashMap::new();
@@ -413,7 +413,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() {
let cancelled = tokio::spawn(async move {
cancelled_pool
.get_cached_keys_raw(
"cancellation_recovery:v3",
"cancellation_recovery:v4",
&["key".to_string()],
move |_| async move {
cancelled_started.notify_one();
@@ -432,7 +432,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() {
let recovered = timeout(
Duration::from_secs(1),
pool.get_cached_keys_raw(
"cancellation_recovery:v3",
"cancellation_recovery:v4",
&["key".to_string()],
|keys| async move {
let values = DashMap::new();
@@ -452,19 +452,19 @@ async fn cache_lock_is_released_after_error_and_cancellation() {
#[actix_rt::test]
async fn expired_cache_value_serves_waiter_while_writer_refreshes() {
let pool = isolated_redis_pool("stale_while_revalidate").await;
let namespace = "stale_while_revalidate:v3";
let namespace = "stale_while_revalidate:v4";
let logical_key = "key".to_string();
let mut connection = pool.connect().await.unwrap();
let redis_key = connection.key().entity(namespace, &logical_key);
connection
.set_serialized(
&redis_key,
json!({
"key": logical_key,
"alias": null,
"iat": 0,
"val": "stale",
}),
RedisValue::<String, String, String>::new(
logical_key.clone(),
None,
0,
"stale".to_string(),
),
None,
)
.await
@@ -539,8 +539,8 @@ async fn case_insensitive_slug_requests_share_one_cache_lock() {
let requested = vec![requested];
barrier.wait().await;
pool.get_cached_keys_raw_with_slug(
"slug_values:v3",
Some("slug_aliases:v3"),
"slug_values:v4",
Some("slug_aliases:v4"),
false,
&requested,
move |_| async move {
@@ -651,11 +651,11 @@ async fn many_get_routes_handle_cross_slot_cache_lifecycle() {
redis.key().entity(VERSIONS_NAMESPACE, alpha_version_id),
redis.key().entity(VERSIONS_NAMESPACE, beta_version_id),
redis.key().entity(
"versions_files:v3",
"versions_files:v4",
format!("sha1_{}", alpha.file_hash),
),
redis.key().entity(
"versions_files:v3",
"versions_files:v4",
format!("sha1_{}", beta.file_hash),
),
];
apps/labrinth/tests/tags.rs
+1 -6
View File
@@ -47,12 +47,7 @@ async fn get_tags_v3() {
let loader_metadata = loaders
.into_iter()
.map(|x| {
(
x.name,
x.metadata.get("platform").and_then(|x| x.as_bool()),
)
})
.map(|x| (x.name, x.metadata.platform))
.collect::<HashMap<_, _>>();
let loader_names =
loader_metadata.keys().cloned().collect::<HashSet<String>>();
packages/api-client/src/modules/labrinth/types.ts
+4 -3
View File
@@ -1153,9 +1153,10 @@ export namespace Labrinth {
side_types_migration_review_status: 'reviewed' | 'pending'
environment?: Environment[]
minecraft_server?: MinecraftServer
minecraft_java_server?: MinecraftJavaServer
minecraft_bedrock_server?: MinecraftBedrockServer
minecraft_server?: MinecraftServer | null
minecraft_java_server?: MinecraftJavaServer | null
minecraft_bedrock_server?: MinecraftBedrockServer | null
minecraft_mod?: unknown | null
/**
* @deprecated Not recommended to use.
packages/serde-binhum/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "serde-binhum"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
repository.workspace = true
[lib]
proc-macro = true
[dependencies]
darling = { workspace = true }
proc-macro2 = { workspace = true }
quote = { workspace = true }
syn = { workspace = true, features = ["full"] }
[lints]
workspace = true
packages/serde-binhum/README.md
+154
View File
@@ -0,0 +1,154 @@
Implement `serde::{Serialize, Deserialize}` on a type with different behavior for human-readable and non-human-readable (binary) formats.
# Motivation
Serde has the concept of human-readable and non-human-readable de/serializers. Human-readable ones, like JSON, are - well - readable by humans, and are usually verbose and self-describing. Human-readable format deserializers implement `deserialize_any`, which lets Serde do more complicated things like internally tagged enums and `serde(flatten)`. However, binary formats, like Postcard, cannot implement `deserialize_any`, and attempting to deserialize a value using `serde(flatten)` using one of these will fail.
```rust
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind")]
enum MyEnum {
Foo,
Bar {
x: i32,
#[serde(flatten)]
data: BarData,
},
}
#[derive(serde::Serialize, serde::Deserialize)]
struct BarData {
y: i32,
}
let error = postcard::to_allocvec(&MyEnum::Bar {
x: 1,
data: BarData { y: 2 },
})
.unwrap_err();
```
To fix this, we can generate two `De/Serialize` implementations: one for human-readable, and another for binary formats. Then, the `De/Serialize` impl on the actual type will delegate to one of those two:
```rust
enum MyEnum {
Foo,
Bar {
x: i32,
data: BarData,
},
}
// note: this type doesn't use any features which require `deserialize_any`,
// so it can safely derive `De/Serialize` as normal
#[derive(serde::Serialize, serde::Deserialize)]
struct BarData {
y: i32,
}
const _: () = {
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(remote = "MyEnum", tag = "kind")]
enum MyEnumHumanProxy {
Foo,
Bar {
x: i32,
#[serde(flatten)]
data: BarData,
},
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(remote = "MyEnum")]
enum MyEnumBinaryProxy {
Foo,
Bar {
x: i32,
data: BarData,
},
}
impl serde::Serialize for MyEnum {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> Result<S::Ok, S::Error>
{
if serializer.is_human_readable() {
MyEnumHumanProxy::serialize(self, serializer)
} else {
MyEnumBinaryProxy::serialize(self, serializer)
}
}
}
impl<'de> serde::Deserialize<'de> for MyEnum {
fn deserialize<D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Self, D::Error>
{
if deserializer.is_human_readable() {
MyEnumHumanProxy::deserialize(deserializer)
} else {
MyEnumBinaryProxy::deserialize(deserializer)
}
}
}
};
```
# Usage
Add `#[serde_binhum::serde_binhum]` on a type, and remove `#[derive(serde::Serialize, serde::Deserialize)]`.
Ordinary `#[serde(...)]` attributes describe the human-readable representation. `serde-binhum` removes attributes such as `tag` and `flatten` from the generated binary proxy:
```rust
#[serde_binhum::serde_binhum]
#[derive(Debug, PartialEq)]
#[serde(tag = "kind")]
enum MyEnum {
Foo,
Bar {
x: i32,
#[serde(flatten)]
data: BarData,
},
}
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct BarData {
y: i32,
}
let value = MyEnum::Bar {
x: 1,
data: BarData { y: 2 },
};
let json = serde_json::to_value(&value).unwrap();
assert_eq!(
json,
serde_json::json!({
"kind": "Bar",
"x": 1,
"y": 2,
}),
);
let bytes = postcard::to_allocvec(&value).unwrap();
let decoded = postcard::from_bytes::<MyEnum>(&bytes).unwrap();
assert_eq!(decoded, value);
```
Representation-specific Serde options can be added with `human(...)` and `binary(...)`:
```rust
#[serde_binhum::serde_binhum]
struct Value {
#[serde_binhum(human(flatten), binary(with = "binary_data"))]
data: Data,
}
```
Use `#[serde_binhum::serde_binhum(schema)]` to forward `utoipa::PartialSchema` and `utoipa::ToSchema` to the human-readable proxy.
packages/serde-binhum/src/lib.rs
+390
View File
@@ -0,0 +1,390 @@
#![doc = include_str!("../README.md")]
use darling::{FromMeta, ast::NestedMeta};
use proc_macro::TokenStream;
use quote::{format_ident, quote};
use syn::punctuated::Punctuated;
use syn::{
Attribute, Error, Fields, Ident, Item, ItemEnum, ItemStruct, Meta, Path,
Result, Token, parse_macro_input, parse_quote,
};
#[derive(Default, FromMeta)]
struct Args {
#[darling(default)]
schema: bool,
}
#[derive(Default, FromMeta)]
struct AttributeArgs {
#[darling(default)]
human: Option<SerdeOptions>,
#[darling(default)]
binary: Option<SerdeOptions>,
}
struct SerdeOptions(Vec<Meta>);
impl FromMeta for SerdeOptions {
fn from_list(items: &[NestedMeta]) -> darling::Result<Self> {
let options = items
.iter()
.map(|item| match item {
NestedMeta::Meta(option) => Ok(option.clone()),
NestedMeta::Lit(literal) => {
Err(darling::Error::custom("expected a Serde option")
.with_span(literal))
}
})
.collect::<darling::Result<Vec<_>>>()?;
Ok(Self(options))
}
}
#[derive(Clone, Copy)]
enum Representation {
Human,
Binary,
}
#[proc_macro_attribute]
/// Implements `Serialize` and `Deserialize` through generated human and binary
/// remote proxies.
///
/// The annotated type must not derive either trait itself. Ordinary
/// `#[serde(...)]` attributes define the human-readable representation.
///
/// # Attributes
///
/// - **`#[serde_binhum]`**: Generates human-readable and binary Serde proxies
/// and delegates `Serialize` and `Deserialize` to the appropriate proxy.
/// - **`#[serde_binhum(schema)]`**: Also forwards `utoipa::PartialSchema` and
/// `utoipa::ToSchema` to the human-readable proxy.
/// - **`#[serde_binhum(human(...))]`**: Adds the enclosed Serde options only
/// to the human-readable proxy. This can be placed on the type, a variant, or
/// a field.
/// - **`#[serde_binhum(binary(...))]`**: Adds the enclosed Serde options only
/// to the binary proxy. This can be placed on the type, a variant, or a field.
pub fn serde_binhum(args: TokenStream, input: TokenStream) -> TokenStream {
let args = match NestedMeta::parse_meta_list(args.into())
.map_err(darling::Error::from)
.and_then(|args| Args::from_list(&args))
{
Ok(args) => args,
Err(error) => return error.write_errors().into(),
};
let item = parse_macro_input!(input as Item);
expand(args, item)
.unwrap_or_else(|error| error.to_compile_error())
.into()
}
fn expand(args: Args, item: Item) -> Result<proc_macro2::TokenStream> {
match item {
Item::Enum(item) => expand_enum(args, item),
Item::Struct(item) => expand_struct(args, item),
item => Err(Error::new_spanned(
item,
"`serde_binhum` only supports structs and enums",
)),
}
}
fn expand_enum(
args: Args,
mut item: ItemEnum,
) -> Result<proc_macro2::TokenStream> {
validate_item(&item.ident, &item.generics.params, &item.attrs)?;
let ident = item.ident.clone();
let human = enum_proxy(&item, Representation::Human, args.schema)?;
let binary = enum_proxy(&item, Representation::Binary, false)?;
clean_attributes(&mut item.attrs);
for variant in &mut item.variants {
clean_attributes(&mut variant.attrs);
clean_fields(&mut variant.fields);
}
let implementations = implementations(&ident, args.schema);
Ok(quote! {
#item
const _: () = {
#human
#binary
#implementations
};
})
}
fn expand_struct(
args: Args,
mut item: ItemStruct,
) -> Result<proc_macro2::TokenStream> {
validate_item(&item.ident, &item.generics.params, &item.attrs)?;
let ident = item.ident.clone();
let human = struct_proxy(&item, Representation::Human, args.schema)?;
let binary = struct_proxy(&item, Representation::Binary, false)?;
clean_attributes(&mut item.attrs);
clean_fields(&mut item.fields);
let implementations = implementations(&ident, args.schema);
Ok(quote! {
#item
const _: () = {
#human
#binary
#implementations
};
})
}
fn validate_item(
ident: &Ident,
generics: &Punctuated<syn::GenericParam, Token![,]>,
attrs: &[Attribute],
) -> Result<()> {
if !generics.is_empty() {
return Err(Error::new_spanned(
generics,
"`serde_binhum` does not yet support generic types",
));
}
for attr in attrs.iter().filter(|attr| attr.path().is_ident("derive")) {
let derives = attr
.parse_args_with(Punctuated::<Path, Token![,]>::parse_terminated)?;
for derive in derives {
let Some(name) = derive.segments.last() else {
continue;
};
if name.ident == "Serialize" || name.ident == "Deserialize" {
return Err(Error::new_spanned(
derive,
format!(
"`{ident}` must not derive `Serialize` or `Deserialize`; `#[serde_binhum]` implements both"
),
));
}
}
}
Ok(())
}
fn enum_proxy(
item: &ItemEnum,
representation: Representation,
schema: bool,
) -> Result<ItemEnum> {
let mut proxy = item.clone();
proxy.ident = match representation {
Representation::Human => format_ident!("HumanProxy"),
Representation::Binary => format_ident!("BinaryProxy"),
};
proxy.vis = syn::Visibility::Inherited;
proxy.attrs = proxy_item_attributes(
&item.attrs,
&item.ident,
representation,
schema,
)?;
for variant in &mut proxy.variants {
variant.attrs = proxy_attributes(&variant.attrs, representation)?;
proxy_fields(&mut variant.fields, representation)?;
}
Ok(proxy)
}
fn struct_proxy(
item: &ItemStruct,
representation: Representation,
schema: bool,
) -> Result<ItemStruct> {
let mut proxy = item.clone();
proxy.ident = match representation {
Representation::Human => format_ident!("HumanProxy"),
Representation::Binary => format_ident!("BinaryProxy"),
};
proxy.vis = syn::Visibility::Inherited;
proxy.attrs = proxy_item_attributes(
&item.attrs,
&item.ident,
representation,
schema,
)?;
proxy_fields(&mut proxy.fields, representation)?;
Ok(proxy)
}
fn proxy_item_attributes(
attrs: &[Attribute],
remote: &Ident,
representation: Representation,
schema: bool,
) -> Result<Vec<Attribute>> {
let mut attrs = proxy_attributes(attrs, representation)?;
let derive = if schema {
parse_quote!(#[derive(serde::Deserialize, serde::Serialize, utoipa::ToSchema)])
} else {
parse_quote!(#[derive(serde::Deserialize, serde::Serialize)])
};
let remote = remote.to_string();
let remote: Attribute = parse_quote!(#[serde(remote = #remote)]);
attrs.insert(0, remote);
attrs.insert(0, derive);
Ok(attrs)
}
fn proxy_fields(
fields: &mut Fields,
representation: Representation,
) -> Result<()> {
for field in fields {
field.attrs = proxy_attributes(&field.attrs, representation)?;
}
Ok(())
}
fn proxy_attributes(
attrs: &[Attribute],
representation: Representation,
) -> Result<Vec<Attribute>> {
let mut output = Vec::new();
let mut representation_options = Vec::new();
for attr in attrs {
if attr.path().is_ident("serde") {
let mut options = attr
.parse_args_with(
Punctuated::<Meta, Token![,]>::parse_terminated,
)?
.into_iter()
.collect::<Vec<_>>();
if matches!(representation, Representation::Binary) {
options.retain(|option| !binary_incompatible(option));
}
if !options.is_empty() {
output.push(parse_quote!(#[serde(#(#options),*)]));
}
} else if attr.path().is_ident("serde_binhum") {
representation_options
.extend(parse_representation_options(attr, representation)?);
} else if attr.path().is_ident("doc") || attr.path().is_ident("schema")
{
output.push(attr.clone());
}
}
if !representation_options.is_empty() {
output.push(parse_quote!(#[serde(#(#representation_options),*)]));
}
Ok(output)
}
fn parse_representation_options(
attr: &Attribute,
representation: Representation,
) -> Result<Vec<Meta>> {
let args = AttributeArgs::from_meta(&attr.meta)
.map_err(|error| Error::new(error.span(), error.to_string()))?;
if args.human.is_none() && args.binary.is_none() {
return Err(Error::new_spanned(
attr,
"expected `human(...)` or `binary(...)`",
));
}
Ok(match representation {
Representation::Human => args.human,
Representation::Binary => args.binary,
}
.map_or_else(Vec::new, |options| options.0))
}
fn binary_incompatible(option: &Meta) -> bool {
let path = option.path();
path.is_ident("flatten")
|| path.is_ident("tag")
|| path.is_ident("content")
|| path.is_ident("untagged")
|| path.is_ident("skip_serializing_if")
}
fn clean_fields(fields: &mut Fields) {
for field in fields {
clean_attributes(&mut field.attrs);
}
}
fn clean_attributes(attrs: &mut Vec<Attribute>) {
attrs.retain(|attr| {
!attr.path().is_ident("serde") && !attr.path().is_ident("serde_binhum")
});
}
fn implementations(ident: &Ident, schema: bool) -> proc_macro2::TokenStream {
let schema = schema.then(|| {
quote! {
impl ::utoipa::PartialSchema for #ident {
fn schema()
-> ::utoipa::openapi::RefOr<::utoipa::openapi::schema::Schema> {
<HumanProxy as ::utoipa::PartialSchema>::schema()
}
}
impl ::utoipa::ToSchema for #ident {
fn schemas(
schemas: &mut ::std::vec::Vec<(
::std::string::String,
::utoipa::openapi::RefOr<::utoipa::openapi::schema::Schema>,
)>,
) {
<HumanProxy as ::utoipa::ToSchema>::schemas(schemas);
}
}
}
});
quote! {
impl ::serde::Serialize for #ident {
fn serialize<S>(
&self,
serializer: S,
) -> ::std::result::Result<S::Ok, S::Error>
where
S: ::serde::Serializer,
{
if serializer.is_human_readable() {
HumanProxy::serialize(self, serializer)
} else {
BinaryProxy::serialize(self, serializer)
}
}
}
impl<'de> ::serde::Deserialize<'de> for #ident {
fn deserialize<D>(
deserializer: D,
) -> ::std::result::Result<Self, D::Error>
where
D: ::serde::Deserializer<'de>,
{
if deserializer.is_human_readable() {
HumanProxy::deserialize(deserializer)
} else {
BinaryProxy::deserialize(deserializer)
}
}
}
#schema
}
}
packages/xredis/Cargo.toml
+2 -1
View File
@@ -10,8 +10,10 @@ ariadne = { workspace = true }
chrono = { workspace = true }
dashmap = { workspace = true }
deadpool-redis = { workspace = true, features = ["cluster-async"] }
eyre = { workspace = true }
futures = { workspace = true }
lz4_flex = { workspace = true }
postcard = { workspace = true }
prometheus = { workspace = true }
redis = { workspace = true, features = [
"ahash",
@@ -20,7 +22,6 @@ redis = { workspace = true, features = [
"tokio-comp"
] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt", "sync", "time"] }
tracing = { workspace = true }
packages/xredis/src/blocking.rs
+34 -24
View File
@@ -1,14 +1,14 @@
use std::time::Duration;
use eyre::{Result, WrapErr, bail};
use prometheus::Registry;
use super::RedisPool;
use super::config::{RedisConfig, RedisTopology};
use super::connection::RedisBackendBuildError;
use super::metrics::{
LogicalPoolStatus, LogicalPoolStatusProvider,
register_blocking_pool_metrics,
};
use super::{Error, RedisPool};
const POOL_RETAIN_INTERVAL: Duration = Duration::from_secs(30);
const MAX_IDLE_CONNECTION_AGE: Duration = Duration::from_secs(5 * 60);
@@ -27,9 +27,7 @@ enum RedisBlockingPoolInner {
}
impl RedisBlockingPool {
pub(super) async fn new(
config: &RedisConfig,
) -> Result<Self, RedisBackendBuildError> {
pub(super) async fn new(config: &RedisConfig) -> Result<Self> {
let pool_size = config.blocking_pool_size();
let inner = match config.topology() {
RedisTopology::Standalone => {
@@ -39,14 +37,16 @@ impl RedisBlockingPool {
let manager = deadpool_redis::Manager::new_with_config(
config.seed_urls()[0].clone(),
connection_config,
)?;
)
.wrap_err("configuring standalone blocking Redis client")?;
let pool = deadpool_redis::Pool::builder(manager)
.max_size(pool_size.max())
.wait_timeout(Some(Duration::from_millis(
config.wait_timeout_ms(),
)))
.runtime(deadpool_redis::Runtime::Tokio1)
.build()?;
.build()
.wrap_err("building standalone blocking Redis pool")?;
retain_standalone_pool(pool.clone());
RedisBlockingPoolInner::Standalone(pool)
}
@@ -54,14 +54,16 @@ impl RedisBlockingPool {
let manager = deadpool_redis::cluster::Manager::new(
config.seed_urls().to_vec(),
false,
)?;
)
.wrap_err("configuring clustered blocking Redis client")?;
let pool = deadpool_redis::cluster::Pool::builder(manager)
.max_size(pool_size.max())
.wait_timeout(Some(Duration::from_millis(
config.wait_timeout_ms(),
)))
.runtime(deadpool_redis::Runtime::Tokio1)
.build()?;
.build()
.wrap_err("building clustered blocking Redis pool")?;
retain_cluster_pool(pool.clone());
RedisBlockingPoolInner::Cluster(pool)
}
@@ -70,10 +72,7 @@ impl RedisBlockingPool {
Ok(Self { inner })
}
pub(super) fn register_metrics(
&self,
registry: &Registry,
) -> Result<(), prometheus::Error> {
pub(super) fn register_metrics(&self, registry: &Registry) -> Result<()> {
register_blocking_pool_metrics(registry, self.clone())
}
@@ -81,22 +80,33 @@ impl RedisBlockingPool {
&self,
key: &str,
timeout: Duration,
) -> Result<Option<[Vec<u8>; 2]>, Error> {
) -> Result<Option<[Vec<u8>; 2]>> {
if timeout.is_zero() {
return Err(Error::InvalidBlockingTimeout);
bail!("redis blocking timeout must be greater than zero");
}
let mut command = redis::cmd("BRPOP");
command.arg(key).arg(timeout.as_secs_f64());
let response: Option<(Vec<u8>, Vec<u8>)> = match &self.inner {
RedisBlockingPoolInner::Standalone(pool) => {
command.query_async(&mut pool.get().await?).await?
}
RedisBlockingPoolInner::Cluster(pool) => {
command.query_async(&mut pool.get().await?).await?
}
};
let response: Option<(Vec<u8>, Vec<u8>)> =
match &self.inner {
RedisBlockingPoolInner::Standalone(pool) => {
let mut connection = pool.get().await.wrap_err(
"fetching standalone blocking Redis connection",
)?;
command.query_async(&mut connection).await.wrap_err(
"reading from standalone Redis blocking queue",
)?
}
RedisBlockingPoolInner::Cluster(pool) => {
let mut connection = pool.get().await.wrap_err(
"fetching clustered blocking Redis connection",
)?;
command.query_async(&mut connection).await.wrap_err(
"reading from clustered Redis blocking queue",
)?
}
};
Ok(response.map(|(key, value)| [key, value]))
}
@@ -120,7 +130,7 @@ impl RedisPool {
&self,
key: &str,
timeout: Duration,
) -> Result<Option<[Vec<u8>; 2]>, Error> {
) -> Result<Option<[Vec<u8>; 2]>> {
self.blocking.brpop(key, timeout).await
}
}
packages/xredis/src/cache.rs
+97 -65
View File
@@ -8,6 +8,7 @@ use std::str::FromStr;
use ariadne::ids::base62_impl::{parse_base62, to_base62};
use chrono::{TimeZone, Utc};
use dashmap::DashMap;
use eyre::{Result, WrapErr, eyre};
use futures::stream::{FuturesUnordered, StreamExt};
use redis::aio::ConnectionLike;
use serde::de::DeserializeOwned;
@@ -16,8 +17,6 @@ use thiserror::Error;
use tokio::time::{Instant, timeout_at};
use tracing::{Instrument, info_span};
use crate::Error;
use super::commands;
use super::connection::RoutableConnection;
use super::key::KeyBuilder;
@@ -33,9 +32,7 @@ const FILL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
pub(super) trait ConnectionProvider {
type Connection: ConnectionLike + RoutableConnection;
fn connect(
&self,
) -> impl Future<Output = Result<Self::Connection, Error>> + Send;
fn connect(&self) -> impl Future<Output = Result<Self::Connection>> + Send;
}
#[derive(Clone, Copy)]
@@ -53,7 +50,7 @@ pub enum Codec {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncodingFormat {
Json,
Postcard,
}
#[derive(Debug, Error)]
@@ -92,7 +89,7 @@ impl FromStr for EncodingFormat {
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"json" => Ok(Self::Json),
"postcard" => Ok(Self::Postcard),
_ => Err(InvalidEncodingFormat),
}
}
@@ -112,12 +109,10 @@ pub struct CacheSettings {
}
impl CacheSettings {
pub fn encode_value<T: Serialize>(
&self,
value: &T,
) -> Result<Vec<u8>, Error> {
pub fn encode_value<T: Serialize>(&self, value: &T) -> Result<Vec<u8>> {
let mut value = match self.encoding_format {
EncodingFormat::Json => serde_json::to_vec(value)?,
EncodingFormat::Postcard => postcard::to_allocvec(value)
.wrap_err("serializing Redis cache value with postcard")?,
};
if self.compression_level > 0
@@ -149,15 +144,23 @@ impl CacheSettings {
T: for<'a> Deserialize<'a>,
{
let (codec, value) = value.split_first()?;
let value = match Codec::try_from(*codec).ok()? {
let Ok(codec) = Codec::try_from(*codec) else {
return None;
};
let value = match codec {
Codec::Raw => Cow::Borrowed(value),
Codec::Lz4 => Cow::Owned(
lz4_flex::block::decompress_size_prepended(value).ok()?,
),
Codec::Lz4 => {
let Ok(value) =
lz4_flex::block::decompress_size_prepended(value)
else {
return None;
};
Cow::Owned(value)
}
};
match self.encoding_format {
EncodingFormat::Json => serde_json::from_slice(&value).ok(),
EncodingFormat::Postcard => postcard::from_bytes(&value).ok(),
}
}
@@ -202,12 +205,12 @@ impl CacheManager {
namespace: &str,
keys: &[K],
closure: F,
) -> Result<Vec<T>, E>
) -> Result<Vec<T>>
where
P: ConnectionProvider,
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
@@ -220,7 +223,8 @@ impl CacheManager {
{
Ok(self
.get_cached_keys_raw(provider, namespace, keys, closure)
.await?
.await
.wrap_err("fetching Redis cache values")?
.into_values()
.collect())
}
@@ -232,12 +236,12 @@ impl CacheManager {
namespace: &str,
keys: &[K],
closure: F,
) -> Result<HashMap<K, T>, E>
) -> Result<HashMap<K, T>>
where
P: ConnectionProvider,
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
@@ -255,11 +259,16 @@ impl CacheManager {
false,
keys,
|ids| async move {
Ok(closure(ids)
.await?
.into_iter()
.map(|(key, value)| (key, (None::<String>, value)))
.collect())
let values = match closure(ids).await {
Ok(values) => values,
Err(error) => return Err(error),
};
Ok::<_, E>(
values
.into_iter()
.map(|(key, value)| (key, (None::<String>, value)))
.collect(),
)
},
)
.await
@@ -274,12 +283,12 @@ impl CacheManager {
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<Vec<T>, E>
) -> Result<Vec<T>>
where
P: ConnectionProvider,
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
@@ -300,7 +309,8 @@ impl CacheManager {
keys,
closure,
)
.await?
.await
.wrap_err("fetching Redis cache values by slug")?
.into_values()
.collect())
}
@@ -314,12 +324,12 @@ impl CacheManager {
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<HashMap<K, T>, E>
) -> Result<HashMap<K, T>>
where
P: ConnectionProvider,
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
@@ -360,7 +370,9 @@ impl CacheManager {
})
.collect::<Vec<_>>();
let mut connection =
provider.connect().await.map_err(E::from)?;
provider.connect().await.wrap_err(
"connecting to Redis for slug lookup",
)?;
let values = match routing {
CacheReadRouting::ReplicaOptional => {
commands::get_many_strings(
@@ -377,8 +389,8 @@ impl CacheManager {
.await
}
}
.map_err(E::from)?;
Ok::<_, E>(
.wrap_err("fetching Redis cache slug values")?;
eyre::Ok(
values
.into_iter()
.flatten()
@@ -386,7 +398,8 @@ impl CacheManager {
)
}
.instrument(info_span!("get slug ids"))
.await?
.await
.wrap_err("resolving Redis cache slugs")?
} else {
Vec::new()
};
@@ -403,8 +416,10 @@ impl CacheManager {
.map(|key| self.key_builder.entity(namespace, key))
.collect::<Vec<_>>();
let mut connection =
provider.connect().await.map_err(E::from)?;
let mut connection = provider
.connect()
.await
.wrap_err("connecting to Redis for cache lookup")?;
let mut cached_values = HashMap::new();
let values = match routing {
CacheReadRouting::ReplicaOptional => {
@@ -415,7 +430,7 @@ impl CacheManager {
.await
}
}
.map_err(E::from)?;
.wrap_err("fetching Redis cache values")?;
for value in values {
if let Some(value) = value.and_then(|value| {
self.settings
@@ -425,7 +440,7 @@ impl CacheManager {
}
}
Ok::<_, E>((cached_values, ids))
eyre::Ok((cached_values, ids))
}
.instrument(info_span!("get_cached_values_closure"))
};
@@ -437,7 +452,9 @@ impl CacheManager {
let deadline = Instant::now() + WAIT_TIMEOUT;
let (cached_values_raw, ids) =
get_cached_values(ids, CacheReadRouting::ReplicaOptional).await?;
get_cached_values(ids, CacheReadRouting::ReplicaOptional)
.await
.wrap_err("reading Redis cache")?;
let mut cached_values = cached_values_raw
.into_iter()
.filter_map(|(key, value)| {
@@ -508,7 +525,10 @@ impl CacheManager {
let values = timeout_at(fill_deadline, closure(fetch_ids))
.await
.map_err(|_| lock_timeout_error(0, waiters.len()))??;
.map_err(|_| lock_timeout_error(0, waiters.len()))
.wrap_err("waiting to fill Redis cache")?;
let values =
values.wrap_err("fetching values to fill Redis cache")?;
let mut return_values = HashMap::new();
let mut encoded_values = Vec::with_capacity(values.len());
@@ -520,13 +540,17 @@ impl CacheManager {
val: value,
alias: slug.clone(),
};
let encoded =
self.settings.encode_value(&value).map_err(E::from)?;
let encoded = self
.settings
.encode_value(&value)
.wrap_err("encoding Redis cache value")?;
encoded_values.push((key, slug, value, encoded));
}
let mut connection =
provider.connect().await.map_err(E::from)?;
let mut connection = provider
.connect()
.await
.wrap_err("connecting to Redis to fill cache")?;
for (key, slug, _, encoded) in &encoded_values {
let redis_key =
self.key_builder.entity(namespace, key.to_string());
@@ -537,7 +561,7 @@ impl CacheManager {
default_expiry,
)
.await
.map_err(E::from)?;
.wrap_err("writing Redis cache value")?;
if let Some(slug) = slug
&& let Some(slug_namespace) = slug_namespace
{
@@ -554,7 +578,7 @@ impl CacheManager {
default_expiry,
)
.await
.map_err(E::from)?;
.wrap_err("writing Redis cache slug")?;
}
}
@@ -563,7 +587,7 @@ impl CacheManager {
return_values.insert(key, value);
}
Result::<_, E>::Ok(return_values)
Result::<_>::Ok(return_values)
}
.await
} else {
@@ -604,13 +628,14 @@ impl CacheManager {
Err(error) => Err(error),
}
}
Err(error) => Err(E::from(error)),
Err(error) => Err(error),
}
}
}
Err(error) => Err(error),
};
cached_values.extend(operation_result?);
cached_values
.extend(operation_result.wrap_err("populating Redis cache")?);
Ok(cached_values
.into_iter()
@@ -679,7 +704,7 @@ fn push_identity(identities: &mut Vec<String>, identity: String) {
async fn wait_for_locks<I>(
waiters: Vec<(I, LockWaiter)>,
deadline: Instant,
) -> Result<Vec<I>, Error> {
) -> Result<Vec<I>> {
let total = waiters.len();
let mut released = Vec::with_capacity(total);
let mut futures = FuturesUnordered::new();
@@ -695,26 +720,24 @@ async fn wait_for_locks<I>(
Ok(()) => {
released.push(key);
}
Err(error)
if is_lock_timeout(&error) || Instant::now() >= deadline =>
{
Err(_) if Instant::now() >= deadline => {
return Err(lock_timeout_error(released.len(), total));
}
Err(error) => return Err(error),
Err(error) => {
return Err(error).wrap_err("waiting for Redis cache lock");
}
}
}
Ok(released)
}
fn is_lock_timeout(error: &Error) -> bool {
matches!(error, Error::LocalCacheTimeout { .. })
}
fn lock_timeout_error(locks_released: usize, locks_waiting: usize) -> Error {
Error::LocalCacheTimeout {
released: locks_released,
total: locks_waiting,
}
fn lock_timeout_error(
locks_released: usize,
locks_waiting: usize,
) -> eyre::Report {
eyre!(
"timeout waiting on local Redis cache lock ({locks_released}/{locks_waiting} released)"
)
}
#[derive(Serialize, Deserialize)]
@@ -726,6 +749,15 @@ pub struct RedisValue<T, K, S> {
}
impl<T, K, S> RedisValue<T, K, S> {
pub fn new(key: K, alias: Option<S>, iat: i64, val: T) -> Self {
Self {
key,
alias,
iat,
val,
}
}
pub fn value(&self) -> &T {
&self.val
}
packages/xredis/src/cache/locking/local.rs
Vendored
+3 -14
View File
@@ -3,11 +3,10 @@ use std::sync::atomic::{AtomicBool, Ordering};
use dashmap::DashMap;
use dashmap::mapref::entry::Entry;
use eyre::{Result, WrapErr};
use tokio::sync::Notify;
use tokio::time::{Instant, timeout_at};
use crate::Error;
#[derive(Clone)]
pub(in crate::cache) struct LockCoordinator {
locks: Arc<DashMap<String, Arc<LockState>>>,
@@ -76,10 +75,7 @@ pub(in crate::cache) struct LockWaiter {
}
impl LockWaiter {
pub(in crate::cache) async fn wait(
self,
deadline: Instant,
) -> Result<(), Error> {
pub(in crate::cache) async fn wait(self, deadline: Instant) -> Result<()> {
loop {
if self.state.released.load(Ordering::Acquire) {
return Ok(());
@@ -94,7 +90,7 @@ impl LockWaiter {
timeout_at(deadline, notified)
.await
.map_err(|_| lock_timeout())?;
.wrap_err("waiting for local Redis cache lock")?;
}
}
}
@@ -112,10 +108,3 @@ impl LockState {
}
}
}
fn lock_timeout() -> Error {
Error::LocalCacheTimeout {
released: 0,
total: 1,
}
}
packages/xredis/src/commands.rs
+60 -45
View File
@@ -1,10 +1,9 @@
use std::fmt::Debug;
use eyre::{Result, WrapErr};
use redis::aio::ConnectionLike;
use redis::{FromRedisValue, ToRedisArgs};
use crate::Error;
use super::cache::CacheSettings;
use super::connection::RoutableConnection;
use super::routing::primary_mget_routing;
@@ -18,7 +17,7 @@ pub async fn set<C, D>(
key: &str,
data: D,
expiry: i64,
) -> Result<(), Error>
) -> Result<()>
where
C: ConnectionLike,
D: ToRedisArgs + Send + Sync + Debug,
@@ -29,7 +28,8 @@ where
.arg("EX")
.arg(expiry)
.query_async::<()>(connection)
.await?;
.await
.wrap_err("writing to Redis")?;
Ok(())
}
@@ -40,7 +40,7 @@ pub async fn set_serialized<C, D>(
data: D,
expiry: Option<i64>,
settings: &CacheSettings,
) -> Result<(), Error>
) -> Result<()>
where
C: ConnectionLike,
D: serde::Serialize,
@@ -48,21 +48,24 @@ where
set(
connection,
key,
settings.encode_value(&data)?,
settings
.encode_value(&data)
.wrap_err("serializing Redis value")?,
expiry.unwrap_or(settings.default_expiry),
)
.await
}
#[tracing::instrument(skip_all)]
pub async fn get<C>(
connection: &mut C,
key: &str,
) -> Result<Option<String>, Error>
pub async fn get<C>(connection: &mut C, key: &str) -> Result<Option<String>>
where
C: ConnectionLike,
{
Ok(cmd("GET").arg(key).query_async(connection).await?)
cmd("GET")
.arg(key)
.query_async(connection)
.await
.wrap_err("fetching from Redis")
}
/// Issues ordinary `MGET` commands in bounded chunks. Cluster routing and
@@ -72,7 +75,7 @@ where
pub async fn get_many<C>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<Vec<u8>>>, Error>
) -> Result<Vec<Option<Vec<u8>>>>
where
C: ConnectionLike,
{
@@ -83,7 +86,7 @@ where
pub async fn get_many_strings<C>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<String>>, Error>
) -> Result<Vec<Option<String>>>
where
C: ConnectionLike,
{
@@ -93,7 +96,7 @@ where
pub(super) async fn get_many_primary<C>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<Vec<u8>>>, Error>
) -> Result<Vec<Option<Vec<u8>>>>
where
C: RoutableConnection,
{
@@ -103,7 +106,7 @@ where
pub(super) async fn get_many_strings_primary<C>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<String>>, Error>
) -> Result<Vec<Option<String>>>
where
C: RoutableConnection,
{
@@ -113,7 +116,7 @@ where
pub(super) async fn get_many_as<C, T>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<T>>, Error>
) -> Result<Vec<Option<T>>>
where
C: ConnectionLike,
T: FromRedisValue,
@@ -123,7 +126,8 @@ where
let part = cmd("MGET")
.arg(chunk)
.query_async::<Vec<Option<T>>>(connection)
.await?;
.await
.wrap_err("fetching multiple values from Redis")?;
values.extend(part);
}
Ok(values)
@@ -132,7 +136,7 @@ where
async fn get_many_primary_as<C, T>(
connection: &mut C,
keys: &[String],
) -> Result<Vec<Option<T>>, Error>
) -> Result<Vec<Option<T>>>
where
C: RoutableConnection,
T: FromRedisValue,
@@ -143,10 +147,14 @@ where
command.arg(chunk);
let value = connection
.route_command(command, primary_mget_routing(chunk))
.await?;
let part =
redis::from_redis_value::<Vec<Option<T>>>(value.extract_error()?)
.map_err(redis::RedisError::from)?;
.await
.wrap_err("fetching multiple values from primary Redis nodes")?;
let value = value
.extract_error()
.wrap_err("extracting Redis response")?;
let part = redis::from_redis_value::<Vec<Option<T>>>(value)
.map_err(redis::RedisError::from)
.wrap_err("decoding Redis response")?;
values.extend(part);
}
Ok(values)
@@ -157,13 +165,16 @@ pub async fn get_deserialized<C, R>(
connection: &mut C,
key: &str,
settings: &CacheSettings,
) -> Result<Option<R>, Error>
) -> Result<Option<R>>
where
C: ConnectionLike,
R: for<'a> serde::Deserialize<'a>,
{
let value: Option<Vec<u8>> =
cmd("GET").arg(key).query_async(connection).await?;
let value: Option<Vec<u8>> = cmd("GET")
.arg(key)
.query_async(connection)
.await
.wrap_err("fetching serialized value from Redis")?;
Ok(value.and_then(|value| settings.decode_value(&value)))
}
@@ -172,47 +183,49 @@ pub async fn get_many_deserialized<C, R>(
connection: &mut C,
keys: &[String],
settings: &CacheSettings,
) -> Result<Vec<Option<R>>, Error>
) -> Result<Vec<Option<R>>>
where
C: ConnectionLike,
R: for<'a> serde::Deserialize<'a>,
{
Ok(get_many(connection, keys)
.await?
.await
.wrap_err("fetching serialized values from Redis")?
.into_iter()
.map(|value| value.and_then(|value| settings.decode_value(&value)))
.collect())
}
#[tracing::instrument(skip_all)]
pub async fn delete<C>(connection: &mut C, key: &str) -> Result<(), Error>
pub async fn delete<C>(connection: &mut C, key: &str) -> Result<()>
where
C: ConnectionLike,
{
cmd("DEL").arg(key).query_async::<()>(connection).await?;
cmd("DEL")
.arg(key)
.query_async::<()>(connection)
.await
.wrap_err("deleting from Redis")?;
Ok(())
}
#[tracing::instrument(skip_all)]
pub async fn delete_many<C>(
connection: &mut C,
keys: &[String],
) -> Result<(), Error>
pub async fn delete_many<C>(connection: &mut C, keys: &[String]) -> Result<()>
where
C: ConnectionLike,
{
if !keys.is_empty() {
cmd("DEL").arg(keys).query_async::<()>(connection).await?;
cmd("DEL")
.arg(keys)
.query_async::<()>(connection)
.await
.wrap_err("deleting multiple values from Redis")?;
}
Ok(())
}
#[tracing::instrument(skip_all)]
pub async fn lpush<C, D>(
connection: &mut C,
key: &str,
value: D,
) -> Result<(), Error>
pub async fn lpush<C, D>(connection: &mut C, key: &str, value: D) -> Result<()>
where
C: ConnectionLike,
D: ToRedisArgs + Send + Sync + Debug,
@@ -221,17 +234,19 @@ where
.arg(key)
.arg(value)
.query_async::<()>(connection)
.await?;
.await
.wrap_err("pushing to Redis list")?;
Ok(())
}
#[tracing::instrument(skip_all)]
pub async fn incr<C>(
connection: &mut C,
key: &str,
) -> Result<Option<u64>, Error>
pub async fn incr<C>(connection: &mut C, key: &str) -> Result<Option<u64>>
where
C: ConnectionLike,
{
Ok(cmd("INCR").arg(key).query_async(connection).await?)
cmd("INCR")
.arg(key)
.query_async(connection)
.await
.wrap_err("incrementing Redis value")
}
packages/xredis/src/config.rs
+30 -22
View File
@@ -1,5 +1,6 @@
use std::{fmt, str::FromStr};
use eyre::{Result, WrapErr};
use thiserror::Error;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
@@ -91,13 +92,11 @@ pub(crate) struct RedisPoolSize {
}
impl RedisPoolSize {
fn new(
name: &'static str,
max: usize,
min: usize,
) -> Result<Self, RedisConfigError> {
fn new(name: &'static str, max: usize, min: usize) -> Result<Self> {
if max == 0 || min > max {
return Err(RedisConfigError::InvalidPoolSize { name, max, min });
return Err(
RedisConfigError::InvalidPoolSize { name, max, min }.into()
);
}
Ok(Self { max, min })
@@ -193,11 +192,12 @@ impl RedisConfig {
blocking_pool_size: (usize, usize),
cache_locking_strategy: CacheLockingStrategy,
read_replica_strategy: ReadReplicaStrategy,
) -> Result<Self, RedisConfigError> {
) -> Result<Self> {
if cache_locking_strategy == CacheLockingStrategy::Distributed {
return Err(RedisConfigError::UnsupportedCacheLockingStrategy {
strategy: cache_locking_strategy,
});
}
.into());
}
let seed_urls = raw_urls
@@ -208,26 +208,32 @@ impl RedisConfig {
.collect::<Vec<_>>();
if seed_urls.is_empty() {
return Err(RedisConfigError::MissingUrl);
return Err(RedisConfigError::MissingUrl.into());
}
let backend = match (mode, connection_type) {
(RedisTopology::Standalone, RedisConnectionType::Pooled) => {
if seed_urls.len() != 1 {
return Err(RedisConfigError::MultipleStandaloneUrls);
return Err(RedisConfigError::MultipleStandaloneUrls.into());
}
RedisBackendConfig::StandalonePooled(RedisPoolSize::new(
"standalone",
standalone_pool_size.0,
standalone_pool_size.1,
)?)
RedisBackendConfig::StandalonePooled(
RedisPoolSize::new(
"standalone",
standalone_pool_size.0,
standalone_pool_size.1,
)
.wrap_err("validating standalone Redis pool size")?,
)
}
(RedisTopology::Cluster, RedisConnectionType::Pooled) => {
RedisBackendConfig::ClusterPooled(RedisPoolSize::new(
"cluster",
cluster_pool_size.0,
cluster_pool_size.1,
)?)
RedisBackendConfig::ClusterPooled(
RedisPoolSize::new(
"cluster",
cluster_pool_size.0,
cluster_pool_size.1,
)
.wrap_err("validating clustered Redis pool size")?,
)
}
(RedisTopology::Cluster, RedisConnectionType::Multiplexed) => {
RedisBackendConfig::ClusterMultiplexed
@@ -236,7 +242,8 @@ impl RedisConfig {
return Err(RedisConfigError::UnsupportedConnectionType {
mode,
connection_type,
});
}
.into());
}
};
@@ -249,7 +256,8 @@ impl RedisConfig {
"blocking",
blocking_pool_size.0,
blocking_pool_size.1,
)?,
)
.wrap_err("validating blocking Redis pool size")?,
cache_locking_strategy,
read_replica_strategy,
})
packages/xredis/src/connection.rs
+46 -42
View File
@@ -1,5 +1,6 @@
use std::time::Duration;
use eyre::{Result, WrapErr};
use futures::future::try_join_all;
use prometheus::Registry;
use redis::aio::ConnectionLike;
@@ -7,7 +8,6 @@ use redis::cluster_read_routing::{
RandomReplicaStrategy, RoundRobinReplicaStrategy,
};
use redis::cluster_routing::RoutingInfo;
use thiserror::Error;
use tracing::warn;
use crate::ReadReplicaStrategy;
@@ -29,16 +29,6 @@ pub(crate) enum RedisBackend {
ClusterMultiplexed(redis::cluster_async::ClusterConnection),
}
#[derive(Debug, Error)]
pub enum RedisBackendBuildError {
#[error("failed to configure Redis client: {0}")]
Redis(#[from] redis::RedisError),
#[error("failed to build Redis pool: {0}")]
PoolBuild(#[from] deadpool_redis::BuildError),
#[error("failed to establish initial Redis pool connections: {0}")]
Pool(#[from] deadpool_redis::PoolError),
}
pub(crate) struct RedisConnection {
inner: RedisConnectionInner,
}
@@ -58,9 +48,7 @@ pub(crate) trait RoutableConnection: ConnectionLike {
}
impl RedisBackend {
pub(crate) async fn new(
config: &RedisConfig,
) -> Result<Self, RedisBackendBuildError> {
pub(crate) async fn new(config: &RedisConfig) -> Result<Self> {
match config.backend() {
RedisBackendConfig::StandalonePooled(pool_size) => {
Self::standalone_pooled(config, pool_size).await
@@ -77,21 +65,25 @@ impl RedisBackend {
async fn standalone_pooled(
config: &RedisConfig,
pool_size: RedisPoolSize,
) -> Result<Self, RedisBackendBuildError> {
) -> Result<Self> {
let connection_config = redis::AsyncConnectionConfig::new()
.set_connection_timeout(None)
.set_response_timeout(None);
let manager = deadpool_redis::Manager::new_with_config(
config.seed_urls()[0].clone(),
connection_config,
)?;
)
.wrap_err("configuring standalone Redis client")?;
let pool = deadpool_redis::Pool::builder(manager)
.max_size(pool_size.max())
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
.runtime(deadpool_redis::Runtime::Tokio1)
.build()?;
.build()
.wrap_err("building standalone Redis pool")?;
warm_standalone_pool(&pool, pool_size.min()).await?;
warm_standalone_pool(&pool, pool_size.min())
.await
.wrap_err("warming standalone Redis pool")?;
retain_standalone_pool(pool.clone());
Ok(Self::StandalonePooled(pool))
@@ -100,16 +92,18 @@ impl RedisBackend {
async fn cluster_pooled(
config: &RedisConfig,
pool_size: RedisPoolSize,
) -> Result<Self, RedisBackendBuildError> {
) -> Result<Self> {
let manager = deadpool_redis::cluster::Manager::new(
config.seed_urls().to_vec(),
false,
)?;
)
.wrap_err("configuring clustered Redis client")?;
let pool = deadpool_redis::cluster::Pool::builder(manager)
.max_size(pool_size.max())
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
.runtime(deadpool_redis::Runtime::Tokio1)
.build()?;
.build()
.wrap_err("building clustered Redis pool")?;
if config.read_replica_strategy() != ReadReplicaStrategy::Primary {
warn!(
@@ -117,15 +111,15 @@ impl RedisBackend {
);
}
warm_cluster_pool(&pool, pool_size.min()).await?;
warm_cluster_pool(&pool, pool_size.min())
.await
.wrap_err("warming clustered Redis pool")?;
retain_cluster_pool(pool.clone());
Ok(Self::ClusterPooled(pool))
}
async fn cluster_multiplexed(
config: &RedisConfig,
) -> Result<Self, RedisBackendBuildError> {
async fn cluster_multiplexed(config: &RedisConfig) -> Result<Self> {
let mut builder = redis::cluster::ClusterClientBuilder::new(
config.seed_urls().iter().map(String::as_str),
);
@@ -141,22 +135,31 @@ impl RedisBackend {
}
}
let client = builder.build()?;
let connection = client.get_async_connection().await?;
let client = builder
.build()
.wrap_err("building multiplexed Redis client")?;
let connection = client
.get_async_connection()
.await
.wrap_err("connecting multiplexed Redis client")?;
Ok(Self::ClusterMultiplexed(connection))
}
pub(crate) async fn connect(
&self,
) -> Result<RedisConnection, deadpool_redis::PoolError> {
pub(crate) async fn connect(&self) -> Result<RedisConnection> {
let inner = match self {
Self::StandalonePooled(pool) => {
RedisConnectionInner::StandalonePooled(pool.get().await?)
}
Self::ClusterPooled(pool) => {
RedisConnectionInner::ClusterPooled(pool.get().await?)
RedisConnectionInner::StandalonePooled(
pool.get()
.await
.wrap_err("fetching standalone Redis connection")?,
)
}
Self::ClusterPooled(pool) => RedisConnectionInner::ClusterPooled(
pool.get()
.await
.wrap_err("fetching clustered Redis connection")?,
),
Self::ClusterMultiplexed(connection) => {
RedisConnectionInner::ClusterMultiplexed(connection.clone())
}
@@ -165,10 +168,7 @@ impl RedisBackend {
Ok(RedisConnection { inner })
}
pub(crate) fn register_metrics(
&self,
registry: &Registry,
) -> Result<(), prometheus::Error> {
pub(crate) fn register_metrics(&self, registry: &Registry) -> Result<()> {
register_command_pool_metrics(registry, self.clone())
}
}
@@ -282,8 +282,10 @@ impl RoutableConnection for RedisConnection {
async fn warm_standalone_pool(
pool: &deadpool_redis::Pool,
min: usize,
) -> Result<(), deadpool_redis::PoolError> {
let connections = try_join_all((0..min).map(|_| pool.get())).await?;
) -> Result<()> {
let connections = try_join_all((0..min).map(|_| pool.get()))
.await
.wrap_err("fetching initial standalone Redis connections")?;
drop(connections);
Ok(())
}
@@ -291,8 +293,10 @@ async fn warm_standalone_pool(
async fn warm_cluster_pool(
pool: &deadpool_redis::cluster::Pool,
min: usize,
) -> Result<(), deadpool_redis::PoolError> {
let connections = try_join_all((0..min).map(|_| pool.get())).await?;
) -> Result<()> {
let connections = try_join_all((0..min).map(|_| pool.get()))
.await
.wrap_err("fetching initial clustered Redis connections")?;
drop(connections);
Ok(())
}
packages/xredis/src/lib.rs
+41 -51
View File
@@ -6,6 +6,7 @@ use std::hash::Hash;
use std::sync::Arc;
use dashmap::DashMap;
use eyre::{Result, WrapErr};
use prometheus::Registry;
use redis::aio::ConnectionLike;
use redis::{FromRedisValue, ToRedisArgs};
@@ -35,25 +36,7 @@ pub use config::{
RedisConfigError, RedisConnectionType, RedisTopology,
};
use connection::RedisBackend;
pub use connection::RedisBackendBuildError;
pub use key::KeyBuilder;
use thiserror::Error as ThisError;
#[derive(Debug, ThisError)]
pub enum Error {
#[error("error while interacting with Redis: {0}")]
Redis(#[from] redis::RedisError),
#[error("Redis pool error: {0}")]
Pool(#[from] deadpool_redis::PoolError),
#[error("error while serializing a Redis cache value: {0}")]
Serialization(#[from] serde_json::Error),
#[error("Redis blocking timeout must be greater than zero")]
InvalidBlockingTimeout,
#[error(
"timeout waiting on local cache lock ({released}/{total} released)"
)]
LocalCacheTimeout { released: usize, total: usize },
}
#[derive(Clone)]
pub struct RedisPool {
@@ -75,15 +58,19 @@ impl RedisPool {
meta_namespace: impl Into<Arc<str>>,
config: RedisConfig,
cache_settings: CacheSettings,
) -> Result<Self, RedisBackendBuildError> {
) -> Result<Self> {
tracing::info!(
strategy = %config.cache_locking_strategy(),
"configured Redis cache locking"
);
let backend = RedisBackend::new(&config).await?;
let backend = RedisBackend::new(&config)
.await
.wrap_err("creating Redis command backend")?;
let blocking = blocking::RedisBlockingPool::new(&config).await?;
let blocking = blocking::RedisBlockingPool::new(&config)
.await
.wrap_err("creating Redis blocking pool")?;
let key_builder = KeyBuilder::new(meta_namespace, config.topology());
let cache = CacheManager::new(key_builder.clone(), cache_settings);
@@ -102,9 +89,13 @@ impl RedisPool {
}
impl RedisPool {
pub async fn connect(&self) -> Result<RedisConnection, Error> {
pub async fn connect(&self) -> Result<RedisConnection> {
Ok(RedisConnection {
inner: self.backend.connect().await?,
inner: self
.backend
.connect()
.await
.wrap_err("connecting to Redis")?,
key_builder: self.key_builder.clone(),
settings: self.cache.settings().clone(),
})
@@ -113,9 +104,13 @@ impl RedisPool {
pub async fn register_and_set_metrics(
&self,
registry: &Registry,
) -> Result<(), prometheus::Error> {
self.backend.register_metrics(registry)?;
self.blocking.register_metrics(registry)
) -> Result<()> {
self.backend
.register_metrics(registry)
.wrap_err("registering Redis command pool metrics")?;
self.blocking
.register_metrics(registry)
.wrap_err("registering Redis blocking pool metrics")
}
pub async fn get_cached_keys<F, Fut, T, K, E>(
@@ -123,11 +118,11 @@ impl RedisPool {
namespace: &str,
keys: &[K],
closure: F,
) -> Result<Vec<T>, E>
) -> Result<Vec<T>>
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
@@ -148,11 +143,11 @@ impl RedisPool {
namespace: &str,
keys: &[K],
closure: F,
) -> Result<std::collections::HashMap<K, T>, E>
) -> Result<std::collections::HashMap<K, T>>
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
K: Display
+ Hash
@@ -175,11 +170,11 @@ impl RedisPool {
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<Vec<T>, E>
) -> Result<Vec<T>>
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
@@ -210,11 +205,11 @@ impl RedisPool {
case_sensitive: bool,
keys: &[I],
closure: F,
) -> Result<std::collections::HashMap<K, T>, E>
) -> Result<std::collections::HashMap<K, T>>
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
E: From<Error>,
E: std::error::Error + Send + Sync + 'static,
T: Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
@@ -242,9 +237,7 @@ impl RedisPool {
impl ConnectionProvider for RedisPool {
type Connection = RedisConnection;
fn connect(
&self,
) -> impl Future<Output = Result<Self::Connection, Error>> + Send {
fn connect(&self) -> impl Future<Output = Result<Self::Connection>> + Send {
RedisPool::connect(self)
}
}
@@ -259,7 +252,7 @@ impl RedisConnection {
key: &str,
data: D,
expiry: Option<i64>,
) -> Result<(), Error>
) -> Result<()>
where
D: ToRedisArgs + Send + Sync + Debug,
{
@@ -277,7 +270,7 @@ impl RedisConnection {
key: &str,
data: D,
expiry: Option<i64>,
) -> Result<(), Error>
) -> Result<()>
where
D: Serialize,
{
@@ -291,31 +284,28 @@ impl RedisConnection {
.await
}
pub async fn get(&mut self, key: &str) -> Result<Option<String>, Error> {
pub async fn get(&mut self, key: &str) -> Result<Option<String>> {
commands::get(&mut self.inner, key).await
}
pub async fn get_many(
&mut self,
keys: &[String],
) -> Result<Vec<Option<Vec<u8>>>, Error> {
) -> Result<Vec<Option<Vec<u8>>>> {
commands::get_many(&mut self.inner, keys).await
}
pub async fn get_many_typed<R>(
&mut self,
keys: &[String],
) -> Result<Vec<Option<R>>, Error>
) -> Result<Vec<Option<R>>>
where
R: FromRedisValue,
{
commands::get_many_as(&mut self.inner, keys).await
}
pub async fn get_deserialized<R>(
&mut self,
key: &str,
) -> Result<Option<R>, Error>
pub async fn get_deserialized<R>(&mut self, key: &str) -> Result<Option<R>>
where
R: for<'a> serde::Deserialize<'a>,
{
@@ -325,7 +315,7 @@ impl RedisConnection {
pub async fn get_many_deserialized<R>(
&mut self,
keys: &[String],
) -> Result<Vec<Option<R>>, Error>
) -> Result<Vec<Option<R>>>
where
R: for<'a> serde::Deserialize<'a>,
{
@@ -333,22 +323,22 @@ impl RedisConnection {
.await
}
pub async fn delete(&mut self, key: &str) -> Result<(), Error> {
pub async fn delete(&mut self, key: &str) -> Result<()> {
commands::delete(&mut self.inner, key).await
}
pub async fn delete_many(&mut self, keys: &[String]) -> Result<(), Error> {
pub async fn delete_many(&mut self, keys: &[String]) -> Result<()> {
commands::delete_many(&mut self.inner, keys).await
}
pub async fn lpush<D>(&mut self, key: &str, value: D) -> Result<(), Error>
pub async fn lpush<D>(&mut self, key: &str, value: D) -> Result<()>
where
D: ToRedisArgs + Send + Sync + Debug,
{
commands::lpush(&mut self.inner, key, value).await
}
pub async fn incr(&mut self, key: &str) -> Result<Option<u64>, Error> {
pub async fn incr(&mut self, key: &str) -> Result<Option<u64>> {
commands::incr(&mut self.inner, key).await
}
}
packages/xredis/src/metrics.rs
+27 -13
View File
@@ -1,5 +1,6 @@
use std::time::Duration;
use eyre::{Result, WrapErr};
use prometheus::{IntGauge, Registry};
const METRICS_UPDATE_INTERVAL: Duration = Duration::from_secs(5);
@@ -72,7 +73,7 @@ impl RedisPoolMetrics {
fn register(
registry: &Registry,
kind: RedisPoolMetricsKind,
) -> Result<Self, prometheus::Error> {
) -> Result<Self> {
let prefix = kind.metric_prefix();
let description = kind.description();
let max_size = IntGauge::new(
@@ -80,28 +81,40 @@ impl RedisPoolMetrics {
format!(
"Maximum logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
),
)?;
)
.wrap_err("creating Redis pool maximum size metric")?;
let size = IntGauge::new(
format!("{prefix}_size"),
format!(
"Current logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
),
)?;
)
.wrap_err("creating Redis pool size metric")?;
let available = IntGauge::new(
format!("{prefix}_available"),
format!("Available logical connections in the {description}"),
)?;
)
.wrap_err("creating Redis pool availability metric")?;
let waiting = IntGauge::new(
format!("{prefix}_waiting"),
format!(
"Number of futures waiting for a logical connection from the {description}"
),
)?;
)
.wrap_err("creating Redis pool waiters metric")?;
registry.register(Box::new(max_size.clone()))?;
registry.register(Box::new(size.clone()))?;
registry.register(Box::new(available.clone()))?;
registry.register(Box::new(waiting.clone()))?;
registry
.register(Box::new(max_size.clone()))
.wrap_err("registering Redis pool maximum size metric")?;
registry
.register(Box::new(size.clone()))
.wrap_err("registering Redis pool size metric")?;
registry
.register(Box::new(available.clone()))
.wrap_err("registering Redis pool availability metric")?;
registry
.register(Box::new(waiting.clone()))
.wrap_err("registering Redis pool waiters metric")?;
Ok(Self {
max_size,
@@ -122,7 +135,7 @@ impl RedisPoolMetrics {
pub(super) fn register_command_pool_metrics<P>(
registry: &Registry,
provider: P,
) -> Result<(), prometheus::Error>
) -> Result<()>
where
P: LogicalPoolStatusProvider,
{
@@ -132,7 +145,7 @@ where
pub(super) fn register_blocking_pool_metrics<P>(
registry: &Registry,
provider: P,
) -> Result<(), prometheus::Error>
) -> Result<()>
where
P: LogicalPoolStatusProvider,
{
@@ -143,11 +156,12 @@ fn register_pool_metrics<P>(
registry: &Registry,
kind: RedisPoolMetricsKind,
provider: P,
) -> Result<(), prometheus::Error>
) -> Result<()>
where
P: LogicalPoolStatusProvider,
{
let metrics = RedisPoolMetrics::register(registry, kind)?;
let metrics = RedisPoolMetrics::register(registry, kind)
.wrap_err("registering Redis pool metrics")?;
metrics.set(provider.logical_pool_status());
tokio::spawn(async move {
packages/xredis/src/pubsub.rs
+19 -12
View File
@@ -1,11 +1,12 @@
use std::time::Duration;
use eyre::{Result, WrapErr};
use futures::StreamExt;
use redis::ToRedisArgs;
use tokio::sync::mpsc;
use tracing::{info, warn};
use super::{Error, RedisPool};
use super::RedisPool;
const PUBSUB_BUFFER_SIZE: usize = 1024;
const INITIAL_RECONNECT_BACKOFF: Duration = Duration::from_millis(250);
@@ -35,21 +36,20 @@ impl RedisPool {
receiver
}
pub async fn publish<M>(
&self,
channel: &str,
message: M,
) -> Result<(), Error>
pub async fn publish<M>(&self, channel: &str, message: M) -> Result<()>
where
M: ToRedisArgs + Send + Sync,
{
let mut connection = self.connect().await?;
let mut connection = self
.connect()
.await
.wrap_err("connecting to Redis for publishing")?;
let _: usize = redis::cmd("PUBLISH")
.arg(channel)
.arg(message)
.query_async(&mut connection)
.await
.map_err(Error::from)?;
.wrap_err("publishing to Redis channel")?;
Ok(())
}
}
@@ -111,10 +111,17 @@ async fn forward_from_seed(
seed_url: &str,
channel: &'static str,
sender: &mpsc::Sender<Vec<u8>>,
) -> redis::RedisResult<SubscriptionOutcome> {
let client = redis::Client::open(seed_url)?;
let mut pubsub = client.get_async_pubsub().await?;
pubsub.subscribe(channel).await?;
) -> Result<SubscriptionOutcome> {
let client = redis::Client::open(seed_url)
.wrap_err("configuring Redis Pub/Sub client")?;
let mut pubsub = client
.get_async_pubsub()
.await
.wrap_err("connecting to Redis Pub/Sub")?;
pubsub
.subscribe(channel)
.await
.wrap_err("subscribing to Redis channel")?;
info!(channel, "Established Redis Pub/Sub subscription");
let mut stream = pubsub.into_on_message();