Compare commits

...
50 changed files with 1802 additions and 309 deletions
Generated
+55
View File
@@ -1795,6 +1795,25 @@ dependencies = [
"libbz2-rs-sys",
]
[[package]]
name = "cached-projection"
version = "0.0.0"
dependencies = [
"cached-projection-derive",
"postcard",
"serde",
"serde_json",
]
[[package]]
name = "cached-projection-derive"
version = "0.0.0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
]
[[package]]
name = "cairo-rs"
version = "0.18.5"
@@ -2133,6 +2152,15 @@ dependencies = [
"cc",
]
[[package]]
name = "cobs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
dependencies = [
"thiserror 2.0.17",
]
[[package]]
name = "color-eyre"
version = "0.6.5"
@@ -3259,6 +3287,18 @@ version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7"
[[package]]
name = "embedded-io"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
[[package]]
name = "embedded-io"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
[[package]]
name = "encode_unicode"
version = "1.0.0"
@@ -5412,6 +5452,7 @@ dependencies = [
"base64 0.22.1",
"bitflags 2.9.4",
"bytes",
"cached-projection",
"censor",
"chrono",
"clap 4.5.48",
@@ -5440,6 +5481,7 @@ dependencies = [
"json-patch 4.1.0",
"labrinth",
"lettre",
"lz4_flex",
"meilisearch-sdk",
"modrinth-content-management",
"modrinth-util",
@@ -5447,6 +5489,7 @@ dependencies = [
"murmur2",
"paste",
"path-util",
"postcard",
"prometheus",
"quick-xml 0.38.3",
"rand 0.8.5",
@@ -7492,6 +7535,18 @@ version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
[[package]]
name = "postcard"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
dependencies = [
"cobs",
"embedded-io 0.4.0",
"embedded-io 0.6.1",
"serde",
]
[[package]]
name = "potential_utf"
version = "0.1.3"
+10
View File
@@ -7,6 +7,8 @@ members = [
"apps/labrinth",
"packages/app-lib",
"packages/ariadne",
"packages/cached-projection",
"packages/cached-projection-derive",
"packages/daedalus",
"packages/labrinth-derive",
"packages/modrinth-content-management",
@@ -54,6 +56,7 @@ bitflags = "2.9.4"
bon = "3.9.3"
bytemuck = "1.24.0"
bytes = "1.10.1"
cached-projection = { path = "packages/cached-projection" }
censor = "0.3.0"
chardetng = "0.1.17"
chrono = "0.4.42"
@@ -118,6 +121,12 @@ lettre = { version = "0.11.19", default-features = false, features = [
"tokio1",
"tokio1-rustls",
] }
lz4_flex = { version = "0.11.5", default-features = false, features = [
"checked-decode",
"safe-decode",
"safe-encode",
"std",
] }
maxminddb = "0.26.0"
meilisearch-sdk = { version = "0.30.0", default-features = false }
modrinth-content-management = { path = "packages/modrinth-content-management" }
@@ -135,6 +144,7 @@ paste = "1.0.15"
path-util = { path = "packages/path-util" }
phf = { version = "0.13.1", features = ["macros"] }
png = "0.18.0"
postcard = { version = "1.1.3", default-features = false, features = ["alloc"] }
proc-macro2 = { version = "1.0" }
prometheus = "0.14.0"
quartz_nbt = "0.2.9"
+3
View File
@@ -33,6 +33,7 @@ aws-sdk-s3 = { workspace = true }
base64 = { workspace = true }
bitflags = { workspace = true }
bytes = { workspace = true }
cached-projection = { workspace = true }
censor = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
clap = { workspace = true, features = ["derive"] }
@@ -73,6 +74,7 @@ image = { workspace = true, features = [
itertools = { workspace = true }
json-patch = { workspace = true }
lettre = { workspace = true }
lz4_flex = { workspace = true }
meilisearch-sdk = { workspace = true, features = ["reqwest"] }
modrinth-content-management = { workspace = true }
modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] }
@@ -80,6 +82,7 @@ muralpay = { workspace = true, features = ["client", "mock", "utoipa"] }
murmur2 = { workspace = true }
paste = { workspace = true }
path-util = { workspace = true }
postcard = { workspace = true }
prometheus = { workspace = true }
quick-xml = { workspace = true }
rand = { workspace = true }
@@ -1,3 +1,4 @@
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use futures::{StreamExt, TryStreamExt};
use sqlx::types::Json;
@@ -11,10 +12,10 @@ use crate::{
};
use serde::{Deserialize, Serialize};
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events";
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v1";
const ANALYTICS_EVENTS_ALL_KEY: &str = "all";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)]
pub struct DBAnalyticsEvent {
pub id: DBAnalyticsEventId,
pub meta: AnalyticsEventMeta,
@@ -88,7 +89,7 @@ impl DBAnalyticsEvent {
let mut redis = redis.connect().await?;
if let Some(events) = redis
.get_deserialized_from_json(
.get_deserialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
)
@@ -119,7 +120,7 @@ impl DBAnalyticsEvent {
.await?;
redis
.set_serialized_to_json(
.set_serialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
&events,
+13 -28
View File
@@ -4,17 +4,18 @@ use crate::database::redis::RedisPool;
use super::DatabaseError;
use super::ids::*;
use cached_projection::CachedProjection;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const TAGS_NAMESPACE: &str = "tags";
const TAGS_NAMESPACE: &str = "tags:v1";
pub struct ProjectType {
pub id: ProjectTypeId,
pub name: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
pub struct Category {
pub id: CategoryId,
pub category: String,
@@ -28,7 +29,7 @@ pub struct ReportType {
pub report_type: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
pub struct LinkPlatform {
pub id: LinkPlatformId,
pub name: String,
@@ -96,9 +97,8 @@ impl Category {
{
let mut redis = redis.connect().await?;
let res: Option<Vec<Category>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "category")
.await?;
let res: Option<Vec<Category>> =
redis.get_deserialized(TAGS_NAMESPACE, "category").await?;
if let Some(res) = res {
return Ok(res);
@@ -127,7 +127,7 @@ impl Category {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "category", &result, None)
.set_serialized(TAGS_NAMESPACE, "category", &result, None)
.await?;
Ok(result)
@@ -166,7 +166,7 @@ impl LinkPlatform {
let mut redis = redis.connect().await?;
let res: Option<Vec<LinkPlatform>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "link_platform")
.get_deserialized(TAGS_NAMESPACE, "link_platform")
.await?;
if let Some(res) = res {
@@ -191,12 +191,7 @@ impl LinkPlatform {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"link_platform",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "link_platform", &result, None)
.await?;
Ok(result)
@@ -235,7 +230,7 @@ impl ReportType {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "report_type")
.get_deserialized(TAGS_NAMESPACE, "report_type")
.await?;
if let Some(res) = res {
@@ -256,12 +251,7 @@ impl ReportType {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"report_type",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "report_type", &result, None)
.await?;
Ok(result)
@@ -300,7 +290,7 @@ impl ProjectType {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "project_type")
.get_deserialized(TAGS_NAMESPACE, "project_type")
.await?;
if let Some(res) = res {
@@ -321,12 +311,7 @@ impl ProjectType {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"project_type",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "project_type", &result, None)
.await?;
Ok(result)
@@ -3,12 +3,13 @@ use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::database::{PgTransaction, models};
use crate::models::collections::CollectionStatus;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const COLLECTIONS_NAMESPACE: &str = "collections";
const COLLECTIONS_NAMESPACE: &str = "collections:v1";
#[derive(Clone)]
pub struct CollectionBuilder {
@@ -43,7 +44,7 @@ impl CollectionBuilder {
Ok(self.collection_id)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)]
pub struct DBCollection {
pub id: DBCollectionId,
pub user_id: DBUserId,
@@ -4,6 +4,7 @@ use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use crate::{auth::AuthProvider, routes::internal::flows::TempUser};
use cached_projection::CachedProjection;
use chrono::Duration;
use rand::Rng;
use rand::distributions::Alphanumeric;
@@ -13,10 +14,9 @@ use serde::{Deserialize, Serialize};
use url::Url;
use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration};
const FLOWS_NAMESPACE: &str = "flows";
const FLOWS_NAMESPACE: &str = "flows:v2";
#[derive(Deserialize, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
#[derive(Deserialize, Serialize, CachedProjection)]
pub enum DBFlow {
OAuth {
user_id: Option<DBUserId>,
@@ -61,9 +61,11 @@ pub enum DBFlow {
},
RegisterPasskey {
user_id: DBUserId,
#[cached_projection(wrap)]
state: PasskeyRegistration,
},
AuthenticatePasskey {
#[cached_projection(wrap)]
state: DiscoverableAuthentication,
},
}
@@ -78,10 +80,10 @@ impl DBFlow {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
FLOWS_NAMESPACE,
&state,
&self,
self,
Some(expires.num_seconds()),
)
.await?;
@@ -109,7 +111,7 @@ impl DBFlow {
) -> Result<Option<DBFlow>, DatabaseError> {
let mut redis = redis.connect().await?;
redis.get_deserialized_from_json(FLOWS_NAMESPACE, id).await
redis.get_deserialized(FLOWS_NAMESPACE, id).await
}
/// Gets the flow and removes it from the cache, but only removes if the flow was present and the predicate returned true
+3 -1
View File
@@ -10,6 +10,7 @@ use crate::models::ids::{
};
use ariadne::ids::base62_impl::to_base62;
use ariadne::ids::{UserId, random_base62_rng, random_base62_rng_range};
use cached_projection::CachedProjection;
use censor::Censor;
use paste::paste;
use rand::SeedableRng;
@@ -95,7 +96,7 @@ macro_rules! generate_bulk_ids {
macro_rules! impl_db_id_interface {
($id_struct:ident, $db_id_struct:ident, $(, generator: $generator_function:ident @ $db_table:expr, $(bulk_generator: $bulk_generator_function:ident,)?)?) => {
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash, utoipa::ToSchema)]
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash, utoipa::ToSchema, CachedProjection)]
#[sqlx(transparent)]
pub struct $db_id_struct(pub i64);
@@ -154,6 +155,7 @@ macro_rules! id_type {
PartialEq,
Hash,
utoipa::ToSchema,
CachedProjection,
)]
#[sqlx(transparent)]
pub struct $name(pub $type);
@@ -2,13 +2,14 @@ use super::ids::*;
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use crate::{database::models::DatabaseError, models::images::ImageContext};
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
const IMAGES_NAMESPACE: &str = "images";
const IMAGES_NAMESPACE: &str = "images:v1";
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)]
pub struct DBImage {
pub id: DBImageId,
pub url: String,
@@ -5,6 +5,7 @@ use super::DatabaseError;
use super::ids::*;
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use cached_projection::CachedProjection;
use chrono::DateTime;
use chrono::Utc;
use dashmap::DashMap;
@@ -12,15 +13,16 @@ use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
const GAMES_LIST_NAMESPACE: &str = "games";
const LOADER_ID: &str = "loader_id";
const LOADERS_LIST_NAMESPACE: &str = "loaders";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values";
const GAMES_LIST_NAMESPACE: &str = "games:v1";
const LOADER_ID: &str = "loader_id:v1";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v2";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v1";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v1";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v1";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str =
"loader_field_enum_values:v2";
#[derive(Clone, Serialize, Deserialize, Debug)]
#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)]
pub struct Game {
pub id: GameId,
pub slug: String,
@@ -54,7 +56,7 @@ impl Game {
{
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis
.get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games")
.get_deserialized(GAMES_LIST_NAMESPACE, "games")
.await?;
if let Some(cached_games) = cached_games {
return Ok(cached_games);
@@ -80,25 +82,21 @@ impl Game {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
GAMES_LIST_NAMESPACE,
"games",
&result,
None,
)
.set_serialized(GAMES_LIST_NAMESPACE, "games", &result, None)
.await?;
Ok(result)
}
}
#[derive(Serialize, Deserialize, Clone)]
#[derive(Serialize, Deserialize, Clone, CachedProjection)]
pub struct Loader {
pub id: LoaderId,
pub loader: String,
pub icon: String,
pub supported_project_types: Vec<String>,
pub supported_games: Vec<String>, // slugs
#[cached_projection(wrap)]
pub metadata: serde_json::Value,
}
@@ -114,7 +112,7 @@ impl Loader {
{
let mut redis = redis.connect().await?;
let cached_id: Option<i32> =
redis.get_deserialized_from_json(LOADER_ID, name).await?;
redis.get_deserialized(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
}
@@ -134,7 +132,7 @@ impl Loader {
if let Some(result) = result {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(LOADER_ID, name, &result.0, None)
.set_serialized(LOADER_ID, name, &result.0, None)
.await?;
}
@@ -151,7 +149,7 @@ impl Loader {
{
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.get_deserialized(LOADERS_LIST_NAMESPACE, "all")
.await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
@@ -193,19 +191,14 @@ impl Loader {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADERS_LIST_NAMESPACE,
"all",
&result,
None,
)
.set_serialized(LOADERS_LIST_NAMESPACE, "all", &result, None)
.await?;
Ok(result)
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)]
pub struct LoaderField {
pub id: LoaderFieldId,
pub field: String,
@@ -274,7 +267,7 @@ impl LoaderFieldType {
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)]
pub struct LoaderFieldEnum {
pub id: LoaderFieldEnumId,
pub enum_name: String,
@@ -282,14 +275,18 @@ pub struct LoaderFieldEnum {
pub hidable: bool,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
#[derive(
Clone, Serialize, Deserialize, Debug, PartialEq, Eq, CachedProjection,
)]
pub struct LoaderFieldEnumValue {
pub id: LoaderFieldEnumValueId,
pub enum_id: LoaderFieldEnumId,
pub value: String,
pub ordering: Option<i32>,
pub created: DateTime<Utc>,
#[serde(flatten)]
#[cached_projection(wrap)]
pub metadata: serde_json::Value,
}
@@ -303,22 +300,33 @@ impl std::hash::Hash for LoaderFieldEnumValue {
}
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
#[derive(
Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash, CachedProjection,
)]
pub struct VersionField {
pub version_id: DBVersionId,
pub field_id: LoaderFieldId,
pub field_name: String,
#[cached_projection(nested)]
pub value: VersionFieldValue,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
#[derive(
Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash, CachedProjection,
)]
pub enum VersionFieldValue {
Integer(i32),
Text(String),
Enum(LoaderFieldEnumId, LoaderFieldEnumValue),
Enum(
LoaderFieldEnumId,
#[cached_projection(nested)] LoaderFieldEnumValue,
),
Boolean(bool),
ArrayInteger(Vec<i32>),
ArrayText(Vec<String>),
ArrayEnum(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>),
ArrayEnum(
LoaderFieldEnumId,
#[cached_projection(nested)] Vec<LoaderFieldEnumValue>,
),
ArrayBoolean(Vec<bool>),
}
@@ -470,10 +478,9 @@ impl LoaderField {
{
let mut redis = redis.connect().await?;
let cached_fields: Option<Vec<LoaderField>> =
redis.get(LOADER_FIELDS_NAMESPACE_ALL, "").await?.and_then(
|x| serde_json::from_str::<Vec<LoaderField>>(&x).ok(),
);
let cached_fields: Option<Vec<LoaderField>> = redis
.get_deserialized(LOADER_FIELDS_NAMESPACE_ALL, "")
.await?;
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
@@ -506,12 +513,7 @@ impl LoaderField {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADER_FIELDS_NAMESPACE_ALL,
"",
&result,
None,
)
.set_serialized(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None)
.await?;
Ok(result)
@@ -530,10 +532,7 @@ impl LoaderFieldEnum {
let mut redis = redis.connect().await?;
let cached_enum = redis
.get_deserialized_from_json(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
)
.get_deserialized(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name)
.await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
@@ -561,7 +560,7 @@ impl LoaderFieldEnum {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
&result,
+2
View File
@@ -76,6 +76,8 @@ pub enum DatabaseError {
RedisPool(#[from] deadpool_redis::PoolError),
#[error("Error while serializing with the cache: {0}")]
SerdeCacheError(#[from] serde_json::Error),
#[error("error while encoding or decoding the cache: {0}")]
PostcardCacheError(#[from] postcard::Error),
#[error("Schema error: {0}")]
SchemaError(String),
#[error(
@@ -1,5 +1,6 @@
use std::collections::HashMap;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
@@ -7,11 +8,11 @@ use crate::database::redis::RedisPool;
use super::{DBOrganizationId, DBUserId, DatabaseError};
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users";
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v1";
const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str =
"moderation_notes_organizations";
"moderation_notes_organizations:v1";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)]
pub struct DBModerationNote {
pub user_id: Option<DBUserId>,
pub organization_id: Option<DBOrganizationId>,
@@ -40,7 +41,7 @@ impl DBModerationNote {
let cached = {
let mut redis = redis.connect().await?;
redis
.get_many_deserialized_from_json::<Self>(
.get_many_deserialized::<Self>(
MODERATION_NOTES_USERS_NAMESPACE,
&ids,
)
@@ -87,7 +88,7 @@ impl DBModerationNote {
if let Some(user_id) = note.user_id {
redis
.set_serialized_to_json(
.set_serialized(
MODERATION_NOTES_USERS_NAMESPACE,
user_id.0,
&note,
@@ -130,7 +131,7 @@ impl DBModerationNote {
let cached = {
let mut redis = redis.connect().await?;
redis
.get_many_deserialized_from_json::<Self>(
.get_many_deserialized::<Self>(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
&ids,
)
@@ -177,7 +178,7 @@ impl DBModerationNote {
if let Some(organization_id) = note.organization_id {
redis
.set_serialized_to_json(
.set_serialized(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
organization_id.0,
&note,
@@ -5,20 +5,22 @@ use crate::models::notifications::{
NotificationBody, NotificationChannel, NotificationDeliveryStatus,
NotificationType,
};
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications";
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v2";
pub struct NotificationBuilder {
pub body: NotificationBody,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
pub struct DBNotification {
pub id: DBNotificationId,
pub user_id: DBUserId,
#[cached_projection(nested)]
pub body: NotificationBody,
pub read: bool,
pub created: DateTime<Utc>,
@@ -435,7 +437,7 @@ impl DBNotification {
let mut redis = redis.connect().await?;
let cached_notifications: Option<Vec<DBNotification>> = redis
.get_deserialized_from_json(
.get_deserialized(
USER_NOTIFICATIONS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -493,7 +495,7 @@ impl DBNotification {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
USER_NOTIFICATIONS_NAMESPACE,
user_id.0,
&db_notifications,
@@ -2,17 +2,19 @@ use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::v3::notifications::{NotificationChannel, NotificationType};
use crate::routes::ApiError;
use cached_projection::CachedProjection;
use serde::{Deserialize, Serialize};
const TEMPLATES_NAMESPACE: &str = "notifications_templates";
const TEMPLATES_HTML_DATA_NAMESPACE: &str = "notifications_templates_html_data";
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v1";
const TEMPLATES_HTML_DATA_NAMESPACE: &str =
"notifications_templates_html_data:v1";
const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str =
"notifications_templates_dynamic_html";
"notifications_templates_dynamic_html:v1";
const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes
const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)]
pub struct NotificationTemplate {
pub id: i64,
pub channel: NotificationChannel,
@@ -56,10 +58,7 @@ impl NotificationTemplate {
let mut redis = redis.connect().await?;
let maybe_cached_templates = redis
.get_deserialized_from_json(
TEMPLATES_NAMESPACE,
channel.as_str(),
)
.get_deserialized(TEMPLATES_NAMESPACE, channel.as_str())
.await?;
if let Some(cached) = maybe_cached_templates {
@@ -82,7 +81,7 @@ impl NotificationTemplate {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
TEMPLATES_NAMESPACE,
channel.as_str(),
&templates,
@@ -99,7 +98,7 @@ impl NotificationTemplate {
) -> Result<Option<String>, DatabaseError> {
let mut redis = redis.connect().await?;
redis
.get_deserialized_from_json(
.get_deserialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
)
@@ -113,7 +112,7 @@ impl NotificationTemplate {
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
&data,
@@ -131,17 +130,14 @@ pub async fn get_or_set_cached_dynamic_html<F>(
where
F: Future<Output = Result<String, ApiError>>,
{
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)]
struct HtmlBody {
html: String,
}
let mut redis_conn = redis.connect().await?;
if let Some(body) = redis_conn
.get_deserialized_from_json::<HtmlBody>(
TEMPLATES_DYNAMIC_HTML_NAMESPACE,
key,
)
.get_deserialized::<HtmlBody>(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key)
.await?
{
return Ok(body.html);
@@ -153,7 +149,7 @@ where
let mut redis_conn = redis.connect().await?;
redis_conn
.set_serialized_to_json(
.set_serialized(
TEMPLATES_DYNAMIC_HTML_NAMESPACE,
key,
&cached,
@@ -1,11 +1,12 @@
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::v3::notifications::NotificationType;
use cached_projection::CachedProjection;
use serde::{Deserialize, Serialize};
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types";
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v1";
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
pub struct NotificationTypeItem {
pub name: NotificationType,
pub delivery_priority: i32,
@@ -43,7 +44,7 @@ impl NotificationTypeItem {
let mut redis = redis.connect().await?;
let cached_types = redis
.get_deserialized_from_json(NOTIFICATION_TYPES_NAMESPACE, "all")
.get_deserialized(NOTIFICATION_TYPES_NAMESPACE, "all")
.await?;
if let Some(types) = cached_types {
@@ -63,12 +64,7 @@ impl NotificationTypeItem {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
NOTIFICATION_TYPES_NAMESPACE,
"all",
&types,
None,
)
.set_serialized(NOTIFICATION_TYPES_NAMESPACE, "all", &types, None)
.await?;
Ok(types)
@@ -1,6 +1,7 @@
use crate::database::PgTransaction;
use crate::database::redis::RedisPool;
use ariadne::ids::base62_impl::parse_base62;
use cached_projection::CachedProjection;
use dashmap::DashMap;
use futures::TryStreamExt;
use std::fmt::{Debug, Display};
@@ -9,10 +10,10 @@ use std::hash::Hash;
use super::{DBTeamMember, ids::*};
use serde::{Deserialize, Serialize};
const ORGANIZATIONS_NAMESPACE: &str = "organizations";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles";
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v1";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)]
/// An organization of users who together control one or more projects and organizations.
pub struct DBOrganization {
/// The id of the organization
+7 -11
View File
@@ -4,6 +4,7 @@ use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::pats::Scopes;
use ariadne::ids::base62_impl::parse_base62;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use futures::TryStreamExt;
@@ -11,11 +12,11 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const PATS_NAMESPACE: &str = "pats";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens";
const PATS_USERS_NAMESPACE: &str = "pats_users";
const PATS_NAMESPACE: &str = "pats:v1";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v1";
const PATS_USERS_NAMESPACE: &str = "pats_users:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)]
pub struct DBPersonalAccessToken {
pub id: DBPatId,
pub name: String,
@@ -161,7 +162,7 @@ impl DBPersonalAccessToken {
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
.get_deserialized::<Vec<i64>>(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -189,12 +190,7 @@ impl DBPersonalAccessToken {
let mut redis = redis.connect().await?;
redis
.set(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
&serde_json::to_string(&db_pats)?,
None,
)
.set_serialized(PATS_USERS_NAMESPACE, user_id.0, &db_pats, None)
.await?;
Ok(db_pats)
}
@@ -3,13 +3,14 @@ use crate::database::models::{
};
use crate::database::redis::RedisPool;
use crate::models::billing::{Price, ProductMetadata};
use cached_projection::CachedProjection;
use dashmap::DashMap;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::convert::TryFrom;
use std::convert::TryInto;
const PRODUCTS_NAMESPACE: &str = "products";
const PRODUCTS_NAMESPACE: &str = "products:v2";
pub struct DBProduct {
pub id: DBProductId,
@@ -131,13 +132,15 @@ impl DBProduct {
}
}
#[derive(Deserialize, Serialize)]
#[derive(Deserialize, Serialize, CachedProjection)]
pub struct QueryProductWithPrices {
pub id: DBProductId,
#[cached_projection(nested)]
pub metadata: ProductMetadata,
pub unitary: bool,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub name: Option<String>,
#[cached_projection(nested)]
pub prices: Vec<DBProductPrice>,
}
@@ -153,9 +156,8 @@ impl QueryProductWithPrices {
{
let mut redis = redis.connect().await?;
let res: Option<Vec<QueryProductWithPrices>> = redis
.get_deserialized_from_json(PRODUCTS_NAMESPACE, "all")
.await?;
let res: Option<Vec<QueryProductWithPrices>> =
redis.get_deserialized(PRODUCTS_NAMESPACE, "all").await?;
if let Some(res) = res {
return Ok(res);
@@ -196,7 +198,7 @@ impl QueryProductWithPrices {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(PRODUCTS_NAMESPACE, "all", &products, None)
.set_serialized(PRODUCTS_NAMESPACE, "all", &products, None)
.await?;
Ok(products)
@@ -243,10 +245,11 @@ impl QueryProductWithPrices {
}
}
#[derive(Deserialize, Serialize)]
#[derive(Deserialize, Serialize, CachedProjection)]
pub struct DBProductPrice {
pub id: DBProductPriceId,
pub product_id: DBProductId,
#[cached_projection(nested)]
pub prices: Price,
pub currency_code: String,
}
@@ -15,6 +15,7 @@ use crate::models::projects::{
use crate::routes::ApiError;
use crate::util::error::Context;
use ariadne::ids::base62_impl::parse_base62;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::{DashMap, DashSet};
use futures::TryStreamExt;
@@ -23,9 +24,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
pub const PROJECTS_NAMESPACE: &str = "projects";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies";
pub const PROJECTS_NAMESPACE: &str = "projects:v2";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v1";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v1";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LinkUrl {
@@ -276,7 +277,7 @@ impl ProjectBuilder {
Ok(self.project_id)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)]
pub struct DBProject {
pub id: DBProjectId,
pub team_id: DBTeamId,
@@ -304,6 +305,7 @@ pub struct DBProject {
pub monetization_status: MonetizationStatus,
pub side_types_migration_review_status: SideTypesMigrationReviewStatus,
pub loaders: Vec<String>,
#[cached_projection(nested)]
pub components: exp::ProjectSerial,
}
@@ -976,7 +978,7 @@ impl DBProject {
let mut redis = redis.connect().await?;
let dependencies = redis
.get_deserialized_from_json::<Dependencies>(
.get_deserialized::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
@@ -1014,7 +1016,7 @@ impl DBProject {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
PROJECTS_DEPENDENCIES_NAMESPACE,
id.0,
&dependencies,
@@ -1050,8 +1052,9 @@ impl DBProject {
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)]
pub struct ProjectQueryResult {
#[cached_projection(nested)]
pub inner: DBProject,
pub categories: Vec<String>,
pub additional_categories: Vec<String>,
@@ -1061,7 +1064,9 @@ pub struct ProjectQueryResult {
pub urls: Vec<LinkUrl>,
pub gallery_items: Vec<DBGalleryItem>,
pub thread_id: DBThreadId,
#[cached_projection(nested)]
pub aggregate_version_fields: Vec<VersionField>,
#[serde(flatten)]
#[cached_projection(nested)]
pub components: exp::ProjectQuery,
}
@@ -3,6 +3,7 @@ use crate::database::PgTransaction;
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use ariadne::ids::base62_impl::parse_base62;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use futures_util::TryStreamExt;
@@ -10,9 +11,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const SESSIONS_NAMESPACE: &str = "sessions";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users";
const SESSIONS_NAMESPACE: &str = "sessions:v1";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v1";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v1";
pub struct SessionBuilder {
pub session: String,
@@ -74,7 +75,7 @@ impl SessionBuilder {
}
}
#[derive(Deserialize, Serialize)]
#[derive(Deserialize, Serialize, CachedProjection)]
pub struct DBSession {
pub id: DBSessionId,
pub session: String,
@@ -226,7 +227,7 @@ impl DBSession {
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
.get_deserialized::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -255,7 +256,7 @@ impl DBSession {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
SESSIONS_USERS_NAMESPACE,
user_id.0,
&db_sessions,
@@ -3,13 +3,14 @@ use crate::{
database::{PgTransaction, redis::RedisPool},
models::teams::{OrganizationPermissions, ProjectPermissions},
};
use cached_projection::CachedProjection;
use dashmap::DashMap;
use futures::TryStreamExt;
use itertools::Itertools;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
const TEAMS_NAMESPACE: &str = "teams";
const TEAMS_NAMESPACE: &str = "teams:v2";
pub struct TeamBuilder {
pub members: Vec<TeamMemberBuilder>,
@@ -164,7 +165,7 @@ impl DBTeam {
}
/// A member of a team
#[derive(Deserialize, Serialize, Clone, Debug)]
#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)]
pub struct DBTeamMember {
pub id: DBTeamMemberId,
pub team_id: DBTeamId,
@@ -183,6 +184,7 @@ pub struct DBTeamMember {
pub organization_permissions: Option<OrganizationPermissions>,
pub accepted: bool,
#[cached_projection(wrap)]
pub payouts_split: Decimal,
pub ordering: i64,
}
@@ -9,6 +9,7 @@ use crate::models::billing::ChargeStatus;
use crate::models::users::Badges;
use crate::util::error::Context;
use ariadne::ids::base62_impl::{parse_base62, to_base62};
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use rust_decimal::Decimal;
@@ -16,11 +17,11 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const USERS_NAMESPACE: &str = "users";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects";
const USERS_NAMESPACE: &str = "users:v1";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v1";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)]
pub struct DBUser {
pub id: DBUserId,
@@ -391,7 +392,7 @@ impl DBUser {
let mut redis = redis.connect().await?;
let cached_projects = redis
.get_deserialized_from_json::<Vec<DBProjectId>>(
.get_deserialized::<Vec<DBProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -419,7 +420,7 @@ impl DBUser {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
USERS_PROJECTS_NAMESPACE,
user_id.0,
&db_projects,
@@ -12,6 +12,7 @@ use crate::models::exp;
use crate::models::projects::{FileType, VersionStatus};
use crate::queue::file_scan::scan_file;
use crate::routes::internal::delphi::DelphiRunParameters;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use dashmap::{DashMap, DashSet};
use futures::TryStreamExt;
@@ -22,8 +23,8 @@ use std::collections::HashMap;
use std::iter;
use tracing::error;
pub const VERSIONS_NAMESPACE: &str = "versions";
const VERSION_FILES_NAMESPACE: &str = "versions_files";
pub const VERSIONS_NAMESPACE: &str = "versions:v2";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v1";
pub async fn cleanup_unused_attribution_files_and_groups(
transaction: &mut PgTransaction<'_>,
@@ -375,7 +376,7 @@ impl DBLoaderVersion {
}
}
#[derive(Clone, Deserialize, Serialize)]
#[derive(Clone, Deserialize, Serialize, CachedProjection)]
pub struct DBVersion {
pub id: DBVersionId,
pub project_id: DBProjectId,
@@ -390,6 +391,7 @@ pub struct DBVersion {
pub status: VersionStatus,
pub requested_status: Option<VersionStatus>,
pub ordering: Option<i32>,
#[cached_projection(nested)]
pub components: exp::VersionSerial,
}
@@ -1090,27 +1092,32 @@ impl DBVersion {
}
}
#[derive(Clone, Deserialize, Serialize)]
#[derive(Clone, Deserialize, Serialize, CachedProjection)]
pub struct VersionQueryResult {
#[cached_projection(nested)]
pub inner: DBVersion,
pub files: Vec<FileQueryResult>,
#[cached_projection(nested)]
pub version_fields: Vec<VersionField>,
pub loaders: Vec<String>,
pub project_types: Vec<String>,
pub games: Vec<String>,
#[cached_projection(nested)]
pub dependencies: Vec<DependencyQueryResult>,
#[serde(flatten)]
#[cached_projection(nested)]
pub components: exp::VersionQuery,
}
#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
#[derive(Clone, Deserialize, Serialize, PartialEq, Eq, CachedProjection)]
pub struct DependencyQueryResult {
pub id: i32,
pub project_id: Option<DBProjectId>,
pub version_id: Option<DBVersionId>,
pub file_name: Option<String>,
pub dependency_type: String,
#[cached_projection(nested)]
pub attribution: Option<crate::models::projects::DependencyAttribution>,
}
@@ -1125,7 +1132,7 @@ pub struct FileQueryResult {
pub file_type: Option<FileType>,
}
#[derive(Clone, Deserialize, Serialize)]
#[derive(Clone, Deserialize, Serialize, CachedProjection)]
pub struct DBFile {
pub id: DBFileId,
pub version_id: DBVersionId,
+165 -46
View File
@@ -2,6 +2,7 @@ use crate::env::ENV;
use super::models::DatabaseError;
use ariadne::ids::base62_impl::{parse_base62, to_base62};
use cached_projection::CachedProjection;
use chrono::{TimeZone, Utc};
use dashmap::DashMap;
use deadpool_redis::{Config, Runtime};
@@ -12,12 +13,15 @@ use prometheus::{IntGauge, Registry};
use redis::ToRedisArgs;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::{Debug, Display};
use std::future::Future;
use std::hash::Hash;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
use tracing::{Instrument, info, info_span};
use util::{cmd, redis_pipe};
@@ -41,8 +45,121 @@ const MGET_CHUNK_SIZE: usize = 32;
// BytesMut peak capacity that builds up under steady load.
const REDIS_MAX_CONN_AGE: Duration = Duration::from_secs(120);
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Codec {
Raw = 0,
Lz4 = 1,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncodingFormat {
Json,
Postcard,
}
#[derive(Debug, Error)]
#[error("invalid redis codec")]
pub struct InvalidCodec;
#[derive(Debug, Error)]
#[error("invalid redis encoding format")]
pub struct InvalidEncodingFormat;
impl TryFrom<u8> for Codec {
type Error = InvalidCodec;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(Self::Raw),
1 => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for Codec {
type Err = InvalidCodec;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"lz4" => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for EncodingFormat {
type Err = InvalidEncodingFormat;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"json" => Ok(Self::Json),
"postcard" => Ok(Self::Postcard),
_ => Err(InvalidEncodingFormat),
}
}
}
fn encode_value<T>(value: &T) -> Result<Vec<u8>, DatabaseError>
where
T: CachedProjection + Serialize,
{
let mut value = match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::to_vec(value)?,
EncodingFormat::Postcard => {
postcard::to_allocvec(&value.project_ref())?
}
};
if ENV.REDIS_COMPRESSION_LEVEL > 0
&& ENV.REDIS_COMPRESSION_ALGORITHM == Codec::Lz4
&& value.len() >= ENV.REDIS_COMPRESSION_THRESHOLD_BYTES
{
let compressed = lz4_flex::block::compress_prepend_size(&value);
let savings_ratio = value.len().saturating_sub(compressed.len()) as f64
/ value.len().max(1) as f64
* 100.0;
if savings_ratio >= ENV.REDIS_COMPRESSION_MIN_SAVINGS_RATIO {
let mut encoded = Vec::with_capacity(compressed.len() + 1);
encoded.push(Codec::Lz4 as u8);
encoded.extend(compressed);
return Ok(encoded);
}
}
let mut encoded = Vec::with_capacity(value.len() + 1);
encoded.push(Codec::Raw as u8);
encoded.append(&mut value);
Ok(encoded)
}
fn decode_value<T>(value: &[u8]) -> Option<T>
where
T: CachedProjection + DeserializeOwned,
{
let (codec, value) = value.split_first()?;
let value = match Codec::try_from(*codec).ok()? {
Codec::Raw => Cow::Borrowed(value),
Codec::Lz4 => {
Cow::Owned(lz4_flex::block::decompress_size_prepended(value).ok()?)
}
};
match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::from_slice(&value).ok(),
EncodingFormat::Postcard => {
let projected =
postcard::from_bytes::<T::ProjectedType>(&value).ok()?;
Some(T::from_projected(projected))
}
}
}
fn cache_expiries(namespace: &str) -> (i64, i64) {
match namespace {
// Namespaces may embed a version suffix like `:v1`, so split it out.
match namespace.split_once(':').map(|t| t.0).unwrap_or(namespace) {
"versions" | "versions_files" => {
(VERSION_DEFAULT_EXPIRY, VERSION_ACTUAL_EXPIRY)
}
@@ -188,7 +305,7 @@ impl RedisPool {
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, DatabaseError>>,
T: Serialize + DeserializeOwned,
T: CachedProjection + Serialize + DeserializeOwned,
K: Display
+ Hash
+ Eq
@@ -216,7 +333,7 @@ impl RedisPool {
where
F: FnOnce(Vec<K>) -> Fut,
Fut: Future<Output = Result<DashMap<K, T>, DatabaseError>>,
T: Serialize + DeserializeOwned,
T: CachedProjection + Serialize + DeserializeOwned,
K: Display
+ Hash
+ Eq
@@ -254,7 +371,7 @@ impl RedisPool {
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, DatabaseError>>,
T: Serialize + DeserializeOwned,
T: CachedProjection + Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
+ Hash
@@ -291,7 +408,7 @@ impl RedisPool {
where
F: FnOnce(Vec<I>) -> Fut,
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, DatabaseError>>,
T: Serialize + DeserializeOwned,
T: CachedProjection + Serialize + DeserializeOwned,
I: Display + Hash + Eq + PartialEq + Clone + Debug,
K: Display
+ Hash
@@ -367,12 +484,11 @@ impl RedisPool {
for chunk in args.chunks(MGET_CHUNK_SIZE) {
let part = cmd("MGET")
.arg(chunk)
.query_async::<Vec<Option<String>>>(&mut connection)
.query_async::<Vec<Option<Vec<u8>>>>(&mut connection)
.await?;
cached_values.extend(part.into_iter().filter_map(|x| {
x.and_then(|val| {
serde_json::from_str::<RedisValue<T, K, S>>(&val)
.ok()
decode_value::<RedisValue<T, K, S>>(&val)
})
.map(|val| (val.key.clone(), val))
}));
@@ -492,7 +608,7 @@ impl RedisPool {
"{}_{namespace}:{key}",
self.meta_namespace
),
serde_json::to_string(&value)?,
encode_value(&value)?,
default_expiry as u64,
);
pipe_cmds += 1;
@@ -624,47 +740,39 @@ impl<'a> Drop for LockSentinel<'a> {
impl RedisConnection {
#[tracing::instrument(skip(self))]
pub async fn set(
pub async fn set<D>(
&mut self,
namespace: &str,
id: &str,
data: &str,
data: D,
expiry: Option<i64>,
) -> Result<(), DatabaseError> {
) -> Result<(), DatabaseError>
where
D: ToRedisArgs + Send + Sync + Debug,
{
let mut cmd = cmd("SET");
redis_args(
&mut cmd,
vec![
format!("{}_{}:{}", self.meta_namespace, namespace, id),
data.to_string(),
"EX".to_string(),
expiry.unwrap_or(DEFAULT_EXPIRY).to_string(),
]
.as_slice(),
);
cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id))
.arg(data)
.arg("EX")
.arg(expiry.unwrap_or(DEFAULT_EXPIRY));
redis_execute::<()>(&mut cmd, &mut self.connection).await?;
Ok(())
}
#[tracing::instrument(skip(self, id, data))]
pub async fn set_serialized_to_json<Id, D>(
pub async fn set_serialized<Id, D>(
&mut self,
namespace: &str,
id: Id,
data: D,
data: &D,
expiry: Option<i64>,
) -> Result<(), DatabaseError>
where
Id: Display,
D: serde::Serialize,
D: CachedProjection + serde::Serialize,
{
self.set(
namespace,
&id.to_string(),
&serde_json::to_string(&data)?,
expiry,
)
.await
self.set(namespace, &id.to_string(), encode_value(data)?, expiry)
.await
}
#[tracing::instrument(skip(self))]
@@ -688,7 +796,7 @@ impl RedisConnection {
&mut self,
namespace: &str,
ids: &[String],
) -> Result<Vec<Option<String>>, DatabaseError> {
) -> Result<Vec<Option<Vec<u8>>>, DatabaseError> {
let mut cmd = cmd("MGET");
redis_args(
&mut cmd,
@@ -702,35 +810,40 @@ impl RedisConnection {
}
#[tracing::instrument(skip(self))]
pub async fn get_deserialized_from_json<R>(
pub async fn get_deserialized<R>(
&mut self,
namespace: &str,
id: &str,
) -> Result<Option<R>, DatabaseError>
where
R: for<'a> serde::Deserialize<'a>,
R: CachedProjection + DeserializeOwned,
{
Ok(self
.get(namespace, id)
.await?
.and_then(|x| serde_json::from_str(&x).ok()))
let mut cmd = cmd("GET");
redis_args(
&mut cmd,
vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
let value: Option<Vec<u8>> =
redis_execute(&mut cmd, &mut self.connection).await?;
Ok(value.and_then(|value| decode_value(&value)))
}
#[tracing::instrument(skip(self))]
pub async fn get_many_deserialized_from_json<R>(
pub async fn get_many_deserialized<R>(
&mut self,
namespace: &str,
ids: &[String],
) -> Result<Vec<Option<R>>, DatabaseError>
where
R: for<'a> serde::Deserialize<'a>,
R: CachedProjection + DeserializeOwned,
{
Ok(self
.get_many(namespace, ids)
.await?
.into_iter()
.map(|x| x.and_then(|val| serde_json::from_str::<R>(&val).ok()))
.collect::<Vec<_>>())
.map(|value| value.and_then(|value| decode_value::<R>(&value)))
.collect())
}
#[tracing::instrument(skip(self, id))]
@@ -799,7 +912,7 @@ impl RedisConnection {
namespace: &str,
key: &str,
timeout: Option<f64>,
) -> Result<Option<[String; 2]>, DatabaseError> {
) -> Result<Option<[Vec<u8>; 2]>, DatabaseError> {
let key = format!("{}_{namespace}:{key}", self.meta_namespace);
// a timeout of 0 is infinite
let timeout = timeout.unwrap_or(0.0);
@@ -826,15 +939,21 @@ impl RedisConnection {
}
}
#[derive(Serialize, Deserialize)]
#[derive(CachedProjection, Serialize, Deserialize)]
pub struct RedisValue<T, K, S> {
key: K,
#[serde(skip_serializing_if = "Option::is_none")]
alias: Option<S>,
iat: i64,
#[cached_projection(nested)]
val: T,
}
impl<T, K, S> RedisValue<T, K, S> {
pub fn value(&self) -> &T {
&self.val
}
}
pub fn redis_args(cmd: &mut util::InstrumentedCmd, args: &[String]) {
for arg in args {
cmd.arg(arg);
+5
View File
@@ -289,6 +289,11 @@ vars! {
REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64;
REDIS_MAX_CONNECTIONS: u32 = 10000u32;
REDIS_MIN_CONNECTIONS: usize = 0usize;
REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Postcard;
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4;
REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize;
REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64;
SEARCH_OPERATION_TIMEOUT: u64 = 300000u64;
+28 -9
View File
@@ -1,5 +1,6 @@
use std::time::Duration;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use eyre::Result;
use serde::{Deserialize, Serialize};
@@ -109,11 +110,11 @@ pub enum Language {
}
component::define! {
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)]
pub struct ModProject {}
/// Listing for a Minecraft server.
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)]
pub struct ServerProject {
#[base(serde(default))]
#[edit(serde(
@@ -160,11 +161,11 @@ component::define! {
}
/// Version of a Minecraft Java server listing.
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)]
pub struct JavaServerVersion {}
/// Listing for a Minecraft Bedrock server.
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)]
pub struct BedrockServerProject {
#[base()]
#[edit(serde(default))]
@@ -200,13 +201,22 @@ impl ProjectComponent for BedrockServerProject {
}
/// Listing for a Minecraft Java server.
#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)]
#[derive(
Debug,
Clone,
Serialize,
Deserialize,
Validate,
utoipa::ToSchema,
CachedProjection,
)]
pub struct JavaServerProject {
/// Address (IP or domain name) of the Java server, excluding port.
#[validate(length(max = 255))]
pub address: String,
/// What game content this server is using.
#[serde(default)]
#[cached_projection(nested)]
pub content: ServerContent,
}
@@ -219,9 +229,12 @@ pub struct JavaServerProjectEdit {
pub content: Option<ServerContent>,
}
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(
Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection,
)]
pub struct JavaServerProjectQuery {
pub address: String,
#[cached_projection(nested)]
pub content: ServerContentQuery,
pub ping: Option<JavaServerPing>,
pub verified_plays_2w: Option<u64>,
@@ -324,7 +337,9 @@ impl ComponentEdit for JavaServerProjectEdit {
}
/// What game content a [`JavaServerProject`] is using.
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(
Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection,
)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ServerContent {
/// Server runs modded content with a modpack found on the Modrinth platform.
@@ -346,7 +361,9 @@ pub enum ServerContent {
}
/// What game content a [`JavaServerProject`] is using.
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(
Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection,
)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ServerContentQuery {
/// Server runs modded content with a modpack found on the Modrinth platform.
@@ -399,7 +416,9 @@ pub enum ServerRegion {
}
/// Recorded ping attempt that Labrinth made to a Minecraft Java server project.
#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(
Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection,
)]
pub struct JavaServerPing {
/// When the ping was performed.
pub when: DateTime<Utc>,
+7 -4
View File
@@ -1,3 +1,4 @@
use cached_projection::CachedProjection;
use eyre::Result;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
@@ -52,11 +53,12 @@ macro_rules! define_project_components {
impl ComponentKind for ProjectComponentKind {}
#[derive(Debug, Clone, Default, Serialize, Deserialize, Validate)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, Validate, CachedProjection)]
pub struct ProjectSerial {
$(
#[validate(nested)]
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cached_projection(nested)]
pub $field_name: Option<$ty>,
)*
}
@@ -113,10 +115,11 @@ macro_rules! define_project_components {
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema, CachedProjection)]
pub struct ProjectQuery {
$(
#[serde(skip_serializing_if = "Option::is_none")]
#[cached_projection(nested)]
pub $field_name: Option<Query<$ty>>,
)*
}
@@ -258,7 +261,7 @@ pub async fn fetch_query_context(
HashMap::new()
} else {
redis
.get_many_deserialized_from_json::<minecraft::JavaServerPing>(
.get_many_deserialized::<minecraft::JavaServerPing>(
server_ping::REDIS_NAMESPACE,
&minecraft_java_server_pings
.iter()
@@ -281,7 +284,7 @@ pub async fn fetch_query_context(
HashMap::new()
} else {
redis
.get_many_deserialized_from_json::<MinecraftServerAnalytics>(
.get_many_deserialized::<MinecraftServerAnalytics>(
MINECRAFT_SERVER_ANALYTICS,
&minecraft_server_analytics
.iter()
+5 -2
View File
@@ -4,6 +4,7 @@ use crate::models::exp::{
minecraft,
};
use cached_projection::CachedProjection;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use validator::Validate;
@@ -43,9 +44,10 @@ macro_rules! define_version_components {
)*
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, CachedProjection)]
pub struct VersionSerial {
$(
#[cached_projection(nested)]
pub $field_name: Option<$ty>,
)*
}
@@ -71,9 +73,10 @@ macro_rules! define_version_components {
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema, CachedProjection)]
pub struct VersionQuery {
$(
#[cached_projection(nested)]
pub $field_name: Option<Query<$ty>>,
)*
}
+3 -2
View File
@@ -2,6 +2,7 @@ use crate::models::ids::{
ChargeId, ProductId, ProductPriceId, UserSubscriptionId,
};
use ariadne::ids::UserId;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -14,7 +15,7 @@ pub struct Product {
pub unitary: bool,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
#[serde(tag = "type", rename_all = "kebab-case")]
pub enum ProductMetadata {
Midas,
@@ -55,7 +56,7 @@ pub struct ProductPrice {
pub currency_code: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, CachedProjection)]
#[serde(tag = "type", rename_all = "kebab-case")]
pub enum Price {
OneTime {
+2 -1
View File
@@ -10,6 +10,7 @@ use crate::models::ids::{
use crate::models::projects::ProjectStatus;
use crate::routes::ApiError;
use ariadne::ids::UserId;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -151,7 +152,7 @@ impl NotificationType {
}
}
#[derive(Serialize, Deserialize, Clone)]
#[derive(Serialize, Deserialize, Clone, CachedProjection)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum NotificationBody {
ProjectUpdate {
+28 -3
View File
@@ -10,6 +10,7 @@ use crate::models::ids::{
};
use crate::routes::{FileHash, HashAlgorithm};
use ariadne::ids::UserId;
use cached_projection::CachedProjection;
use chrono::{DateTime, Utc};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
@@ -676,7 +677,14 @@ pub struct FlameProject {
}
#[derive(
Debug, Serialize, Deserialize, Clone, PartialEq, Eq, utoipa::ToSchema,
Debug,
Serialize,
Deserialize,
Clone,
PartialEq,
Eq,
utoipa::ToSchema,
CachedProjection,
)]
#[serde(untagged)]
pub enum AttributionLicense {
@@ -685,11 +693,19 @@ pub enum AttributionLicense {
}
#[derive(
Debug, Serialize, Deserialize, Clone, PartialEq, Eq, utoipa::ToSchema,
Debug,
Serialize,
Deserialize,
Clone,
PartialEq,
Eq,
utoipa::ToSchema,
CachedProjection,
)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum AttributionResolutionKind {
License {
#[cached_projection(nested)]
license: AttributionLicense,
link_to_work: Url,
},
@@ -697,6 +713,7 @@ pub enum AttributionResolutionKind {
link_to_work: Url,
},
MyProject {
#[cached_projection(nested)]
license: AttributionLicense,
},
SpecialPermissions {
@@ -1004,12 +1021,20 @@ pub struct Dependency {
}
#[derive(
Serialize, Deserialize, Clone, Debug, PartialEq, Eq, utoipa::ToSchema,
Serialize,
Deserialize,
Clone,
Debug,
PartialEq,
Eq,
utoipa::ToSchema,
CachedProjection,
)]
pub struct DependencyAttribution {
#[serde(skip_serializing_if = "Option::is_none")]
pub flame_project: Option<FlameProject>,
#[serde(skip_serializing_if = "Option::is_none")]
#[cached_projection(nested)]
pub resolution: Option<AttributionResolutionKind>,
}
+5 -4
View File
@@ -1,5 +1,6 @@
use std::collections::HashMap;
use cached_projection::CachedProjection;
use const_format::formatcp;
use eyre::{Result, eyre};
use serde::{Deserialize, Serialize};
@@ -13,9 +14,9 @@ use crate::{
util::error::Context,
};
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics";
pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v1";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)]
pub struct MinecraftServerAnalytics {
pub verified_plays_2w: u64,
pub verified_plays_4w: u64,
@@ -117,10 +118,10 @@ pub async fn cache_analytics(
debug!("Caching analytics for {project_id}: {analytics:?}");
redis
.set_serialized_to_json(
.set_serialized(
MINECRAFT_SERVER_ANALYTICS,
project_id.to_string(),
analytics,
&analytics,
None,
)
.await
+3 -3
View File
@@ -13,9 +13,9 @@ use tracing::trace;
pub mod cache;
const DOWNLOADS_NAMESPACE: &str = "downloads";
const VIEWS_NAMESPACE: &str = "views";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays";
const DOWNLOADS_NAMESPACE: &str = "downloads:v1";
const VIEWS_NAMESPACE: &str = "views:v1";
const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v1";
const MINECRAFT_SERVER_PLAYS_EXPIRY: u64 = 86_400; // 24 hours
const MINECRAFT_SERVER_PLAYS_LIMIT: u32 = 5;
+6 -10
View File
@@ -26,8 +26,9 @@ pub struct ServerPingQueue {
pub incremental_search_queue: IncrementalSearchQueue,
}
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping";
pub const REDIS_FAILURE_NAMESPACE: &str = "minecraft_java_server_ping_failures";
pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v1";
pub const REDIS_FAILURE_NAMESPACE: &str =
"minecraft_java_server_ping_failures:v1";
pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings";
impl ServerPingQueue {
@@ -133,12 +134,7 @@ impl ServerPingQueue {
// ping succeeded; immediately update its online status in redis
redis
.set_serialized_to_json(
REDIS_NAMESPACE,
project_id,
ping,
None,
)
.set_serialized(REDIS_NAMESPACE, project_id, ping, None)
.await
.wrap_err("failed to set redis key")?;
updated_project = true;
@@ -160,7 +156,7 @@ impl ServerPingQueue {
&& count >= ENV.SERVER_PING_MAX_FAIL_COUNT
{
redis
.set_serialized_to_json(
.set_serialized(
REDIS_NAMESPACE,
project_id,
ping,
@@ -254,7 +250,7 @@ impl ServerPingQueue {
.collect::<Vec<_>>();
let all_server_last_pings = redis
.get_many_deserialized_from_json::<exp::minecraft::JavaServerPing>(
.get_many_deserialized::<exp::minecraft::JavaServerPing>(
REDIS_NAMESPACE,
&all_project_ids,
)
@@ -1,5 +1,6 @@
use actix_web::{HttpRequest, get, post, web};
use base64::Engine;
use cached_projection::CachedProjection;
use chrono::{DateTime, Duration, Utc};
use eyre::eyre;
use hmac::{Hmac, Mac};
@@ -60,15 +61,19 @@ struct TiltifyMeta {
subscription_source_type: String,
}
#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)]
#[derive(
Clone, Debug, Serialize, Deserialize, utoipa::ToSchema, CachedProjection,
)]
pub struct CampaignInfo {
#[cached_projection(wrap)]
total_donations_usd: Decimal,
#[cached_projection(wrap)]
target_usd: Decimal,
num_donators: usize,
cached_at: DateTime<Utc>,
}
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info";
const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v2";
const CAMPAIGN_INFO_CACHE_STALE_SECONDS: i64 = 15 * 60;
const CAMPAIGN_INFO_CACHE_TTL_SECONDS: i64 = 24 * 60 * 60;
@@ -326,7 +331,7 @@ pub async fn pride_26(
.wrap_internal_err("connecting to redis")?;
let cached = redis_connection
.get_deserialized_from_json::<CampaignInfo>(
.get_deserialized::<CampaignInfo>(
CAMPAIGN_INFO_CACHE_NAMESPACE,
campaign_id,
)
@@ -382,7 +387,7 @@ pub async fn pride_26(
};
redis_connection
.set_serialized_to_json(
.set_serialized(
CAMPAIGN_INFO_CACHE_NAMESPACE,
campaign_id,
&campaign_info,
@@ -427,7 +427,7 @@ async fn broadcast_notifications(
redis,
RedisFriendsMessage::Notification {
to_user,
notification,
notification_id,
},
)
.await
+1 -1
View File
@@ -2126,7 +2126,7 @@ async fn validate_2fa_code(
)
.map_err(|_| AuthenticationError::InvalidCredentials)?;
const TOTP_NAMESPACE: &str = "used_totp";
const TOTP_NAMESPACE: &str = "used_totp:v1";
let mut conn = redis.connect().await?;
// Check if TOTP has already been used
@@ -70,11 +70,11 @@ pub async fn success_callback(
let body = base64::engine::general_purpose::STANDARD.encode(&body);
let redis_msg = serde_json::to_string(&Ok::<
let redis_msg = postcard::to_allocvec(&Ok::<
GotenbergDocument,
GotenbergError,
>(GotenbergDocument { body }))
.wrap_internal_err("failed to serialize document to JSON")?;
.wrap_internal_err("failed to serialize Redis document response")?;
redis
.lpush(
@@ -139,11 +139,11 @@ pub async fn error_callback(
.await
.wrap_internal_err("failed to get Redis connection")?;
let redis_msg = serde_json::to_string(&Err::<
let redis_msg = postcard::to_allocvec(&Err::<
GotenbergDocument,
GotenbergError,
>(error_body))
.wrap_internal_err("failed to serialize error to JSON")?;
.wrap_internal_err("failed to serialize Redis error response")?;
redis
.lpush(
+7 -5
View File
@@ -14,6 +14,7 @@ use crate::queue::session::AuthQueue;
use actix_web::{HttpRequest, post, web};
use ariadne::ids::base62_impl::parse_base62;
use async_trait::async_trait;
use cached_projection::CachedProjection;
use modrinth_content_management::{
ContentMetadataProvider, Error as ResolveError, ResolveContentPlan,
ResolveContentRequest,
@@ -22,8 +23,8 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat";
const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v2";
const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v1";
const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v1";
const CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS: i64 = 60 * 60 * 24;
@@ -90,9 +91,10 @@ struct ResolveContentTrace {
project_versions: BTreeMap<String, String>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, CachedProjection)]
struct CachedResolveContentPlan {
trace: ResolveContentTrace,
#[cached_projection(wrap)]
plan: ResolveContentPlan,
}
@@ -340,7 +342,7 @@ async fn get_cached_resolve_content_plan(
};
match redis
.get_deserialized_from_json(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key)
.get_deserialized(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key)
.await
{
Ok(cached) => cached,
@@ -368,7 +370,7 @@ async fn set_cached_resolve_content_plan(
};
if let Err(error) = redis
.set_serialized_to_json(
.set_serialized(
CONTENT_RESOLVE_CACHE_NAMESPACE,
cache_key,
cached,
+1 -1
View File
@@ -151,7 +151,7 @@ async fn hydrate_search_results(
} else {
let mut redis = redis_pool.connect().await?;
let ping_results = redis
.get_many_deserialized_from_json::<JavaServerPing>(
.get_many_deserialized::<JavaServerPing>(
server_ping::REDIS_NAMESPACE,
&project_ids
.iter()
+17 -12
View File
@@ -1,4 +1,6 @@
use crate::database::PgPool;
use crate::database::models::notification_item::DBNotification;
use crate::models::ids::NotificationId;
use crate::models::notifications::Notification;
use crate::queue::socket::ActiveSockets;
use crate::routes::internal::statuses::{
@@ -13,10 +15,9 @@ use redis::{RedisWrite, ToRedisArgs};
use serde::{Deserialize, Serialize};
use tokio_stream::StreamExt;
pub const FRIENDS_CHANNEL_NAME: &str = "friends";
pub const FRIENDS_CHANNEL_NAME: &str = "friends:v1";
#[derive(Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum RedisFriendsMessage {
StatusUpdate {
status: UserStatus,
@@ -30,7 +31,7 @@ pub enum RedisFriendsMessage {
},
Notification {
to_user: UserId,
notification: Notification,
notification_id: NotificationId,
},
}
@@ -39,7 +40,7 @@ impl ToRedisArgs for RedisFriendsMessage {
where
W: ?Sized + RedisWrite,
{
out.write_arg(&serde_json::to_vec(&self).unwrap())
out.write_arg(&postcard::to_allocvec(&self).unwrap())
}
}
@@ -54,7 +55,7 @@ pub async fn handle_pubsub(
if message.get_channel_name() != FRIENDS_CHANNEL_NAME {
continue;
}
let payload = serde_json::from_slice(message.get_payload_bytes());
let payload = postcard::from_bytes(message.get_payload_bytes());
let pool = pool.clone();
let sockets = sockets.clone();
@@ -94,14 +95,18 @@ pub async fn handle_pubsub(
Ok(RedisFriendsMessage::Notification {
to_user,
notification,
notification_id,
}) => {
let _ = send_notification_to_user(
&sockets,
to_user,
&notification,
)
.await;
if let Ok(Some(notification)) =
DBNotification::get(notification_id.into(), &pool).await
{
let _ = send_notification_to_user(
&sockets,
to_user,
&Notification::from(notification),
)
.await;
}
}
Err(_) => {}
+6 -6
View File
@@ -17,10 +17,10 @@ pub async fn get_user_status(
if let Ok(mut conn) = redis.pool.get().await
&& let Ok(mut statuses) =
conn.sscan::<_, String>(get_field_name(user)).await
&& let Some(status_json) = statuses.next_item().await
conn.sscan::<_, Vec<u8>>(get_field_name(user)).await
&& let Some(status) = statuses.next_item().await
{
return serde_json::from_str::<UserStatus>(&status_json).ok();
return postcard::from_bytes::<UserStatus>(&status).ok();
}
None
@@ -40,11 +40,11 @@ pub async fn replace_user_status(
let mut pipe = redis::pipe();
pipe.atomic();
if let Some(status) = old_status {
pipe.srem(&field_name, serde_json::to_string(&status).unwrap())
pipe.srem(&field_name, postcard::to_allocvec(status).unwrap())
.ignore();
}
if let Some(status) = new_status {
pipe.sadd(&field_name, serde_json::to_string(&status).unwrap())
pipe.sadd(&field_name, postcard::to_allocvec(status).unwrap())
.ignore();
pipe.expire(&field_name, EXPIRY_TIME_SECONDS).ignore();
}
@@ -65,5 +65,5 @@ pub async fn push_back_user_expiry(
}
fn get_field_name(user: UserId) -> String {
format!("user_status:{user}")
format!("user_status:v1:{user}")
}
+2 -2
View File
@@ -14,7 +14,7 @@ pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName =
HeaderName::from_static("modrinth-generated-pdf-type");
pub const MODRINTH_PAYMENT_ID: HeaderName =
HeaderName::from_static("modrinth-payment-id");
pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements";
pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v1";
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct PaymentStatement {
@@ -203,7 +203,7 @@ impl GotenbergClient {
.wrap_internal_err("failed to get document over Redis")?
.wrap_internal_err("no document was returned from Redis")?;
let document = serde_json::from_str::<
let document = postcard::from_bytes::<
Result<GotenbergDocument, GotenbergError>,
>(&document)
.wrap_internal_err("failed to deserialize Redis document response")?
+1 -1
View File
@@ -11,7 +11,7 @@ use chrono::Utc;
use std::str::FromStr;
use std::sync::Arc;
const RATE_LIMIT_NAMESPACE: &str = "rate_limit";
const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v1";
const RATE_LIMIT_EXPIRY: i64 = 300; // 5 minutes
const MINUTE_IN_NANOS: i64 = 60_000_000_000;
+11 -7
View File
@@ -18,9 +18,11 @@ use common::environment::{
use common::permissions::{PermissionsTest, PermissionsTestContext};
use futures::StreamExt;
use hex::ToHex;
use labrinth::database::models::DBProjectId;
use labrinth::database::models::project_item::{
PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE,
PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, ProjectQueryResult,
};
use labrinth::database::redis::RedisValue;
use labrinth::models::ids::ProjectId;
use labrinth::models::teams::ProjectPermissions;
use labrinth::util::actix::{MultipartSegment, MultipartSegmentData};
@@ -67,19 +69,21 @@ async fn test_get_project() {
Some(parse_base62(alpha_project_id).unwrap() as i64)
);
let cached_project = redis_pool
.get(
let cached_project: RedisValue<
ProjectQueryResult,
DBProjectId,
String,
> = redis_pool
.get_deserialized(
PROJECTS_NAMESPACE,
&parse_base62(alpha_project_id).unwrap().to_string(),
)
.await
.unwrap()
.unwrap();
let cached_project: serde_json::Value =
serde_json::from_str(&cached_project).unwrap();
assert_eq!(
cached_project["val"]["inner"]["slug"],
json!(alpha_project_slug)
cached_project.value().inner.slug.as_ref(),
Some(alpha_project_slug)
);
// Make the request again, this time it should be cached
+12 -6
View File
@@ -14,7 +14,11 @@ use common::asserts::assert_common_version_ids;
use common::database::USER_USER_PAT;
use common::environment::{with_test_environment, with_test_environment_all};
use futures::StreamExt;
use labrinth::database::models::version_item::VERSIONS_NAMESPACE;
use labrinth::database::models::DBVersionId;
use labrinth::database::models::version_item::{
VERSIONS_NAMESPACE, VersionQueryResult,
};
use labrinth::database::redis::RedisValue;
use labrinth::models::ids::VersionId;
use labrinth::models::projects::{
Dependency, DependencyType, VersionStatus, VersionType,
@@ -47,18 +51,20 @@ async fn test_get_version() {
assert_eq!(&version.id.to_string(), alpha_version_id);
let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap();
let cached_project = redis_conn
.get(
let cached_version: RedisValue<
VersionQueryResult,
DBVersionId,
String,
> = redis_conn
.get_deserialized(
VERSIONS_NAMESPACE,
&parse_base62(alpha_version_id).unwrap().to_string(),
)
.await
.unwrap()
.unwrap();
let cached_project: serde_json::Value =
serde_json::from_str(&cached_project).unwrap();
assert_eq!(
cached_project["val"]["inner"]["project_id"],
cached_version.value().inner.project_id.0,
json!(parse_base62(alpha_project_id).unwrap())
);
@@ -0,0 +1,16 @@
[package]
name = "cached-projection-derive"
edition.workspace = true
rust-version.workspace = true
repository.workspace = true
[lib]
proc-macro = true
[dependencies]
proc-macro2 = { workspace = true }
quote = { workspace = true }
syn = { workspace = true, features = ["full"] }
[lints]
workspace = true
@@ -0,0 +1,765 @@
use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use quote::{format_ident, quote};
use syn::punctuated::Punctuated;
use syn::{
Attribute, Data, DataEnum, DataStruct, DeriveInput, Field, Fields,
GenericParam, Generics, Lifetime, LifetimeParam, Meta, Token, Type,
parse_macro_input, parse_quote,
};
#[proc_macro_derive(CachedProjection, attributes(cached_projection))]
pub fn cached_projection(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as DeriveInput);
derive_cached_projection(&input)
.unwrap_or_else(syn::Error::into_compile_error)
.into()
}
#[derive(Clone, Copy)]
enum Projection {
Ordinary,
Nested,
Wrap,
}
fn derive_cached_projection(input: &DeriveInput) -> syn::Result<TokenStream2> {
let name = &input.ident;
let projected_name = format_ident!("{name}ProjectedType");
let projected_ref_name = format_ident!("{name}ProjectedTypeRef");
let visibility = &input.vis;
let lifetime = Lifetime::new("'__cached_projection", name.span());
let fields = all_fields(&input.data);
let ref_lifetime = (!fields.is_empty()).then_some(&lifetime);
let projections = fields
.iter()
.map(|field| field_projection(field))
.collect::<syn::Result<Vec<_>>>()?;
let mut implementation_generics = input.generics.clone();
add_projection_bounds(
&mut implementation_generics,
fields.iter().zip(&projections),
);
let (impl_generics, _, where_clause) =
implementation_generics.split_for_impl();
let (_, type_generics, _) = input.generics.split_for_impl();
let mut ref_generics = implementation_generics.clone();
if ref_lifetime.is_some() {
ref_generics.params.insert(
0,
GenericParam::Lifetime(LifetimeParam::new(lifetime.clone())),
);
add_outlives_bounds(&mut ref_generics, fields.iter(), &lifetime);
}
let ref_type_arguments = generic_arguments(&input.generics, ref_lifetime);
let owned_definition = projected_definition(
&input.data,
&projected_name,
visibility,
&implementation_generics,
None,
)?;
let ref_definition = projected_definition(
&input.data,
&projected_ref_name,
visibility,
&ref_generics,
ref_lifetime,
)?;
let project_ref_body =
project_ref_body(&input.data, &projected_ref_name, &projections)?;
let into_projected_body =
into_projected_body(&input.data, &projected_name, &projections)?;
let from_projected_body =
from_projected_body(&input.data, &projected_name, &projections)?;
Ok(quote! {
#owned_definition
#ref_definition
impl #impl_generics ::cached_projection::CachedProjection for #name #type_generics #where_clause {
type ProjectedType = #projected_name #type_generics;
type ProjectedTypeRef<#lifetime> = #projected_ref_name #ref_type_arguments
where
Self: #lifetime;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
#project_ref_body
}
fn into_projected(self) -> Self::ProjectedType {
#into_projected_body
}
fn from_projected(projected: Self::ProjectedType) -> Self {
#from_projected_body
}
}
})
}
fn all_fields(data: &Data) -> Vec<&Field> {
match data {
Data::Struct(data) => data.fields.iter().collect(),
Data::Enum(data) => data
.variants
.iter()
.flat_map(|variant| variant.fields.iter())
.collect(),
Data::Union(data) => data.fields.named.iter().collect(),
}
}
fn field_projection(field: &Field) -> syn::Result<Projection> {
let mut nested = false;
let mut wrap = false;
for attribute in &field.attrs {
if !attribute.path().is_ident("cached_projection") {
continue;
}
attribute.parse_nested_meta(|meta| {
if meta.path.is_ident("nested") {
nested = true;
Ok(())
} else if meta.path.is_ident("wrap") {
wrap = true;
Ok(())
} else {
Err(meta.error("unsupported cached projection option"))
}
})?;
}
if nested && wrap {
return Err(syn::Error::new_spanned(
field,
"`nested` cannot be combined with `wrap`",
));
}
Ok(if nested {
Projection::Nested
} else if wrap {
Projection::Wrap
} else {
Projection::Ordinary
})
}
fn add_projection_bounds<'a>(
generics: &mut Generics,
fields: impl Iterator<Item = (&'a &'a Field, &'a Projection)>,
) {
let where_clause = generics.make_where_clause();
for (field, projection) in fields {
let ty = &field.ty;
let predicate = match projection {
Projection::Nested => {
parse_quote!(#ty: ::cached_projection::CachedProjection)
}
Projection::Ordinary | Projection::Wrap => parse_quote!(
#ty: ::serde::Serialize + ::serde::de::DeserializeOwned
),
};
where_clause.predicates.push(predicate);
}
}
fn add_outlives_bounds<'a>(
generics: &mut Generics,
fields: impl Iterator<Item = &'a &'a Field>,
lifetime: &Lifetime,
) {
let where_clause = generics.make_where_clause();
for field in fields {
let ty = &field.ty;
where_clause.predicates.push(parse_quote!(#ty: #lifetime));
}
}
fn generic_arguments(
generics: &Generics,
lifetime: Option<&Lifetime>,
) -> TokenStream2 {
let mut arguments = Vec::new();
if let Some(lifetime) = lifetime {
arguments.push(quote!(#lifetime));
}
arguments.extend(generics.params.iter().map(|parameter| match parameter {
GenericParam::Lifetime(parameter) => {
let lifetime = &parameter.lifetime;
quote!(#lifetime)
}
GenericParam::Type(parameter) => {
let ident = &parameter.ident;
quote!(#ident)
}
GenericParam::Const(parameter) => {
let ident = &parameter.ident;
quote!(#ident)
}
}));
if arguments.is_empty() {
quote!()
} else {
quote!(<#(#arguments),*>)
}
}
fn projected_definition(
data: &Data,
name: &syn::Ident,
visibility: &syn::Visibility,
generics: &Generics,
lifetime: Option<&Lifetime>,
) -> syn::Result<TokenStream2> {
let derive = if lifetime.is_some() {
quote!(#[derive(::serde::Serialize)])
} else {
quote!(#[derive(::serde::Serialize, ::serde::Deserialize)])
};
let serde_bound = serde_bound_attribute(data, lifetime);
match data {
Data::Struct(data) => {
let fields = projected_fields(&data.fields, lifetime)?;
let where_clause = &generics.where_clause;
let declaration = match &data.fields {
Fields::Named(_) => quote!({ #(#fields),* }),
Fields::Unnamed(_) => {
quote!(( #(#fields),* ) #where_clause;)
}
Fields::Unit => quote!(#where_clause;),
};
let leading_where_clause = match &data.fields {
Fields::Named(_) => quote!(#where_clause),
Fields::Unnamed(_) | Fields::Unit => quote!(),
};
Ok(quote! {
#[doc(hidden)]
#derive
#serde_bound
#visibility struct #name #generics #leading_where_clause #declaration
})
}
Data::Enum(data) => {
let where_clause = &generics.where_clause;
let variants = data
.variants
.iter()
.map(|variant| {
let attributes = definition_attributes(&variant.attrs)?;
let ident = &variant.ident;
let fields = projected_fields(&variant.fields, lifetime)?;
let declaration = match &variant.fields {
Fields::Named(_) => quote!({ #(#fields),* }),
Fields::Unnamed(_) => quote!(( #(#fields),* )),
Fields::Unit => quote!(),
};
Ok(quote!(#(#attributes)* #ident #declaration))
})
.collect::<syn::Result<Vec<_>>>()?;
Ok(quote! {
#[doc(hidden)]
#derive
#serde_bound
#visibility enum #name #generics #where_clause {
#(#variants),*
}
})
}
Data::Union(data) => Err(syn::Error::new_spanned(
data.union_token,
"CachedProjection cannot be derived for unions",
)),
}
}
fn serde_bound_attribute(
data: &Data,
lifetime: Option<&Lifetime>,
) -> TokenStream2 {
let projected_types = all_fields(data)
.into_iter()
.map(|field| {
let projection = field_projection(field)
.expect("field projection was already validated");
projected_field_type(&field.ty, projection, lifetime)
})
.collect::<Vec<_>>();
if projected_types.is_empty() {
return quote!();
}
let serialize_bounds = projected_types
.iter()
.map(|ty| quote!(#ty: ::serde::Serialize))
.collect::<Vec<_>>();
let serialize_bounds = syn::LitStr::new(
&quote!(#(#serialize_bounds),*).to_string(),
proc_macro2::Span::call_site(),
);
if lifetime.is_some() {
quote!(#[serde(bound(serialize = #serialize_bounds))])
} else {
let deserialize_bounds = projected_types
.iter()
.map(|ty| quote!(#ty: ::serde::de::DeserializeOwned))
.collect::<Vec<_>>();
let deserialize_bounds = syn::LitStr::new(
&quote!(#(#deserialize_bounds),*).to_string(),
proc_macro2::Span::call_site(),
);
quote!(#[serde(bound(
serialize = #serialize_bounds,
deserialize = #deserialize_bounds,
))])
}
}
fn projected_fields(
fields: &Fields,
lifetime: Option<&Lifetime>,
) -> syn::Result<Vec<TokenStream2>> {
let projected = fields
.iter()
.map(|field| {
let attributes = definition_attributes(&field.attrs)?;
let visibility = &field.vis;
let ident = &field.ident;
let projection = field_projection(field)?;
let ty = projected_field_type(&field.ty, projection, lifetime);
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #visibility #ident: #ty)
} else {
quote!(#(#attributes)* #visibility #ty)
})
})
.collect::<syn::Result<Vec<_>>>()?;
Ok(projected)
}
fn projected_field_type(
ty: &Type,
projection: Projection,
lifetime: Option<&Lifetime>,
) -> TokenStream2 {
match (projection, lifetime) {
(Projection::Ordinary, None) => quote!(#ty),
(Projection::Ordinary, Some(lifetime)) => quote!(&#lifetime #ty),
(Projection::Nested, None) => quote!(
<#ty as ::cached_projection::CachedProjection>::ProjectedType
),
(Projection::Nested, Some(lifetime)) => quote!(
<#ty as ::cached_projection::CachedProjection>::ProjectedTypeRef<#lifetime>
),
(Projection::Wrap, None) => quote!(
::cached_projection::DeserializeAnyJsonWrapper<#ty>
),
(Projection::Wrap, Some(lifetime)) => quote!(
::cached_projection::DeserializeAnyJsonWrapperRef<#lifetime, #ty>
),
}
}
fn definition_attributes(
attributes: &[Attribute],
) -> syn::Result<Vec<Attribute>> {
let mut filtered = Vec::new();
for attribute in attributes {
if attribute.path().is_ident("cfg") {
filtered.push(attribute.clone());
} else if attribute.path().is_ident("serde") {
if serde_has_skip(attribute)? {
filtered.push(parse_quote!(#[serde(skip)]));
}
} else if attribute.path().is_ident("cfg_attr")
&& let Some(attribute) = filter_cfg_attr(attribute, true)? {
filtered.push(attribute);
}
}
Ok(filtered)
}
fn structural_attributes(
attributes: &[Attribute],
) -> syn::Result<Vec<Attribute>> {
let mut filtered = Vec::new();
for attribute in attributes {
if attribute.path().is_ident("cfg") {
filtered.push(attribute.clone());
} else if attribute.path().is_ident("cfg_attr")
&& let Some(attribute) = filter_cfg_attr(attribute, false)? {
filtered.push(attribute);
}
}
Ok(filtered)
}
fn serde_has_skip(attribute: &Attribute) -> syn::Result<bool> {
let Meta::List(list) = &attribute.meta else {
return Ok(false);
};
let items =
list.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)?;
Ok(items
.iter()
.any(|item| matches!(item, Meta::Path(path) if path.is_ident("skip"))))
}
fn filter_cfg_attr(
attribute: &Attribute,
include_serde_skip: bool,
) -> syn::Result<Option<Attribute>> {
let Meta::List(list) = &attribute.meta else {
return Ok(None);
};
let mut items = list
.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)?
.into_iter();
let Some(condition) = items.next() else {
return Ok(None);
};
let mut retained = Vec::new();
for item in items {
if item.path().is_ident("cfg") {
retained.push(item);
} else if include_serde_skip && item.path().is_ident("serde") {
let attribute: Attribute = parse_quote!(#[#item]);
if serde_has_skip(&attribute)? {
retained.push(parse_quote!(serde(skip)));
}
}
}
if retained.is_empty() {
Ok(None)
} else {
Ok(Some(parse_quote!(#[cfg_attr(#condition, #(#retained),*)])))
}
}
fn project_ref_body(
data: &Data,
projected_name: &syn::Ident,
projections: &[Projection],
) -> syn::Result<TokenStream2> {
match data {
Data::Struct(data) => struct_projection_body(
data,
projected_name,
projections,
Conversion::ProjectRef,
),
Data::Enum(data) => enum_projection_body(
data,
projected_name,
projections,
Conversion::ProjectRef,
),
Data::Union(_) => unreachable!(),
}
}
fn into_projected_body(
data: &Data,
projected_name: &syn::Ident,
projections: &[Projection],
) -> syn::Result<TokenStream2> {
match data {
Data::Struct(data) => struct_projection_body(
data,
projected_name,
projections,
Conversion::IntoProjected,
),
Data::Enum(data) => enum_projection_body(
data,
projected_name,
projections,
Conversion::IntoProjected,
),
Data::Union(_) => unreachable!(),
}
}
fn from_projected_body(
data: &Data,
projected_name: &syn::Ident,
projections: &[Projection],
) -> syn::Result<TokenStream2> {
match data {
Data::Struct(data) => {
struct_from_projected_body(data, projected_name, projections)
}
Data::Enum(data) => {
enum_from_projected_body(data, projected_name, projections)
}
Data::Union(_) => unreachable!(),
}
}
#[derive(Clone, Copy)]
enum Conversion {
ProjectRef,
IntoProjected,
}
fn struct_projection_body(
data: &DataStruct,
projected_name: &syn::Ident,
projections: &[Projection],
conversion: Conversion,
) -> syn::Result<TokenStream2> {
let bindings = field_bindings(&data.fields);
let pattern = fields_pattern(&data.fields, &bindings, None)?;
let target = parse_quote!(#projected_name);
let values = data
.fields
.iter()
.zip(bindings.iter())
.zip(projections)
.map(|((field, binding), projection)| {
let attributes = structural_attributes(&field.attrs)?;
let value = conversion_expression(binding, *projection, conversion);
let ident = &field.ident;
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #ident: #value)
} else {
quote!(#(#attributes)* #value)
})
})
.collect::<syn::Result<Vec<_>>>()?;
let construction = fields_construction(&data.fields, &target, &values);
Ok(quote! {
let #pattern = self;
#construction
})
}
fn struct_from_projected_body(
data: &DataStruct,
projected_name: &syn::Ident,
projections: &[Projection],
) -> syn::Result<TokenStream2> {
let bindings = field_bindings(&data.fields);
let source = parse_quote!(#projected_name);
let pattern = fields_pattern(&data.fields, &bindings, Some(&source))?;
let values = data
.fields
.iter()
.zip(bindings.iter())
.zip(projections)
.map(|((field, binding), projection)| {
let attributes = structural_attributes(&field.attrs)?;
let value = from_expression(binding, &field.ty, *projection);
let ident = &field.ident;
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #ident: #value)
} else {
quote!(#(#attributes)* #value)
})
})
.collect::<syn::Result<Vec<_>>>()?;
let construction =
fields_construction(&data.fields, &parse_quote!(Self), &values);
Ok(quote! {
let #pattern = projected;
#construction
})
}
fn enum_projection_body(
data: &DataEnum,
projected_name: &syn::Ident,
projections: &[Projection],
conversion: Conversion,
) -> syn::Result<TokenStream2> {
let mut offset = 0;
let arms = data
.variants
.iter()
.map(|variant| {
let count = variant.fields.len();
let variant_projections = &projections[offset..offset + count];
offset += count;
let attributes = structural_attributes(&variant.attrs)?;
let bindings = field_bindings(&variant.fields);
let variant_ident = &variant.ident;
let pattern = fields_pattern(
&variant.fields,
&bindings,
Some(&parse_quote!(Self::#variant_ident)),
)?;
let values = variant
.fields
.iter()
.zip(bindings.iter())
.zip(variant_projections)
.map(|((field, binding), projection)| {
let attributes = structural_attributes(&field.attrs)?;
let value =
conversion_expression(binding, *projection, conversion);
let ident = &field.ident;
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #ident: #value)
} else {
quote!(#(#attributes)* #value)
})
})
.collect::<syn::Result<Vec<_>>>()?;
let target = parse_quote!(#projected_name::#variant_ident);
let construction =
fields_construction(&variant.fields, &target, &values);
Ok(quote!(#(#attributes)* #pattern => #construction))
})
.collect::<syn::Result<Vec<_>>>()?;
Ok(quote!(match self { #(#arms),* }))
}
fn enum_from_projected_body(
data: &DataEnum,
projected_name: &syn::Ident,
projections: &[Projection],
) -> syn::Result<TokenStream2> {
let mut offset = 0;
let arms = data
.variants
.iter()
.map(|variant| {
let count = variant.fields.len();
let variant_projections = &projections[offset..offset + count];
offset += count;
let attributes = structural_attributes(&variant.attrs)?;
let bindings = field_bindings(&variant.fields);
let variant_ident = &variant.ident;
let source = parse_quote!(#projected_name::#variant_ident);
let pattern =
fields_pattern(&variant.fields, &bindings, Some(&source))?;
let values = variant
.fields
.iter()
.zip(bindings.iter())
.zip(variant_projections)
.map(|((field, binding), projection)| {
let attributes = structural_attributes(&field.attrs)?;
let value =
from_expression(binding, &field.ty, *projection);
let ident = &field.ident;
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #ident: #value)
} else {
quote!(#(#attributes)* #value)
})
})
.collect::<syn::Result<Vec<_>>>()?;
let target = parse_quote!(Self::#variant_ident);
let construction =
fields_construction(&variant.fields, &target, &values);
Ok(quote!(#(#attributes)* #pattern => #construction))
})
.collect::<syn::Result<Vec<_>>>()?;
Ok(quote!(match projected { #(#arms),* }))
}
fn field_bindings(fields: &Fields) -> Vec<syn::Ident> {
fields
.iter()
.enumerate()
.map(|(index, field)| {
field
.ident
.clone()
.unwrap_or_else(|| format_ident!("field_{index}"))
})
.collect()
}
fn fields_pattern(
fields: &Fields,
bindings: &[syn::Ident],
prefix: Option<&syn::Path>,
) -> syn::Result<TokenStream2> {
let prefix = prefix
.map(|prefix| quote!(#prefix))
.unwrap_or_else(|| quote!(Self));
let entries = fields
.iter()
.zip(bindings)
.map(|(field, binding)| {
let attributes = structural_attributes(&field.attrs)?;
let ident = &field.ident;
Ok(if let Some(ident) = ident {
quote!(#(#attributes)* #ident)
} else {
quote!(#(#attributes)* #binding)
})
})
.collect::<syn::Result<Vec<_>>>()?;
Ok(match fields {
Fields::Named(_) => quote!(#prefix { #(#entries),* }),
Fields::Unnamed(_) => quote!(#prefix( #(#entries),* )),
Fields::Unit => quote!(#prefix),
})
}
fn fields_construction(
fields: &Fields,
target: &syn::Path,
values: &[TokenStream2],
) -> TokenStream2 {
match fields {
Fields::Named(_) => quote!(#target { #(#values),* }),
Fields::Unnamed(_) => quote!(#target( #(#values),* )),
Fields::Unit => quote!(#target),
}
}
fn conversion_expression(
binding: &syn::Ident,
projection: Projection,
conversion: Conversion,
) -> TokenStream2 {
match (projection, conversion) {
(Projection::Ordinary, _) => quote!(#binding),
(Projection::Nested, Conversion::ProjectRef) => quote!(
::cached_projection::CachedProjection::project_ref(#binding)
),
(Projection::Nested, Conversion::IntoProjected) => quote!(
::cached_projection::CachedProjection::into_projected(#binding)
),
(Projection::Wrap, Conversion::ProjectRef) => quote!(
::cached_projection::DeserializeAnyJsonWrapperRef(#binding)
),
(Projection::Wrap, Conversion::IntoProjected) => quote!(
::cached_projection::DeserializeAnyJsonWrapper(#binding)
),
}
}
fn from_expression(
binding: &syn::Ident,
ty: &Type,
projection: Projection,
) -> TokenStream2 {
match projection {
Projection::Ordinary => quote!(#binding),
Projection::Nested => quote!(
<#ty as ::cached_projection::CachedProjection>::from_projected(#binding)
),
Projection::Wrap => quote!(#binding.0),
}
}
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "cached-projection"
edition.workspace = true
rust-version.workspace = true
repository.workspace = true
[dependencies]
cached-projection-derive = { path = "../cached-projection-derive" }
serde = { workspace = true, features = ["derive", "rc"] }
serde_json = { workspace = true }
[dev-dependencies]
postcard = { workspace = true }
[lints]
workspace = true
+429
View File
@@ -0,0 +1,429 @@
// for macro expansion
extern crate self as cached_projection;
use std::collections::{BTreeMap, HashMap};
use std::hash::Hash;
use std::sync::Arc;
pub use cached_projection_derive::CachedProjection;
use serde::de::DeserializeOwned;
use serde::ser::{SerializeMap, SerializeSeq};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
pub trait CachedProjection: Sized {
type ProjectedType: Serialize + DeserializeOwned;
type ProjectedTypeRef<'a>: Serialize
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_>;
fn into_projected(self) -> Self::ProjectedType;
fn from_projected(projected: Self::ProjectedType) -> Self;
}
pub struct DeserializeAnyJsonWrapper<T>(pub T);
impl<T> Serialize for DeserializeAnyJsonWrapper<T>
where
T: Serialize,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serde_json::to_string(&self.0)
.map_err(serde::ser::Error::custom)?
.serialize(serializer)
}
}
impl<'de, T> Deserialize<'de> for DeserializeAnyJsonWrapper<T>
where
T: DeserializeOwned,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
serde_json::from_str(&value)
.map(Self)
.map_err(serde::de::Error::custom)
}
}
pub struct DeserializeAnyJsonWrapperRef<'a, T: ?Sized>(pub &'a T);
impl<T> Serialize for DeserializeAnyJsonWrapperRef<'_, T>
where
T: Serialize + ?Sized,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serde_json::to_string(self.0)
.map_err(serde::ser::Error::custom)?
.serialize(serializer)
}
}
macro_rules! identity_projection {
($($ty:ty),* $(,)?) => {
$(
impl CachedProjection for $ty {
type ProjectedType = Self;
type ProjectedTypeRef<'a> = &'a Self;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
self
}
fn into_projected(self) -> Self::ProjectedType {
self
}
fn from_projected(projected: Self::ProjectedType) -> Self {
projected
}
}
)*
};
}
identity_projection!(
(),
bool,
char,
String,
i8,
i16,
i32,
i64,
i128,
isize,
u8,
u16,
u32,
u64,
u128,
usize,
f32,
f64,
);
impl<T> CachedProjection for Option<T>
where
T: CachedProjection,
{
type ProjectedType = Option<T::ProjectedType>;
type ProjectedTypeRef<'a>
= Option<T::ProjectedTypeRef<'a>>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
self.as_ref().map(CachedProjection::project_ref)
}
fn into_projected(self) -> Self::ProjectedType {
self.map(CachedProjection::into_projected)
}
fn from_projected(projected: Self::ProjectedType) -> Self {
projected.map(CachedProjection::from_projected)
}
}
pub struct ProjectedSliceRef<'a, T>(&'a [T]);
impl<T> Serialize for ProjectedSliceRef<'_, T>
where
T: CachedProjection,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut sequence = serializer.serialize_seq(Some(self.0.len()))?;
for value in self.0 {
sequence.serialize_element(&value.project_ref())?;
}
sequence.end()
}
}
impl<T> CachedProjection for Vec<T>
where
T: CachedProjection,
{
type ProjectedType = Vec<T::ProjectedType>;
type ProjectedTypeRef<'a>
= ProjectedSliceRef<'a, T>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
ProjectedSliceRef(self)
}
fn into_projected(self) -> Self::ProjectedType {
self.into_iter()
.map(CachedProjection::into_projected)
.collect()
}
fn from_projected(projected: Self::ProjectedType) -> Self {
projected
.into_iter()
.map(CachedProjection::from_projected)
.collect()
}
}
impl<T> CachedProjection for Box<T>
where
T: CachedProjection,
{
type ProjectedType = T::ProjectedType;
type ProjectedTypeRef<'a>
= T::ProjectedTypeRef<'a>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
self.as_ref().project_ref()
}
fn into_projected(self) -> Self::ProjectedType {
(*self).into_projected()
}
fn from_projected(projected: Self::ProjectedType) -> Self {
Box::new(T::from_projected(projected))
}
}
impl<T> CachedProjection for Arc<T>
where
T: CachedProjection + Clone,
{
type ProjectedType = T::ProjectedType;
type ProjectedTypeRef<'a>
= T::ProjectedTypeRef<'a>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
self.as_ref().project_ref()
}
fn into_projected(self) -> Self::ProjectedType {
Arc::unwrap_or_clone(self).into_projected()
}
fn from_projected(projected: Self::ProjectedType) -> Self {
Arc::new(T::from_projected(projected))
}
}
pub struct ProjectedHashMapRef<'a, K, V>(&'a HashMap<K, V>);
impl<K, V> Serialize for ProjectedHashMapRef<'_, K, V>
where
K: Serialize,
V: CachedProjection,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = serializer.serialize_map(Some(self.0.len()))?;
for (key, value) in self.0 {
map.serialize_entry(key, &value.project_ref())?;
}
map.end()
}
}
impl<K, V> CachedProjection for HashMap<K, V>
where
K: Serialize + DeserializeOwned + Eq + Hash,
V: CachedProjection,
{
type ProjectedType = HashMap<K, V::ProjectedType>;
type ProjectedTypeRef<'a>
= ProjectedHashMapRef<'a, K, V>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
ProjectedHashMapRef(self)
}
fn into_projected(self) -> Self::ProjectedType {
self.into_iter()
.map(|(key, value)| (key, value.into_projected()))
.collect()
}
fn from_projected(projected: Self::ProjectedType) -> Self {
projected
.into_iter()
.map(|(key, value)| (key, V::from_projected(value)))
.collect()
}
}
pub struct ProjectedBTreeMapRef<'a, K, V>(&'a BTreeMap<K, V>);
impl<K, V> Serialize for ProjectedBTreeMapRef<'_, K, V>
where
K: Serialize,
V: CachedProjection,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = serializer.serialize_map(Some(self.0.len()))?;
for (key, value) in self.0 {
map.serialize_entry(key, &value.project_ref())?;
}
map.end()
}
}
impl<K, V> CachedProjection for BTreeMap<K, V>
where
K: Serialize + DeserializeOwned + Ord,
V: CachedProjection,
{
type ProjectedType = BTreeMap<K, V::ProjectedType>;
type ProjectedTypeRef<'a>
= ProjectedBTreeMapRef<'a, K, V>
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
ProjectedBTreeMapRef(self)
}
fn into_projected(self) -> Self::ProjectedType {
self.into_iter()
.map(|(key, value)| (key, value.into_projected()))
.collect()
}
fn from_projected(projected: Self::ProjectedType) -> Self {
projected
.into_iter()
.map(|(key, value)| (key, V::from_projected(value)))
.collect()
}
}
impl<A, B> CachedProjection for (A, B)
where
A: CachedProjection,
B: CachedProjection,
{
type ProjectedType = (A::ProjectedType, B::ProjectedType);
type ProjectedTypeRef<'a>
= (A::ProjectedTypeRef<'a>, B::ProjectedTypeRef<'a>)
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
(self.0.project_ref(), self.1.project_ref())
}
fn into_projected(self) -> Self::ProjectedType {
(self.0.into_projected(), self.1.into_projected())
}
fn from_projected(projected: Self::ProjectedType) -> Self {
(
A::from_projected(projected.0),
B::from_projected(projected.1),
)
}
}
impl<A, B, C> CachedProjection for (A, B, C)
where
A: CachedProjection,
B: CachedProjection,
C: CachedProjection,
{
type ProjectedType = (A::ProjectedType, B::ProjectedType, C::ProjectedType);
type ProjectedTypeRef<'a>
= (
A::ProjectedTypeRef<'a>,
B::ProjectedTypeRef<'a>,
C::ProjectedTypeRef<'a>,
)
where
Self: 'a;
fn project_ref(&self) -> Self::ProjectedTypeRef<'_> {
(
self.0.project_ref(),
self.1.project_ref(),
self.2.project_ref(),
)
}
fn into_projected(self) -> Self::ProjectedType {
(
self.0.into_projected(),
self.1.into_projected(),
self.2.into_projected(),
)
}
fn from_projected(projected: Self::ProjectedType) -> Self {
(
A::from_projected(projected.0),
B::from_projected(projected.1),
C::from_projected(projected.2),
)
}
}
#[cfg(test)]
mod tests {
use super::CachedProjection;
use serde::{Deserialize, Serialize};
#[derive(CachedProjection, Serialize, Deserialize)]
struct Child {
value: String,
}
#[derive(CachedProjection, Serialize, Deserialize)]
struct Parent {
#[cached_projection(nested)]
child: Child,
#[cached_projection(wrap)]
metadata: serde_json::Value,
}
#[test]
fn postcard_projection_round_trip() {
let value = Parent {
child: Child {
value: "value".to_owned(),
},
metadata: serde_json::json!({ "key": [1, 2, 3] }),
};
let bytes = postcard::to_allocvec(&value.project_ref()).unwrap();
let projected =
postcard::from_bytes::<ParentProjectedType>(&bytes).unwrap();
let _value = Parent::from_projected(projected);
}
}