sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-08-26 17:44:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

redis compression (#6769)

Browse Source 
* feat(labrinth): encode/decode redis values with postcard, add version in namespaces

* chore(labrinth): split cache ex special cases without version

* chore(labrinth): simplify get_many

* fix: tombi

* feat(labrinth): compression support

* chore(labrinth): cleanup redis ser/de

* fix(labrinth): use get_deserialized in tests
This commit is contained in:
François-Xavier Talbot
2026-07-19 01:25:00 +02:00
committed by GitHub
parent 6b62e3b480
commit 6b569e4ee4
40 changed files with 361 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/labrinth/src/database/models/analytics_event_item.rs
+3 -3
View File
@@ -11,7 +11,7 @@ use crate::{
};
use serde::{Deserialize, Serialize};
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events";
const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v1";
const ANALYTICS_EVENTS_ALL_KEY: &str = "all";
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -88,7 +88,7 @@ impl DBAnalyticsEvent {
let mut redis = redis.connect().await?;
if let Some(events) = redis
.get_deserialized_from_json(
.get_deserialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
)
@@ -119,7 +119,7 @@ impl DBAnalyticsEvent {
.await?;
redis
.set_serialized_to_json(
.set_serialized(
ANALYTICS_EVENTS_NAMESPACE,
ANALYTICS_EVENTS_ALL_KEY,
&events,
apps/labrinth/src/database/models/categories.rs
+10 -26
View File
@@ -7,7 +7,7 @@ use super::ids::*;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const TAGS_NAMESPACE: &str = "tags";
const TAGS_NAMESPACE: &str = "tags:v1";
pub struct ProjectType {
pub id: ProjectTypeId,
@@ -96,9 +96,8 @@ impl Category {
{
let mut redis = redis.connect().await?;
let res: Option<Vec<Category>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "category")
.await?;
let res: Option<Vec<Category>> =
redis.get_deserialized(TAGS_NAMESPACE, "category").await?;
if let Some(res) = res {
return Ok(res);
@@ -127,7 +126,7 @@ impl Category {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "category", &result, None)
.set_serialized(TAGS_NAMESPACE, "category", &result, None)
.await?;
Ok(result)
@@ -166,7 +165,7 @@ impl LinkPlatform {
let mut redis = redis.connect().await?;
let res: Option<Vec<LinkPlatform>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "link_platform")
.get_deserialized(TAGS_NAMESPACE, "link_platform")
.await?;
if let Some(res) = res {
@@ -191,12 +190,7 @@ impl LinkPlatform {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"link_platform",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "link_platform", &result, None)
.await?;
Ok(result)
@@ -235,7 +229,7 @@ impl ReportType {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "report_type")
.get_deserialized(TAGS_NAMESPACE, "report_type")
.await?;
if let Some(res) = res {
@@ -256,12 +250,7 @@ impl ReportType {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"report_type",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "report_type", &result, None)
.await?;
Ok(result)
@@ -300,7 +289,7 @@ impl ProjectType {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "project_type")
.get_deserialized(TAGS_NAMESPACE, "project_type")
.await?;
if let Some(res) = res {
@@ -321,12 +310,7 @@ impl ProjectType {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
"project_type",
&result,
None,
)
.set_serialized(TAGS_NAMESPACE, "project_type", &result, None)
.await?;
Ok(result)
apps/labrinth/src/database/models/collection_item.rs
+1 -1
View File
@@ -8,7 +8,7 @@ use dashmap::DashMap;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const COLLECTIONS_NAMESPACE: &str = "collections";
const COLLECTIONS_NAMESPACE: &str = "collections:v1";
#[derive(Clone)]
pub struct CollectionBuilder {
apps/labrinth/src/database/models/flow_item.rs
+3 -4
View File
@@ -13,10 +13,9 @@ use serde::{Deserialize, Serialize};
use url::Url;
use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration};
const FLOWS_NAMESPACE: &str = "flows";
const FLOWS_NAMESPACE: &str = "flows:v1";
#[derive(Deserialize, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum DBFlow {
OAuth {
user_id: Option<DBUserId>,
@@ -78,7 +77,7 @@ impl DBFlow {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
FLOWS_NAMESPACE,
&state,
&self,
@@ -109,7 +108,7 @@ impl DBFlow {
) -> Result<Option<DBFlow>, DatabaseError> {
let mut redis = redis.connect().await?;
redis.get_deserialized_from_json(FLOWS_NAMESPACE, id).await
redis.get_deserialized(FLOWS_NAMESPACE, id).await
}
/// Gets the flow and removes it from the cache, but only removes if the flow was present and the predicate returned true
apps/labrinth/src/database/models/image_item.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use chrono::{DateTime, Utc};
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
const IMAGES_NAMESPACE: &str = "images";
const IMAGES_NAMESPACE: &str = "images:v1";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DBImage {
apps/labrinth/src/database/models/loader_fields.rs
+20 -38
View File
@@ -12,13 +12,14 @@ use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
const GAMES_LIST_NAMESPACE: &str = "games";
const LOADER_ID: &str = "loader_id";
const LOADERS_LIST_NAMESPACE: &str = "loaders";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values";
const GAMES_LIST_NAMESPACE: &str = "games:v1";
const LOADER_ID: &str = "loader_id:v1";
const LOADERS_LIST_NAMESPACE: &str = "loaders:v1";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v1";
const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v1";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v1";
pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str =
"loader_field_enum_values:v1";
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct Game {
@@ -54,7 +55,7 @@ impl Game {
{
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis
.get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games")
.get_deserialized(GAMES_LIST_NAMESPACE, "games")
.await?;
if let Some(cached_games) = cached_games {
return Ok(cached_games);
@@ -80,12 +81,7 @@ impl Game {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
GAMES_LIST_NAMESPACE,
"games",
&result,
None,
)
.set_serialized(GAMES_LIST_NAMESPACE, "games", &result, None)
.await?;
Ok(result)
@@ -114,7 +110,7 @@ impl Loader {
{
let mut redis = redis.connect().await?;
let cached_id: Option<i32> =
redis.get_deserialized_from_json(LOADER_ID, name).await?;
redis.get_deserialized(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
}
@@ -134,7 +130,7 @@ impl Loader {
if let Some(result) = result {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(LOADER_ID, name, &result.0, None)
.set_serialized(LOADER_ID, name, &result.0, None)
.await?;
}
@@ -151,7 +147,7 @@ impl Loader {
{
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.get_deserialized(LOADERS_LIST_NAMESPACE, "all")
.await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
@@ -193,12 +189,7 @@ impl Loader {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADERS_LIST_NAMESPACE,
"all",
&result,
None,
)
.set_serialized(LOADERS_LIST_NAMESPACE, "all", &result, None)
.await?;
Ok(result)
@@ -470,10 +461,9 @@ impl LoaderField {
{
let mut redis = redis.connect().await?;
let cached_fields: Option<Vec<LoaderField>> =
redis.get(LOADER_FIELDS_NAMESPACE_ALL, "").await?.and_then(
|x| serde_json::from_str::<Vec<LoaderField>>(&x).ok(),
);
let cached_fields: Option<Vec<LoaderField>> = redis
.get_deserialized(LOADER_FIELDS_NAMESPACE_ALL, "")
.await?;
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
@@ -506,12 +496,7 @@ impl LoaderField {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADER_FIELDS_NAMESPACE_ALL,
"",
&result,
None,
)
.set_serialized(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None)
.await?;
Ok(result)
@@ -530,10 +515,7 @@ impl LoaderFieldEnum {
let mut redis = redis.connect().await?;
let cached_enum = redis
.get_deserialized_from_json(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
)
.get_deserialized(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name)
.await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
@@ -561,7 +543,7 @@ impl LoaderFieldEnum {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
&result,
apps/labrinth/src/database/models/mod.rs
+2
View File
@@ -76,6 +76,8 @@ pub enum DatabaseError {
RedisPool(#[from] deadpool_redis::PoolError),
#[error("Error while serializing with the cache: {0}")]
SerdeCacheError(#[from] serde_json::Error),
#[error("error while encoding or decoding the cache: {0}")]
PostcardCacheError(#[from] postcard::Error),
#[error("Schema error: {0}")]
SchemaError(String),
#[error(
apps/labrinth/src/database/models/moderation_note_item.rs
+6 -6
View File
@@ -7,9 +7,9 @@ use crate::database::redis::RedisPool;
use super::{DBOrganizationId, DBUserId, DatabaseError};
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users";
const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v1";
const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str =
"moderation_notes_organizations";
"moderation_notes_organizations:v1";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DBModerationNote {
@@ -40,7 +40,7 @@ impl DBModerationNote {
let cached = {
let mut redis = redis.connect().await?;
redis
.get_many_deserialized_from_json::<Self>(
.get_many_deserialized::<Self>(
MODERATION_NOTES_USERS_NAMESPACE,
&ids,
)
@@ -87,7 +87,7 @@ impl DBModerationNote {
if let Some(user_id) = note.user_id {
redis
.set_serialized_to_json(
.set_serialized(
MODERATION_NOTES_USERS_NAMESPACE,
user_id.0,
&note,
@@ -130,7 +130,7 @@ impl DBModerationNote {
let cached = {
let mut redis = redis.connect().await?;
redis
.get_many_deserialized_from_json::<Self>(
.get_many_deserialized::<Self>(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
&ids,
)
@@ -177,7 +177,7 @@ impl DBModerationNote {
if let Some(organization_id) = note.organization_id {
redis
.set_serialized_to_json(
.set_serialized(
MODERATION_NOTES_ORGANIZATIONS_NAMESPACE,
organization_id.0,
&note,
apps/labrinth/src/database/models/notification_item.rs
+3 -3
View File
@@ -9,7 +9,7 @@ use chrono::{DateTime, Utc};
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications";
const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v1";
pub struct NotificationBuilder {
pub body: NotificationBody,
@@ -435,7 +435,7 @@ impl DBNotification {
let mut redis = redis.connect().await?;
let cached_notifications: Option<Vec<DBNotification>> = redis
.get_deserialized_from_json(
.get_deserialized(
USER_NOTIFICATIONS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -493,7 +493,7 @@ impl DBNotification {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
USER_NOTIFICATIONS_NAMESPACE,
user_id.0,
&db_notifications,
apps/labrinth/src/database/models/notifications_template_item.rs
+10 -15
View File
@@ -4,10 +4,11 @@ use crate::models::v3::notifications::{NotificationChannel, NotificationType};
use crate::routes::ApiError;
use serde::{Deserialize, Serialize};
const TEMPLATES_NAMESPACE: &str = "notifications_templates";
const TEMPLATES_HTML_DATA_NAMESPACE: &str = "notifications_templates_html_data";
const TEMPLATES_NAMESPACE: &str = "notifications_templates:v1";
const TEMPLATES_HTML_DATA_NAMESPACE: &str =
"notifications_templates_html_data:v1";
const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str =
"notifications_templates_dynamic_html";
"notifications_templates_dynamic_html:v1";
const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes
const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes
@@ -56,10 +57,7 @@ impl NotificationTemplate {
let mut redis = redis.connect().await?;
let maybe_cached_templates = redis
.get_deserialized_from_json(
TEMPLATES_NAMESPACE,
channel.as_str(),
)
.get_deserialized(TEMPLATES_NAMESPACE, channel.as_str())
.await?;
if let Some(cached) = maybe_cached_templates {
@@ -82,7 +80,7 @@ impl NotificationTemplate {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
TEMPLATES_NAMESPACE,
channel.as_str(),
&templates,
@@ -99,7 +97,7 @@ impl NotificationTemplate {
) -> Result<Option<String>, DatabaseError> {
let mut redis = redis.connect().await?;
redis
.get_deserialized_from_json(
.get_deserialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
)
@@ -113,7 +111,7 @@ impl NotificationTemplate {
) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
TEMPLATES_HTML_DATA_NAMESPACE,
&self.id.to_string(),
&data,
@@ -138,10 +136,7 @@ where
let mut redis_conn = redis.connect().await?;
if let Some(body) = redis_conn
.get_deserialized_from_json::<HtmlBody>(
TEMPLATES_DYNAMIC_HTML_NAMESPACE,
key,
)
.get_deserialized::<HtmlBody>(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key)
.await?
{
return Ok(body.html);
@@ -153,7 +148,7 @@ where
let mut redis_conn = redis.connect().await?;
redis_conn
.set_serialized_to_json(
.set_serialized(
TEMPLATES_DYNAMIC_HTML_NAMESPACE,
key,
&cached,
apps/labrinth/src/database/models/notifications_type_item.rs
+3 -8
View File
@@ -3,7 +3,7 @@ use crate::database::redis::RedisPool;
use crate::models::v3::notifications::NotificationType;
use serde::{Deserialize, Serialize};
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types";
const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v1";
#[derive(Serialize, Deserialize)]
pub struct NotificationTypeItem {
@@ -43,7 +43,7 @@ impl NotificationTypeItem {
let mut redis = redis.connect().await?;
let cached_types = redis
.get_deserialized_from_json(NOTIFICATION_TYPES_NAMESPACE, "all")
.get_deserialized(NOTIFICATION_TYPES_NAMESPACE, "all")
.await?;
if let Some(types) = cached_types {
@@ -63,12 +63,7 @@ impl NotificationTypeItem {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
NOTIFICATION_TYPES_NAMESPACE,
"all",
&types,
None,
)
.set_serialized(NOTIFICATION_TYPES_NAMESPACE, "all", &types, None)
.await?;
Ok(types)
apps/labrinth/src/database/models/organization_item.rs
+2 -2
View File
@@ -9,8 +9,8 @@ use std::hash::Hash;
use super::{DBTeamMember, ids::*};
use serde::{Deserialize, Serialize};
const ORGANIZATIONS_NAMESPACE: &str = "organizations";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles";
const ORGANIZATIONS_NAMESPACE: &str = "organizations:v1";
const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
/// An organization of users who together control one or more projects and organizations.
apps/labrinth/src/database/models/pat_item.rs
+5 -10
View File
@@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const PATS_NAMESPACE: &str = "pats";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens";
const PATS_USERS_NAMESPACE: &str = "pats_users";
const PATS_NAMESPACE: &str = "pats:v1";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v1";
const PATS_USERS_NAMESPACE: &str = "pats_users:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBPersonalAccessToken {
@@ -161,7 +161,7 @@ impl DBPersonalAccessToken {
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
.get_deserialized::<Vec<i64>>(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -189,12 +189,7 @@ impl DBPersonalAccessToken {
let mut redis = redis.connect().await?;
redis
.set(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
&serde_json::to_string(&db_pats)?,
None,
)
.set_serialized(PATS_USERS_NAMESPACE, user_id.0, &db_pats, None)
.await?;
Ok(db_pats)
}
apps/labrinth/src/database/models/product_item.rs
+4 -5
View File
@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
use std::convert::TryFrom;
use std::convert::TryInto;
const PRODUCTS_NAMESPACE: &str = "products";
const PRODUCTS_NAMESPACE: &str = "products:v1";
pub struct DBProduct {
pub id: DBProductId,
@@ -153,9 +153,8 @@ impl QueryProductWithPrices {
{
let mut redis = redis.connect().await?;
let res: Option<Vec<QueryProductWithPrices>> = redis
.get_deserialized_from_json(PRODUCTS_NAMESPACE, "all")
.await?;
let res: Option<Vec<QueryProductWithPrices>> =
redis.get_deserialized(PRODUCTS_NAMESPACE, "all").await?;
if let Some(res) = res {
return Ok(res);
@@ -196,7 +195,7 @@ impl QueryProductWithPrices {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(PRODUCTS_NAMESPACE, "all", &products, None)
.set_serialized(PRODUCTS_NAMESPACE, "all", &products, None)
.await?;
Ok(products)
apps/labrinth/src/database/models/project_item.rs
+5 -5
View File
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
pub const PROJECTS_NAMESPACE: &str = "projects";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies";
pub const PROJECTS_NAMESPACE: &str = "projects:v1";
pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v1";
const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v1";
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LinkUrl {
@@ -976,7 +976,7 @@ impl DBProject {
let mut redis = redis.connect().await?;
let dependencies = redis
.get_deserialized_from_json::<Dependencies>(
.get_deserialized::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
@@ -1014,7 +1014,7 @@ impl DBProject {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
PROJECTS_DEPENDENCIES_NAMESPACE,
id.0,
&dependencies,
apps/labrinth/src/database/models/session_item.rs
+5 -5
View File
@@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const SESSIONS_NAMESPACE: &str = "sessions";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users";
const SESSIONS_NAMESPACE: &str = "sessions:v1";
const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v1";
const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v1";
pub struct SessionBuilder {
pub session: String,
@@ -226,7 +226,7 @@ impl DBSession {
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
.get_deserialized::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -255,7 +255,7 @@ impl DBSession {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
SESSIONS_USERS_NAMESPACE,
user_id.0,
&db_sessions,
apps/labrinth/src/database/models/team_item.rs
+1 -1
View File
@@ -9,7 +9,7 @@ use itertools::Itertools;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
const TEAMS_NAMESPACE: &str = "teams";
const TEAMS_NAMESPACE: &str = "teams:v1";
pub struct TeamBuilder {
pub members: Vec<TeamMemberBuilder>,
apps/labrinth/src/database/models/user_item.rs
+5 -5
View File
@@ -16,9 +16,9 @@ use serde::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::hash::Hash;
const USERS_NAMESPACE: &str = "users";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects";
const USERS_NAMESPACE: &str = "users:v1";
const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v1";
const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v1";
#[derive(Deserialize, Serialize, Clone, Debug)]
pub struct DBUser {
@@ -391,7 +391,7 @@ impl DBUser {
let mut redis = redis.connect().await?;
let cached_projects = redis
.get_deserialized_from_json::<Vec<DBProjectId>>(
.get_deserialized::<Vec<DBProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
@@ -419,7 +419,7 @@ impl DBUser {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
.set_serialized(
USERS_PROJECTS_NAMESPACE,
user_id.0,
&db_projects,
apps/labrinth/src/database/models/version_item.rs
+2 -2
View File
@@ -22,8 +22,8 @@ use std::collections::HashMap;
use std::iter;
use tracing::error;
pub const VERSIONS_NAMESPACE: &str = "versions";
const VERSION_FILES_NAMESPACE: &str = "versions_files";
pub const VERSIONS_NAMESPACE: &str = "versions:v1";
const VERSION_FILES_NAMESPACE: &str = "versions_files:v1";
pub async fn cleanup_unused_attribution_files_and_groups(
transaction: &mut PgTransaction<'_>,
apps/labrinth/src/database/redis/mod.rs
+145 -37
View File
@@ -12,12 +12,15 @@ use prometheus::{IntGauge, Registry};
use redis::ToRedisArgs;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::{Debug, Display};
use std::future::Future;
use std::hash::Hash;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
use tracing::{Instrument, info, info_span};
use util::{cmd, redis_pipe};
@@ -41,8 +44,112 @@ const MGET_CHUNK_SIZE: usize = 32;
// BytesMut peak capacity that builds up under steady load.
const REDIS_MAX_CONN_AGE: Duration = Duration::from_secs(120);
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Codec {
Raw = 0,
Lz4 = 1,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncodingFormat {
Json,
Postcard,
}
#[derive(Debug, Error)]
#[error("invalid redis codec")]
pub struct InvalidCodec;
#[derive(Debug, Error)]
#[error("invalid redis encoding format")]
pub struct InvalidEncodingFormat;
impl TryFrom<u8> for Codec {
type Error = InvalidCodec;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(Self::Raw),
1 => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for Codec {
type Err = InvalidCodec;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"lz4" => Ok(Self::Lz4),
_ => Err(InvalidCodec),
}
}
}
impl FromStr for EncodingFormat {
type Err = InvalidEncodingFormat;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"json" => Ok(Self::Json),
"postcard" => Ok(Self::Postcard),
_ => Err(InvalidEncodingFormat),
}
}
}
fn encode_value<T: Serialize>(value: &T) -> Result<Vec<u8>, DatabaseError> {
let mut value = match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::to_vec(value)?,
EncodingFormat::Postcard => postcard::to_allocvec(value)?,
};
if ENV.REDIS_COMPRESSION_LEVEL > 0
&& ENV.REDIS_COMPRESSION_ALGORITHM == Codec::Lz4
&& value.len() >= ENV.REDIS_COMPRESSION_THRESHOLD_BYTES
{
let compressed = lz4_flex::block::compress_prepend_size(&value);
let savings_ratio = value.len().saturating_sub(compressed.len()) as f64
/ value.len().max(1) as f64
* 100.0;
if savings_ratio >= ENV.REDIS_COMPRESSION_MIN_SAVINGS_RATIO {
let mut encoded = Vec::with_capacity(compressed.len() + 1);
encoded.push(Codec::Lz4 as u8);
encoded.extend(compressed);
return Ok(encoded);
}
}
let mut encoded = Vec::with_capacity(value.len() + 1);
encoded.push(Codec::Raw as u8);
encoded.append(&mut value);
Ok(encoded)
}
fn decode_value<T>(value: &[u8]) -> Option<T>
where
T: for<'a> Deserialize<'a>,
{
let (codec, value) = value.split_first()?;
let value = match Codec::try_from(*codec).ok()? {
Codec::Raw => Cow::Borrowed(value),
Codec::Lz4 => {
Cow::Owned(lz4_flex::block::decompress_size_prepended(value).ok()?)
}
};
match ENV.REDIS_ENCODING_FORMAT {
EncodingFormat::Json => serde_json::from_slice(&value).ok(),
EncodingFormat::Postcard => postcard::from_bytes(&value).ok(),
}
}
fn cache_expiries(namespace: &str) -> (i64, i64) {
match namespace {
// Namespaces may embed a version suffix like `:v1`, so split it out.
match namespace.split_once(':').map(|t| t.0).unwrap_or(namespace) {
"versions" | "versions_files" => {
(VERSION_DEFAULT_EXPIRY, VERSION_ACTUAL_EXPIRY)
}
@@ -367,12 +474,11 @@ impl RedisPool {
for chunk in args.chunks(MGET_CHUNK_SIZE) {
let part = cmd("MGET")
.arg(chunk)
.query_async::<Vec<Option<String>>>(&mut connection)
.query_async::<Vec<Option<Vec<u8>>>>(&mut connection)
.await?;
cached_values.extend(part.into_iter().filter_map(|x| {
x.and_then(|val| {
serde_json::from_str::<RedisValue<T, K, S>>(&val)
.ok()
decode_value::<RedisValue<T, K, S>>(&val)
})
.map(|val| (val.key.clone(), val))
}));
@@ -492,7 +598,7 @@ impl RedisPool {
"{}_{namespace}:{key}",
self.meta_namespace
),
serde_json::to_string(&value)?,
encode_value(&value)?,
default_expiry as u64,
);
pipe_cmds += 1;
@@ -624,30 +730,27 @@ impl<'a> Drop for LockSentinel<'a> {
impl RedisConnection {
#[tracing::instrument(skip(self))]
pub async fn set(
pub async fn set<D>(
&mut self,
namespace: &str,
id: &str,
data: &str,
data: D,
expiry: Option<i64>,
) -> Result<(), DatabaseError> {
) -> Result<(), DatabaseError>
where
D: ToRedisArgs + Send + Sync + Debug,
{
let mut cmd = cmd("SET");
redis_args(
&mut cmd,
vec![
format!("{}_{}:{}", self.meta_namespace, namespace, id),
data.to_string(),
"EX".to_string(),
expiry.unwrap_or(DEFAULT_EXPIRY).to_string(),
]
.as_slice(),
);
cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id))
.arg(data)
.arg("EX")
.arg(expiry.unwrap_or(DEFAULT_EXPIRY));
redis_execute::<()>(&mut cmd, &mut self.connection).await?;
Ok(())
}
#[tracing::instrument(skip(self, id, data))]
pub async fn set_serialized_to_json<Id, D>(
pub async fn set_serialized<Id, D>(
&mut self,
namespace: &str,
id: Id,
@@ -658,13 +761,8 @@ impl RedisConnection {
Id: Display,
D: serde::Serialize,
{
self.set(
namespace,
&id.to_string(),
&serde_json::to_string(&data)?,
expiry,
)
.await
self.set(namespace, &id.to_string(), encode_value(&data)?, expiry)
.await
}
#[tracing::instrument(skip(self))]
@@ -688,7 +786,7 @@ impl RedisConnection {
&mut self,
namespace: &str,
ids: &[String],
) -> Result<Vec<Option<String>>, DatabaseError> {
) -> Result<Vec<Option<Vec<u8>>>, DatabaseError> {
let mut cmd = cmd("MGET");
redis_args(
&mut cmd,
@@ -702,7 +800,7 @@ impl RedisConnection {
}
#[tracing::instrument(skip(self))]
pub async fn get_deserialized_from_json<R>(
pub async fn get_deserialized<R>(
&mut self,
namespace: &str,
id: &str,
@@ -710,14 +808,19 @@ impl RedisConnection {
where
R: for<'a> serde::Deserialize<'a>,
{
Ok(self
.get(namespace, id)
.await?
.and_then(|x| serde_json::from_str(&x).ok()))
let mut cmd = cmd("GET");
redis_args(
&mut cmd,
vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)]
.as_slice(),
);
let value: Option<Vec<u8>> =
redis_execute(&mut cmd, &mut self.connection).await?;
Ok(value.and_then(|value| decode_value(&value)))
}
#[tracing::instrument(skip(self))]
pub async fn get_many_deserialized_from_json<R>(
pub async fn get_many_deserialized<R>(
&mut self,
namespace: &str,
ids: &[String],
@@ -729,8 +832,8 @@ impl RedisConnection {
.get_many(namespace, ids)
.await?
.into_iter()
.map(|x| x.and_then(|val| serde_json::from_str::<R>(&val).ok()))
.collect::<Vec<_>>())
.map(|value| value.and_then(|value| decode_value::<R>(&value)))
.collect())
}
#[tracing::instrument(skip(self, id))]
@@ -799,7 +902,7 @@ impl RedisConnection {
namespace: &str,
key: &str,
timeout: Option<f64>,
) -> Result<Option<[String; 2]>, DatabaseError> {
) -> Result<Option<[Vec<u8>; 2]>, DatabaseError> {
let key = format!("{}_{namespace}:{key}", self.meta_namespace);
// a timeout of 0 is infinite
let timeout = timeout.unwrap_or(0.0);
@@ -829,12 +932,17 @@ impl RedisConnection {
#[derive(Serialize, Deserialize)]
pub struct RedisValue<T, K, S> {
key: K,
#[serde(skip_serializing_if = "Option::is_none")]
alias: Option<S>,
iat: i64,
val: T,
}
impl<T, K, S> RedisValue<T, K, S> {
pub fn value(&self) -> &T {
&self.val
}
}
pub fn redis_args(cmd: &mut util::InstrumentedCmd, args: &[String]) {
for arg in args {
cmd.arg(arg);