mirror of
https://github.com/modrinth/code.git
synced 2026-08-27 18:14:49 +00:00
redo error handling in xredis
This commit is contained in:
Generated
+2
-1
@@ -13443,12 +13443,13 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"dashmap",
|
"dashmap",
|
||||||
"deadpool-redis",
|
"deadpool-redis",
|
||||||
|
"eyre",
|
||||||
"futures",
|
"futures",
|
||||||
"lz4_flex",
|
"lz4_flex",
|
||||||
|
"postcard",
|
||||||
"prometheus",
|
"prometheus",
|
||||||
"redis",
|
"redis",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.17",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|||||||
@@ -12,8 +12,10 @@
|
|||||||
- no trailing punctuation
|
- no trailing punctuation
|
||||||
- wrap code items e.g. type names in backticks
|
- wrap code items e.g. type names in backticks
|
||||||
- Prefer `wrap_internal_err`, `wrap_request_err` when attaching context to an existing error (like Anyhow `context` or Eyre `wrap_err`)
|
- Prefer `wrap_internal_err`, `wrap_request_err` when attaching context to an existing error (like Anyhow `context` or Eyre `wrap_err`)
|
||||||
|
- Prefer importing `eyre::Result` and using `Result<T>` instead of `eyre::Result<T>`
|
||||||
|
- Prefer `eyre::Ok(value)` instead of `Ok::<_, eyre::Report>(value)` when an explicit Eyre result type is needed
|
||||||
- All operations should ideally have some context attached
|
- All operations should ideally have some context attached
|
||||||
- Database operations can have a message like `.wrap_internal_err("failed to fetch XYZ")`
|
- Database operations can have a message like `.wrap_internal_err("fetching XYZ")`
|
||||||
- You can perform real-time queries against the databases in the Docker Compose
|
- You can perform real-time queries against the databases in the Docker Compose
|
||||||
- `docker exec labrinth-postgres psql -c "select 1"`
|
- `docker exec labrinth-postgres psql -c "select 1"`
|
||||||
- `docker exec labrinth-redis redis-cli flushall`
|
- `docker exec labrinth-redis redis-cli flushall`
|
||||||
|
|||||||
@@ -59,12 +59,6 @@ pub enum AuthenticationError {
|
|||||||
Url,
|
Url,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<xredis::Error> for AuthenticationError {
|
|
||||||
fn from(error: xredis::Error) -> Self {
|
|
||||||
Self::Database(error.into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl actix_web::ResponseError for AuthenticationError {
|
impl actix_web::ResponseError for AuthenticationError {
|
||||||
fn status_code(&self) -> StatusCode {
|
fn status_code(&self) -> StatusCode {
|
||||||
match self {
|
match self {
|
||||||
|
|||||||
@@ -77,8 +77,6 @@ pub enum DatabaseError {
|
|||||||
SerdeCacheError(#[from] serde_json::Error),
|
SerdeCacheError(#[from] serde_json::Error),
|
||||||
#[error("error while encoding or decoding the cache: {0}")]
|
#[error("error while encoding or decoding the cache: {0}")]
|
||||||
PostcardCacheError(#[from] postcard::Error),
|
PostcardCacheError(#[from] postcard::Error),
|
||||||
#[error(transparent)]
|
|
||||||
Redis(#[from] xredis::Error),
|
|
||||||
#[error("Schema error: {0}")]
|
#[error("Schema error: {0}")]
|
||||||
SchemaError(String),
|
SchemaError(String),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use crate::database::models::DatabaseError;
|
use crate::database::models::DatabaseError;
|
||||||
use crate::models::v3::notifications::{NotificationChannel, NotificationType};
|
use crate::models::v3::notifications::{NotificationChannel, NotificationType};
|
||||||
use crate::routes::ApiError;
|
use crate::routes::ApiError;
|
||||||
|
use crate::util::error::Context;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use xredis::RedisPool;
|
use xredis::RedisPool;
|
||||||
|
|
||||||
@@ -123,12 +124,16 @@ where
|
|||||||
html: String,
|
html: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut redis_conn = redis.connect().await?;
|
let mut redis_conn = redis.connect().await.wrap_internal_err(
|
||||||
|
"connecting to redis for dynamic notification html",
|
||||||
|
)?;
|
||||||
let redis_key = redis_conn
|
let redis_key = redis_conn
|
||||||
.key()
|
.key()
|
||||||
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
|
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
|
||||||
if let Some(body) =
|
if let Some(body) = redis_conn
|
||||||
redis_conn.get_deserialized::<HtmlBody>(&redis_key).await?
|
.get_deserialized::<HtmlBody>(&redis_key)
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("fetching dynamic notification html from redis")?
|
||||||
{
|
{
|
||||||
return Ok(body.html);
|
return Ok(body.html);
|
||||||
}
|
}
|
||||||
@@ -136,14 +141,17 @@ where
|
|||||||
drop(redis_conn);
|
drop(redis_conn);
|
||||||
|
|
||||||
let cached = HtmlBody { html: get().await? };
|
let cached = HtmlBody { html: get().await? };
|
||||||
let mut redis_conn = redis.connect().await?;
|
let mut redis_conn = redis.connect().await.wrap_internal_err(
|
||||||
|
"connecting to redis for dynamic notification html",
|
||||||
|
)?;
|
||||||
let redis_key = redis_conn
|
let redis_key = redis_conn
|
||||||
.key()
|
.key()
|
||||||
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
|
.metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key);
|
||||||
|
|
||||||
redis_conn
|
redis_conn
|
||||||
.set_serialized(&redis_key, &cached, Some(HTML_DATA_CACHE_EXPIRY))
|
.set_serialized(&redis_key, &cached, Some(HTML_DATA_CACHE_EXPIRY))
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_internal_err("writing dynamic notification html to redis")?;
|
||||||
|
|
||||||
Ok(cached.html)
|
Ok(cached.html)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -946,7 +946,7 @@ impl DBProject {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.wrap_internal_err("failed to fetch cached projects")?;
|
.wrap_internal_err("fetching cached projects")?;
|
||||||
|
|
||||||
Ok(val)
|
Ok(val)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::env::ENV;
|
use crate::env::ENV;
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
|
|
||||||
struct RedisConfig {
|
struct RedisConfig {
|
||||||
inner: xredis::RedisConfig,
|
inner: xredis::RedisConfig,
|
||||||
@@ -8,7 +9,7 @@ struct RedisConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RedisConfig {
|
impl RedisConfig {
|
||||||
fn from_env() -> Result<Self, xredis::RedisConfigError> {
|
fn from_env() -> Result<Self> {
|
||||||
let inner = xredis::RedisConfig::new(
|
let inner = xredis::RedisConfig::new(
|
||||||
ENV.REDIS_TOPOLOGY,
|
ENV.REDIS_TOPOLOGY,
|
||||||
ENV.REDIS_CONNECTION_TYPE,
|
ENV.REDIS_CONNECTION_TYPE,
|
||||||
@@ -25,7 +26,8 @@ impl RedisConfig {
|
|||||||
(ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0),
|
(ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0),
|
||||||
ENV.REDIS_CACHE_LOCKING_STRATEGY,
|
ENV.REDIS_CACHE_LOCKING_STRATEGY,
|
||||||
ENV.REDIS_READ_REPLICA_STRATEGY,
|
ENV.REDIS_READ_REPLICA_STRATEGY,
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("loading Redis configuration from environment")?;
|
||||||
let cache_settings = xredis::CacheSettings {
|
let cache_settings = xredis::CacheSettings {
|
||||||
default_expiry: ENV.REDIS_DEFAULT_EXPIRY,
|
default_expiry: ENV.REDIS_DEFAULT_EXPIRY,
|
||||||
actual_expiry: ENV.REDIS_ACTUAL_EXPIRY,
|
actual_expiry: ENV.REDIS_ACTUAL_EXPIRY,
|
||||||
|
|||||||
@@ -165,7 +165,7 @@ vars! {
|
|||||||
REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 256u32;
|
REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 256u32;
|
||||||
|
|
||||||
// The encoding format used for Redis cache values.
|
// The encoding format used for Redis cache values.
|
||||||
REDIS_ENCODING_FORMAT: xredis::EncodingFormat = xredis::EncodingFormat::Json;
|
REDIS_ENCODING_FORMAT: xredis::EncodingFormat = xredis::EncodingFormat::Postcard;
|
||||||
// The level of LZ4 compression used for Redis cache values. A value of 0 disables compression (supports 1-12)
|
// The level of LZ4 compression used for Redis cache values. A value of 0 disables compression (supports 1-12)
|
||||||
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
|
REDIS_COMPRESSION_LEVEL: i32 = 0i32;
|
||||||
// The compression algorithm used for Redis cache values. Currently only LZ4 is supported.
|
// The compression algorithm used for Redis cache values. Currently only LZ4 is supported.
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use crate::models::analytics::{
|
|||||||
};
|
};
|
||||||
use crate::routes::ApiError;
|
use crate::routes::ApiError;
|
||||||
use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
|
use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
|
||||||
|
use crate::util::error::Context;
|
||||||
use dashmap::{DashMap, DashSet};
|
use dashmap::{DashMap, DashSet};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use tracing::trace;
|
use tracing::trace;
|
||||||
@@ -142,10 +143,15 @@ impl AnalyticsQueue {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut redis_connection = redis.connect().await?;
|
let mut redis_connection =
|
||||||
|
redis.connect().await.wrap_internal_err(
|
||||||
|
"connecting to redis for server play counts",
|
||||||
|
)?;
|
||||||
|
|
||||||
let results =
|
let results = redis_connection
|
||||||
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
|
.get_many_typed::<u32>(&redis_keys)
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("fetching server play counts from redis")?;
|
||||||
for (idx, count) in results.into_iter().enumerate() {
|
for (idx, count) in results.into_iter().enumerate() {
|
||||||
let new_count = if let Some(count) = count {
|
let new_count = if let Some(count) = count {
|
||||||
if count >= MINECRAFT_SERVER_PLAYS_LIMIT {
|
if count >= MINECRAFT_SERVER_PLAYS_LIMIT {
|
||||||
@@ -164,7 +170,8 @@ impl AnalyticsQueue {
|
|||||||
new_count,
|
new_count,
|
||||||
Some(MINECRAFT_SERVER_PLAYS_EXPIRY as i64),
|
Some(MINECRAFT_SERVER_PLAYS_EXPIRY as i64),
|
||||||
)
|
)
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_internal_err("writing server play count to redis")?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut plays = client
|
let mut plays = client
|
||||||
@@ -198,10 +205,15 @@ impl AnalyticsQueue {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut redis_connection = redis.connect().await?;
|
let mut redis_connection = redis
|
||||||
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("connecting to redis for view counts")?;
|
||||||
|
|
||||||
let results =
|
let results = redis_connection
|
||||||
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
|
.get_many_typed::<u32>(&redis_keys)
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("fetching view counts from redis")?;
|
||||||
for (idx, count) in results.into_iter().enumerate() {
|
for (idx, count) in results.into_iter().enumerate() {
|
||||||
let new_count =
|
let new_count =
|
||||||
if let Some((views, monetized)) = raw_views.get_mut(idx) {
|
if let Some((views, monetized)) = raw_views.get_mut(idx) {
|
||||||
@@ -226,7 +238,8 @@ impl AnalyticsQueue {
|
|||||||
let key = &redis_keys[idx];
|
let key = &redis_keys[idx];
|
||||||
redis_connection
|
redis_connection
|
||||||
.set(key, new_count, Some(6 * 60 * 60))
|
.set(key, new_count, Some(6 * 60 * 60))
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_internal_err("writing view count to redis")?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut views = client.insert::<PageView>("views").await?;
|
let mut views = client.insert::<PageView>("views").await?;
|
||||||
@@ -267,10 +280,15 @@ impl AnalyticsQueue {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut redis_connection = redis.connect().await?;
|
let mut redis_connection = redis
|
||||||
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("connecting to redis for download counts")?;
|
||||||
|
|
||||||
let results =
|
let results = redis_connection
|
||||||
redis_connection.get_many_typed::<u32>(&redis_keys).await?;
|
.get_many_typed::<u32>(&redis_keys)
|
||||||
|
.await
|
||||||
|
.wrap_internal_err("fetching download counts from redis")?;
|
||||||
for (idx, count) in results.into_iter().enumerate() {
|
for (idx, count) in results.into_iter().enumerate() {
|
||||||
let new_count = if let Some(count) = count {
|
let new_count = if let Some(count) = count {
|
||||||
if count > 5 {
|
if count > 5 {
|
||||||
@@ -286,7 +304,8 @@ impl AnalyticsQueue {
|
|||||||
let key = &redis_keys[idx];
|
let key = &redis_keys[idx];
|
||||||
redis_connection
|
redis_connection
|
||||||
.set(key, new_count, Some(6 * 60 * 60))
|
.set(key, new_count, Some(6 * 60 * 60))
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_internal_err("writing download count to redis")?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut transaction = pool.begin().await?;
|
let mut transaction = pool.begin().await?;
|
||||||
|
|||||||
@@ -267,12 +267,6 @@ pub enum ApiError {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<xredis::Error> for ApiError {
|
|
||||||
fn from(error: xredis::Error) -> Self {
|
|
||||||
Self::Database(error.into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ApiError {
|
impl ApiError {
|
||||||
pub fn delphi(err: impl Into<eyre::Error>) -> Self {
|
pub fn delphi(err: impl Into<eyre::Error>) -> Self {
|
||||||
Self::Delphi(err.into())
|
Self::Delphi(err.into())
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ ariadne = { workspace = true }
|
|||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
dashmap = { workspace = true }
|
dashmap = { workspace = true }
|
||||||
deadpool-redis = { workspace = true, features = ["cluster-async"] }
|
deadpool-redis = { workspace = true, features = ["cluster-async"] }
|
||||||
|
eyre = { workspace = true }
|
||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
lz4_flex = { workspace = true }
|
lz4_flex = { workspace = true }
|
||||||
prometheus = { workspace = true }
|
prometheus = { workspace = true }
|
||||||
@@ -20,7 +21,7 @@ redis = { workspace = true, features = [
|
|||||||
"tokio-comp"
|
"tokio-comp"
|
||||||
] }
|
] }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true }
|
postcard = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["rt", "sync", "time"] }
|
tokio = { workspace = true, features = ["rt", "sync", "time"] }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr, bail};
|
||||||
use prometheus::Registry;
|
use prometheus::Registry;
|
||||||
|
|
||||||
|
use super::RedisPool;
|
||||||
use super::config::{RedisConfig, RedisTopology};
|
use super::config::{RedisConfig, RedisTopology};
|
||||||
use super::connection::RedisBackendBuildError;
|
|
||||||
use super::metrics::{
|
use super::metrics::{
|
||||||
LogicalPoolStatus, LogicalPoolStatusProvider,
|
LogicalPoolStatus, LogicalPoolStatusProvider,
|
||||||
register_blocking_pool_metrics,
|
register_blocking_pool_metrics,
|
||||||
};
|
};
|
||||||
use super::{Error, RedisPool};
|
|
||||||
|
|
||||||
const POOL_RETAIN_INTERVAL: Duration = Duration::from_secs(30);
|
const POOL_RETAIN_INTERVAL: Duration = Duration::from_secs(30);
|
||||||
const MAX_IDLE_CONNECTION_AGE: Duration = Duration::from_secs(5 * 60);
|
const MAX_IDLE_CONNECTION_AGE: Duration = Duration::from_secs(5 * 60);
|
||||||
@@ -27,9 +27,7 @@ enum RedisBlockingPoolInner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RedisBlockingPool {
|
impl RedisBlockingPool {
|
||||||
pub(super) async fn new(
|
pub(super) async fn new(config: &RedisConfig) -> Result<Self> {
|
||||||
config: &RedisConfig,
|
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
|
||||||
let pool_size = config.blocking_pool_size();
|
let pool_size = config.blocking_pool_size();
|
||||||
let inner = match config.topology() {
|
let inner = match config.topology() {
|
||||||
RedisTopology::Standalone => {
|
RedisTopology::Standalone => {
|
||||||
@@ -39,14 +37,16 @@ impl RedisBlockingPool {
|
|||||||
let manager = deadpool_redis::Manager::new_with_config(
|
let manager = deadpool_redis::Manager::new_with_config(
|
||||||
config.seed_urls()[0].clone(),
|
config.seed_urls()[0].clone(),
|
||||||
connection_config,
|
connection_config,
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("configuring standalone blocking Redis client")?;
|
||||||
let pool = deadpool_redis::Pool::builder(manager)
|
let pool = deadpool_redis::Pool::builder(manager)
|
||||||
.max_size(pool_size.max())
|
.max_size(pool_size.max())
|
||||||
.wait_timeout(Some(Duration::from_millis(
|
.wait_timeout(Some(Duration::from_millis(
|
||||||
config.wait_timeout_ms(),
|
config.wait_timeout_ms(),
|
||||||
)))
|
)))
|
||||||
.runtime(deadpool_redis::Runtime::Tokio1)
|
.runtime(deadpool_redis::Runtime::Tokio1)
|
||||||
.build()?;
|
.build()
|
||||||
|
.wrap_err("building standalone blocking Redis pool")?;
|
||||||
retain_standalone_pool(pool.clone());
|
retain_standalone_pool(pool.clone());
|
||||||
RedisBlockingPoolInner::Standalone(pool)
|
RedisBlockingPoolInner::Standalone(pool)
|
||||||
}
|
}
|
||||||
@@ -54,14 +54,16 @@ impl RedisBlockingPool {
|
|||||||
let manager = deadpool_redis::cluster::Manager::new(
|
let manager = deadpool_redis::cluster::Manager::new(
|
||||||
config.seed_urls().to_vec(),
|
config.seed_urls().to_vec(),
|
||||||
false,
|
false,
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("configuring clustered blocking Redis client")?;
|
||||||
let pool = deadpool_redis::cluster::Pool::builder(manager)
|
let pool = deadpool_redis::cluster::Pool::builder(manager)
|
||||||
.max_size(pool_size.max())
|
.max_size(pool_size.max())
|
||||||
.wait_timeout(Some(Duration::from_millis(
|
.wait_timeout(Some(Duration::from_millis(
|
||||||
config.wait_timeout_ms(),
|
config.wait_timeout_ms(),
|
||||||
)))
|
)))
|
||||||
.runtime(deadpool_redis::Runtime::Tokio1)
|
.runtime(deadpool_redis::Runtime::Tokio1)
|
||||||
.build()?;
|
.build()
|
||||||
|
.wrap_err("building clustered blocking Redis pool")?;
|
||||||
retain_cluster_pool(pool.clone());
|
retain_cluster_pool(pool.clone());
|
||||||
RedisBlockingPoolInner::Cluster(pool)
|
RedisBlockingPoolInner::Cluster(pool)
|
||||||
}
|
}
|
||||||
@@ -70,10 +72,7 @@ impl RedisBlockingPool {
|
|||||||
Ok(Self { inner })
|
Ok(Self { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn register_metrics(
|
pub(super) fn register_metrics(&self, registry: &Registry) -> Result<()> {
|
||||||
&self,
|
|
||||||
registry: &Registry,
|
|
||||||
) -> Result<(), prometheus::Error> {
|
|
||||||
register_blocking_pool_metrics(registry, self.clone())
|
register_blocking_pool_metrics(registry, self.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,22 +80,33 @@ impl RedisBlockingPool {
|
|||||||
&self,
|
&self,
|
||||||
key: &str,
|
key: &str,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> Result<Option<[Vec<u8>; 2]>, Error> {
|
) -> Result<Option<[Vec<u8>; 2]>> {
|
||||||
if timeout.is_zero() {
|
if timeout.is_zero() {
|
||||||
return Err(Error::InvalidBlockingTimeout);
|
bail!("redis blocking timeout must be greater than zero");
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut command = redis::cmd("BRPOP");
|
let mut command = redis::cmd("BRPOP");
|
||||||
command.arg(key).arg(timeout.as_secs_f64());
|
command.arg(key).arg(timeout.as_secs_f64());
|
||||||
|
|
||||||
let response: Option<(Vec<u8>, Vec<u8>)> = match &self.inner {
|
let response: Option<(Vec<u8>, Vec<u8>)> =
|
||||||
RedisBlockingPoolInner::Standalone(pool) => {
|
match &self.inner {
|
||||||
command.query_async(&mut pool.get().await?).await?
|
RedisBlockingPoolInner::Standalone(pool) => {
|
||||||
}
|
let mut connection = pool.get().await.wrap_err(
|
||||||
RedisBlockingPoolInner::Cluster(pool) => {
|
"fetching standalone blocking Redis connection",
|
||||||
command.query_async(&mut pool.get().await?).await?
|
)?;
|
||||||
}
|
command.query_async(&mut connection).await.wrap_err(
|
||||||
};
|
"reading from standalone Redis blocking queue",
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
RedisBlockingPoolInner::Cluster(pool) => {
|
||||||
|
let mut connection = pool.get().await.wrap_err(
|
||||||
|
"fetching clustered blocking Redis connection",
|
||||||
|
)?;
|
||||||
|
command.query_async(&mut connection).await.wrap_err(
|
||||||
|
"reading from clustered Redis blocking queue",
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
Ok(response.map(|(key, value)| [key, value]))
|
Ok(response.map(|(key, value)| [key, value]))
|
||||||
}
|
}
|
||||||
@@ -120,7 +130,7 @@ impl RedisPool {
|
|||||||
&self,
|
&self,
|
||||||
key: &str,
|
key: &str,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> Result<Option<[Vec<u8>; 2]>, Error> {
|
) -> Result<Option<[Vec<u8>; 2]>> {
|
||||||
self.blocking.brpop(key, timeout).await
|
self.blocking.brpop(key, timeout).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use std::str::FromStr;
|
|||||||
use ariadne::ids::base62_impl::{parse_base62, to_base62};
|
use ariadne::ids::base62_impl::{parse_base62, to_base62};
|
||||||
use chrono::{TimeZone, Utc};
|
use chrono::{TimeZone, Utc};
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
|
use eyre::{Result, WrapErr, eyre};
|
||||||
use futures::stream::{FuturesUnordered, StreamExt};
|
use futures::stream::{FuturesUnordered, StreamExt};
|
||||||
use redis::aio::ConnectionLike;
|
use redis::aio::ConnectionLike;
|
||||||
use serde::de::DeserializeOwned;
|
use serde::de::DeserializeOwned;
|
||||||
@@ -16,8 +17,6 @@ use thiserror::Error;
|
|||||||
use tokio::time::{Instant, timeout_at};
|
use tokio::time::{Instant, timeout_at};
|
||||||
use tracing::{Instrument, info_span};
|
use tracing::{Instrument, info_span};
|
||||||
|
|
||||||
use crate::Error;
|
|
||||||
|
|
||||||
use super::commands;
|
use super::commands;
|
||||||
use super::connection::RoutableConnection;
|
use super::connection::RoutableConnection;
|
||||||
use super::key::KeyBuilder;
|
use super::key::KeyBuilder;
|
||||||
@@ -33,9 +32,7 @@ const FILL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
|||||||
pub(super) trait ConnectionProvider {
|
pub(super) trait ConnectionProvider {
|
||||||
type Connection: ConnectionLike + RoutableConnection;
|
type Connection: ConnectionLike + RoutableConnection;
|
||||||
|
|
||||||
fn connect(
|
fn connect(&self) -> impl Future<Output = Result<Self::Connection>> + Send;
|
||||||
&self,
|
|
||||||
) -> impl Future<Output = Result<Self::Connection, Error>> + Send;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
@@ -53,7 +50,7 @@ pub enum Codec {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum EncodingFormat {
|
pub enum EncodingFormat {
|
||||||
Json,
|
Postcard,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
@@ -92,7 +89,7 @@ impl FromStr for EncodingFormat {
|
|||||||
|
|
||||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||||
match value {
|
match value {
|
||||||
"json" => Ok(Self::Json),
|
"postcard" => Ok(Self::Postcard),
|
||||||
_ => Err(InvalidEncodingFormat),
|
_ => Err(InvalidEncodingFormat),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -112,12 +109,10 @@ pub struct CacheSettings {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CacheSettings {
|
impl CacheSettings {
|
||||||
pub fn encode_value<T: Serialize>(
|
pub fn encode_value<T: Serialize>(&self, value: &T) -> Result<Vec<u8>> {
|
||||||
&self,
|
|
||||||
value: &T,
|
|
||||||
) -> Result<Vec<u8>, Error> {
|
|
||||||
let mut value = match self.encoding_format {
|
let mut value = match self.encoding_format {
|
||||||
EncodingFormat::Json => serde_json::to_vec(value)?,
|
EncodingFormat::Postcard => postcard::to_allocvec(value)
|
||||||
|
.wrap_err("serializing Redis cache value with postcard")?,
|
||||||
};
|
};
|
||||||
|
|
||||||
if self.compression_level > 0
|
if self.compression_level > 0
|
||||||
@@ -148,16 +143,26 @@ impl CacheSettings {
|
|||||||
where
|
where
|
||||||
T: for<'a> Deserialize<'a>,
|
T: for<'a> Deserialize<'a>,
|
||||||
{
|
{
|
||||||
let (codec, value) = value.split_first()?;
|
let Some((codec, value)) = value.split_first() else {
|
||||||
let value = match Codec::try_from(*codec).ok()? {
|
return None;
|
||||||
|
};
|
||||||
|
let Ok(codec) = Codec::try_from(*codec) else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
let value = match codec {
|
||||||
Codec::Raw => Cow::Borrowed(value),
|
Codec::Raw => Cow::Borrowed(value),
|
||||||
Codec::Lz4 => Cow::Owned(
|
Codec::Lz4 => {
|
||||||
lz4_flex::block::decompress_size_prepended(value).ok()?,
|
let Ok(value) =
|
||||||
),
|
lz4_flex::block::decompress_size_prepended(value)
|
||||||
|
else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
Cow::Owned(value)
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
match self.encoding_format {
|
match self.encoding_format {
|
||||||
EncodingFormat::Json => serde_json::from_slice(&value).ok(),
|
EncodingFormat::Postcard => postcard::from_bytes(&value).ok(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -202,12 +207,12 @@ impl CacheManager {
|
|||||||
namespace: &str,
|
namespace: &str,
|
||||||
keys: &[K],
|
keys: &[K],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<Vec<T>, E>
|
) -> Result<Vec<T>>
|
||||||
where
|
where
|
||||||
P: ConnectionProvider,
|
P: ConnectionProvider,
|
||||||
F: FnOnce(Vec<K>) -> Fut,
|
F: FnOnce(Vec<K>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
K: Display
|
K: Display
|
||||||
+ Hash
|
+ Hash
|
||||||
@@ -220,7 +225,8 @@ impl CacheManager {
|
|||||||
{
|
{
|
||||||
Ok(self
|
Ok(self
|
||||||
.get_cached_keys_raw(provider, namespace, keys, closure)
|
.get_cached_keys_raw(provider, namespace, keys, closure)
|
||||||
.await?
|
.await
|
||||||
|
.wrap_err("fetching Redis cache values")?
|
||||||
.into_values()
|
.into_values()
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
@@ -232,12 +238,12 @@ impl CacheManager {
|
|||||||
namespace: &str,
|
namespace: &str,
|
||||||
keys: &[K],
|
keys: &[K],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<HashMap<K, T>, E>
|
) -> Result<HashMap<K, T>>
|
||||||
where
|
where
|
||||||
P: ConnectionProvider,
|
P: ConnectionProvider,
|
||||||
F: FnOnce(Vec<K>) -> Fut,
|
F: FnOnce(Vec<K>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
K: Display
|
K: Display
|
||||||
+ Hash
|
+ Hash
|
||||||
@@ -255,11 +261,16 @@ impl CacheManager {
|
|||||||
false,
|
false,
|
||||||
keys,
|
keys,
|
||||||
|ids| async move {
|
|ids| async move {
|
||||||
Ok(closure(ids)
|
let values = match closure(ids).await {
|
||||||
.await?
|
Ok(values) => values,
|
||||||
.into_iter()
|
Err(error) => return Err(error),
|
||||||
.map(|(key, value)| (key, (None::<String>, value)))
|
};
|
||||||
.collect())
|
Ok::<_, E>(
|
||||||
|
values
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, value)| (key, (None::<String>, value)))
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -274,12 +285,12 @@ impl CacheManager {
|
|||||||
case_sensitive: bool,
|
case_sensitive: bool,
|
||||||
keys: &[I],
|
keys: &[I],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<Vec<T>, E>
|
) -> Result<Vec<T>>
|
||||||
where
|
where
|
||||||
P: ConnectionProvider,
|
P: ConnectionProvider,
|
||||||
F: FnOnce(Vec<I>) -> Fut,
|
F: FnOnce(Vec<I>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
||||||
K: Display
|
K: Display
|
||||||
@@ -300,7 +311,8 @@ impl CacheManager {
|
|||||||
keys,
|
keys,
|
||||||
closure,
|
closure,
|
||||||
)
|
)
|
||||||
.await?
|
.await
|
||||||
|
.wrap_err("fetching Redis cache values by slug")?
|
||||||
.into_values()
|
.into_values()
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
@@ -314,12 +326,12 @@ impl CacheManager {
|
|||||||
case_sensitive: bool,
|
case_sensitive: bool,
|
||||||
keys: &[I],
|
keys: &[I],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<HashMap<K, T>, E>
|
) -> Result<HashMap<K, T>>
|
||||||
where
|
where
|
||||||
P: ConnectionProvider,
|
P: ConnectionProvider,
|
||||||
F: FnOnce(Vec<I>) -> Fut,
|
F: FnOnce(Vec<I>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
||||||
K: Display
|
K: Display
|
||||||
@@ -360,7 +372,9 @@ impl CacheManager {
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut connection =
|
let mut connection =
|
||||||
provider.connect().await.map_err(E::from)?;
|
provider.connect().await.wrap_err(
|
||||||
|
"connecting to Redis for slug lookup",
|
||||||
|
)?;
|
||||||
let values = match routing {
|
let values = match routing {
|
||||||
CacheReadRouting::ReplicaOptional => {
|
CacheReadRouting::ReplicaOptional => {
|
||||||
commands::get_many_strings(
|
commands::get_many_strings(
|
||||||
@@ -377,8 +391,8 @@ impl CacheManager {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.map_err(E::from)?;
|
.wrap_err("fetching Redis cache slug values")?;
|
||||||
Ok::<_, E>(
|
eyre::Ok(
|
||||||
values
|
values
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flatten()
|
.flatten()
|
||||||
@@ -386,7 +400,8 @@ impl CacheManager {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
.instrument(info_span!("get slug ids"))
|
.instrument(info_span!("get slug ids"))
|
||||||
.await?
|
.await
|
||||||
|
.wrap_err("resolving Redis cache slugs")?
|
||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
@@ -403,8 +418,10 @@ impl CacheManager {
|
|||||||
.map(|key| self.key_builder.entity(namespace, key))
|
.map(|key| self.key_builder.entity(namespace, key))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
let mut connection =
|
let mut connection = provider
|
||||||
provider.connect().await.map_err(E::from)?;
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting to Redis for cache lookup")?;
|
||||||
let mut cached_values = HashMap::new();
|
let mut cached_values = HashMap::new();
|
||||||
let values = match routing {
|
let values = match routing {
|
||||||
CacheReadRouting::ReplicaOptional => {
|
CacheReadRouting::ReplicaOptional => {
|
||||||
@@ -415,7 +432,7 @@ impl CacheManager {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.map_err(E::from)?;
|
.wrap_err("fetching Redis cache values")?;
|
||||||
for value in values {
|
for value in values {
|
||||||
if let Some(value) = value.and_then(|value| {
|
if let Some(value) = value.and_then(|value| {
|
||||||
self.settings
|
self.settings
|
||||||
@@ -425,7 +442,7 @@ impl CacheManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok::<_, E>((cached_values, ids))
|
eyre::Ok((cached_values, ids))
|
||||||
}
|
}
|
||||||
.instrument(info_span!("get_cached_values_closure"))
|
.instrument(info_span!("get_cached_values_closure"))
|
||||||
};
|
};
|
||||||
@@ -437,7 +454,9 @@ impl CacheManager {
|
|||||||
let deadline = Instant::now() + WAIT_TIMEOUT;
|
let deadline = Instant::now() + WAIT_TIMEOUT;
|
||||||
|
|
||||||
let (cached_values_raw, ids) =
|
let (cached_values_raw, ids) =
|
||||||
get_cached_values(ids, CacheReadRouting::ReplicaOptional).await?;
|
get_cached_values(ids, CacheReadRouting::ReplicaOptional)
|
||||||
|
.await
|
||||||
|
.wrap_err("reading Redis cache")?;
|
||||||
let mut cached_values = cached_values_raw
|
let mut cached_values = cached_values_raw
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|(key, value)| {
|
.filter_map(|(key, value)| {
|
||||||
@@ -508,7 +527,10 @@ impl CacheManager {
|
|||||||
|
|
||||||
let values = timeout_at(fill_deadline, closure(fetch_ids))
|
let values = timeout_at(fill_deadline, closure(fetch_ids))
|
||||||
.await
|
.await
|
||||||
.map_err(|_| lock_timeout_error(0, waiters.len()))??;
|
.map_err(|_| lock_timeout_error(0, waiters.len()))
|
||||||
|
.wrap_err("waiting to fill Redis cache")?;
|
||||||
|
let values =
|
||||||
|
values.wrap_err("fetching values to fill Redis cache")?;
|
||||||
|
|
||||||
let mut return_values = HashMap::new();
|
let mut return_values = HashMap::new();
|
||||||
let mut encoded_values = Vec::with_capacity(values.len());
|
let mut encoded_values = Vec::with_capacity(values.len());
|
||||||
@@ -520,13 +542,17 @@ impl CacheManager {
|
|||||||
val: value,
|
val: value,
|
||||||
alias: slug.clone(),
|
alias: slug.clone(),
|
||||||
};
|
};
|
||||||
let encoded =
|
let encoded = self
|
||||||
self.settings.encode_value(&value).map_err(E::from)?;
|
.settings
|
||||||
|
.encode_value(&value)
|
||||||
|
.wrap_err("encoding Redis cache value")?;
|
||||||
encoded_values.push((key, slug, value, encoded));
|
encoded_values.push((key, slug, value, encoded));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut connection =
|
let mut connection = provider
|
||||||
provider.connect().await.map_err(E::from)?;
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting to Redis to fill cache")?;
|
||||||
for (key, slug, _, encoded) in &encoded_values {
|
for (key, slug, _, encoded) in &encoded_values {
|
||||||
let redis_key =
|
let redis_key =
|
||||||
self.key_builder.entity(namespace, key.to_string());
|
self.key_builder.entity(namespace, key.to_string());
|
||||||
@@ -537,7 +563,7 @@ impl CacheManager {
|
|||||||
default_expiry,
|
default_expiry,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(E::from)?;
|
.wrap_err("writing Redis cache value")?;
|
||||||
if let Some(slug) = slug
|
if let Some(slug) = slug
|
||||||
&& let Some(slug_namespace) = slug_namespace
|
&& let Some(slug_namespace) = slug_namespace
|
||||||
{
|
{
|
||||||
@@ -554,7 +580,7 @@ impl CacheManager {
|
|||||||
default_expiry,
|
default_expiry,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(E::from)?;
|
.wrap_err("writing Redis cache slug")?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -563,7 +589,7 @@ impl CacheManager {
|
|||||||
return_values.insert(key, value);
|
return_values.insert(key, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
Result::<_, E>::Ok(return_values)
|
Result::<_>::Ok(return_values)
|
||||||
}
|
}
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -604,13 +630,14 @@ impl CacheManager {
|
|||||||
Err(error) => Err(error),
|
Err(error) => Err(error),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(error) => Err(E::from(error)),
|
Err(error) => Err(error),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(error) => Err(error),
|
Err(error) => Err(error),
|
||||||
};
|
};
|
||||||
cached_values.extend(operation_result?);
|
cached_values
|
||||||
|
.extend(operation_result.wrap_err("populating Redis cache")?);
|
||||||
|
|
||||||
Ok(cached_values
|
Ok(cached_values
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -679,7 +706,7 @@ fn push_identity(identities: &mut Vec<String>, identity: String) {
|
|||||||
async fn wait_for_locks<I>(
|
async fn wait_for_locks<I>(
|
||||||
waiters: Vec<(I, LockWaiter)>,
|
waiters: Vec<(I, LockWaiter)>,
|
||||||
deadline: Instant,
|
deadline: Instant,
|
||||||
) -> Result<Vec<I>, Error> {
|
) -> Result<Vec<I>> {
|
||||||
let total = waiters.len();
|
let total = waiters.len();
|
||||||
let mut released = Vec::with_capacity(total);
|
let mut released = Vec::with_capacity(total);
|
||||||
let mut futures = FuturesUnordered::new();
|
let mut futures = FuturesUnordered::new();
|
||||||
@@ -695,26 +722,24 @@ async fn wait_for_locks<I>(
|
|||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
released.push(key);
|
released.push(key);
|
||||||
}
|
}
|
||||||
Err(error)
|
Err(_) if Instant::now() >= deadline => {
|
||||||
if is_lock_timeout(&error) || Instant::now() >= deadline =>
|
|
||||||
{
|
|
||||||
return Err(lock_timeout_error(released.len(), total));
|
return Err(lock_timeout_error(released.len(), total));
|
||||||
}
|
}
|
||||||
Err(error) => return Err(error),
|
Err(error) => {
|
||||||
|
return Err(error).wrap_err("waiting for Redis cache lock");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(released)
|
Ok(released)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_lock_timeout(error: &Error) -> bool {
|
fn lock_timeout_error(
|
||||||
matches!(error, Error::LocalCacheTimeout { .. })
|
locks_released: usize,
|
||||||
}
|
locks_waiting: usize,
|
||||||
|
) -> eyre::Report {
|
||||||
fn lock_timeout_error(locks_released: usize, locks_waiting: usize) -> Error {
|
eyre!(
|
||||||
Error::LocalCacheTimeout {
|
"timeout waiting on local Redis cache lock ({locks_released}/{locks_waiting} released)"
|
||||||
released: locks_released,
|
)
|
||||||
total: locks_waiting,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
#[derive(Serialize, Deserialize)]
|
||||||
|
|||||||
+3
-14
@@ -3,11 +3,10 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
|||||||
|
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use dashmap::mapref::entry::Entry;
|
use dashmap::mapref::entry::Entry;
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tokio::time::{Instant, timeout_at};
|
use tokio::time::{Instant, timeout_at};
|
||||||
|
|
||||||
use crate::Error;
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(in crate::cache) struct LockCoordinator {
|
pub(in crate::cache) struct LockCoordinator {
|
||||||
locks: Arc<DashMap<String, Arc<LockState>>>,
|
locks: Arc<DashMap<String, Arc<LockState>>>,
|
||||||
@@ -76,10 +75,7 @@ pub(in crate::cache) struct LockWaiter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl LockWaiter {
|
impl LockWaiter {
|
||||||
pub(in crate::cache) async fn wait(
|
pub(in crate::cache) async fn wait(self, deadline: Instant) -> Result<()> {
|
||||||
self,
|
|
||||||
deadline: Instant,
|
|
||||||
) -> Result<(), Error> {
|
|
||||||
loop {
|
loop {
|
||||||
if self.state.released.load(Ordering::Acquire) {
|
if self.state.released.load(Ordering::Acquire) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -94,7 +90,7 @@ impl LockWaiter {
|
|||||||
|
|
||||||
timeout_at(deadline, notified)
|
timeout_at(deadline, notified)
|
||||||
.await
|
.await
|
||||||
.map_err(|_| lock_timeout())?;
|
.wrap_err("waiting for local Redis cache lock")?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -112,10 +108,3 @@ impl LockState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn lock_timeout() -> Error {
|
|
||||||
Error::LocalCacheTimeout {
|
|
||||||
released: 0,
|
|
||||||
total: 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use redis::aio::ConnectionLike;
|
use redis::aio::ConnectionLike;
|
||||||
use redis::{FromRedisValue, ToRedisArgs};
|
use redis::{FromRedisValue, ToRedisArgs};
|
||||||
|
|
||||||
use crate::Error;
|
|
||||||
|
|
||||||
use super::cache::CacheSettings;
|
use super::cache::CacheSettings;
|
||||||
use super::connection::RoutableConnection;
|
use super::connection::RoutableConnection;
|
||||||
use super::routing::primary_mget_routing;
|
use super::routing::primary_mget_routing;
|
||||||
@@ -18,7 +17,7 @@ pub async fn set<C, D>(
|
|||||||
key: &str,
|
key: &str,
|
||||||
data: D,
|
data: D,
|
||||||
expiry: i64,
|
expiry: i64,
|
||||||
) -> Result<(), Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
D: ToRedisArgs + Send + Sync + Debug,
|
D: ToRedisArgs + Send + Sync + Debug,
|
||||||
@@ -29,7 +28,8 @@ where
|
|||||||
.arg("EX")
|
.arg("EX")
|
||||||
.arg(expiry)
|
.arg(expiry)
|
||||||
.query_async::<()>(connection)
|
.query_async::<()>(connection)
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_err("writing to Redis")?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,7 +40,7 @@ pub async fn set_serialized<C, D>(
|
|||||||
data: D,
|
data: D,
|
||||||
expiry: Option<i64>,
|
expiry: Option<i64>,
|
||||||
settings: &CacheSettings,
|
settings: &CacheSettings,
|
||||||
) -> Result<(), Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
D: serde::Serialize,
|
D: serde::Serialize,
|
||||||
@@ -48,21 +48,24 @@ where
|
|||||||
set(
|
set(
|
||||||
connection,
|
connection,
|
||||||
key,
|
key,
|
||||||
settings.encode_value(&data)?,
|
settings
|
||||||
|
.encode_value(&data)
|
||||||
|
.wrap_err("serializing Redis value")?,
|
||||||
expiry.unwrap_or(settings.default_expiry),
|
expiry.unwrap_or(settings.default_expiry),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub async fn get<C>(
|
pub async fn get<C>(connection: &mut C, key: &str) -> Result<Option<String>>
|
||||||
connection: &mut C,
|
|
||||||
key: &str,
|
|
||||||
) -> Result<Option<String>, Error>
|
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
Ok(cmd("GET").arg(key).query_async(connection).await?)
|
cmd("GET")
|
||||||
|
.arg(key)
|
||||||
|
.query_async(connection)
|
||||||
|
.await
|
||||||
|
.wrap_err("fetching from Redis")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Issues ordinary `MGET` commands in bounded chunks. Cluster routing and
|
/// Issues ordinary `MGET` commands in bounded chunks. Cluster routing and
|
||||||
@@ -72,7 +75,7 @@ where
|
|||||||
pub async fn get_many<C>(
|
pub async fn get_many<C>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<Vec<u8>>>, Error>
|
) -> Result<Vec<Option<Vec<u8>>>>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
@@ -83,7 +86,7 @@ where
|
|||||||
pub async fn get_many_strings<C>(
|
pub async fn get_many_strings<C>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<String>>, Error>
|
) -> Result<Vec<Option<String>>>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
@@ -93,7 +96,7 @@ where
|
|||||||
pub(super) async fn get_many_primary<C>(
|
pub(super) async fn get_many_primary<C>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<Vec<u8>>>, Error>
|
) -> Result<Vec<Option<Vec<u8>>>>
|
||||||
where
|
where
|
||||||
C: RoutableConnection,
|
C: RoutableConnection,
|
||||||
{
|
{
|
||||||
@@ -103,7 +106,7 @@ where
|
|||||||
pub(super) async fn get_many_strings_primary<C>(
|
pub(super) async fn get_many_strings_primary<C>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<String>>, Error>
|
) -> Result<Vec<Option<String>>>
|
||||||
where
|
where
|
||||||
C: RoutableConnection,
|
C: RoutableConnection,
|
||||||
{
|
{
|
||||||
@@ -113,7 +116,7 @@ where
|
|||||||
pub(super) async fn get_many_as<C, T>(
|
pub(super) async fn get_many_as<C, T>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<T>>, Error>
|
) -> Result<Vec<Option<T>>>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
T: FromRedisValue,
|
T: FromRedisValue,
|
||||||
@@ -123,7 +126,8 @@ where
|
|||||||
let part = cmd("MGET")
|
let part = cmd("MGET")
|
||||||
.arg(chunk)
|
.arg(chunk)
|
||||||
.query_async::<Vec<Option<T>>>(connection)
|
.query_async::<Vec<Option<T>>>(connection)
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_err("fetching multiple values from Redis")?;
|
||||||
values.extend(part);
|
values.extend(part);
|
||||||
}
|
}
|
||||||
Ok(values)
|
Ok(values)
|
||||||
@@ -132,7 +136,7 @@ where
|
|||||||
async fn get_many_primary_as<C, T>(
|
async fn get_many_primary_as<C, T>(
|
||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<T>>, Error>
|
) -> Result<Vec<Option<T>>>
|
||||||
where
|
where
|
||||||
C: RoutableConnection,
|
C: RoutableConnection,
|
||||||
T: FromRedisValue,
|
T: FromRedisValue,
|
||||||
@@ -143,10 +147,14 @@ where
|
|||||||
command.arg(chunk);
|
command.arg(chunk);
|
||||||
let value = connection
|
let value = connection
|
||||||
.route_command(command, primary_mget_routing(chunk))
|
.route_command(command, primary_mget_routing(chunk))
|
||||||
.await?;
|
.await
|
||||||
let part =
|
.wrap_err("fetching multiple values from primary Redis nodes")?;
|
||||||
redis::from_redis_value::<Vec<Option<T>>>(value.extract_error()?)
|
let value = value
|
||||||
.map_err(redis::RedisError::from)?;
|
.extract_error()
|
||||||
|
.wrap_err("extracting Redis response")?;
|
||||||
|
let part = redis::from_redis_value::<Vec<Option<T>>>(value)
|
||||||
|
.map_err(redis::RedisError::from)
|
||||||
|
.wrap_err("decoding Redis response")?;
|
||||||
values.extend(part);
|
values.extend(part);
|
||||||
}
|
}
|
||||||
Ok(values)
|
Ok(values)
|
||||||
@@ -157,13 +165,16 @@ pub async fn get_deserialized<C, R>(
|
|||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
key: &str,
|
key: &str,
|
||||||
settings: &CacheSettings,
|
settings: &CacheSettings,
|
||||||
) -> Result<Option<R>, Error>
|
) -> Result<Option<R>>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
R: for<'a> serde::Deserialize<'a>,
|
R: for<'a> serde::Deserialize<'a>,
|
||||||
{
|
{
|
||||||
let value: Option<Vec<u8>> =
|
let value: Option<Vec<u8>> = cmd("GET")
|
||||||
cmd("GET").arg(key).query_async(connection).await?;
|
.arg(key)
|
||||||
|
.query_async(connection)
|
||||||
|
.await
|
||||||
|
.wrap_err("fetching serialized value from Redis")?;
|
||||||
Ok(value.and_then(|value| settings.decode_value(&value)))
|
Ok(value.and_then(|value| settings.decode_value(&value)))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,47 +183,49 @@ pub async fn get_many_deserialized<C, R>(
|
|||||||
connection: &mut C,
|
connection: &mut C,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
settings: &CacheSettings,
|
settings: &CacheSettings,
|
||||||
) -> Result<Vec<Option<R>>, Error>
|
) -> Result<Vec<Option<R>>>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
R: for<'a> serde::Deserialize<'a>,
|
R: for<'a> serde::Deserialize<'a>,
|
||||||
{
|
{
|
||||||
Ok(get_many(connection, keys)
|
Ok(get_many(connection, keys)
|
||||||
.await?
|
.await
|
||||||
|
.wrap_err("fetching serialized values from Redis")?
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|value| value.and_then(|value| settings.decode_value(&value)))
|
.map(|value| value.and_then(|value| settings.decode_value(&value)))
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub async fn delete<C>(connection: &mut C, key: &str) -> Result<(), Error>
|
pub async fn delete<C>(connection: &mut C, key: &str) -> Result<()>
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
cmd("DEL").arg(key).query_async::<()>(connection).await?;
|
cmd("DEL")
|
||||||
|
.arg(key)
|
||||||
|
.query_async::<()>(connection)
|
||||||
|
.await
|
||||||
|
.wrap_err("deleting from Redis")?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub async fn delete_many<C>(
|
pub async fn delete_many<C>(connection: &mut C, keys: &[String]) -> Result<()>
|
||||||
connection: &mut C,
|
|
||||||
keys: &[String],
|
|
||||||
) -> Result<(), Error>
|
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
if !keys.is_empty() {
|
if !keys.is_empty() {
|
||||||
cmd("DEL").arg(keys).query_async::<()>(connection).await?;
|
cmd("DEL")
|
||||||
|
.arg(keys)
|
||||||
|
.query_async::<()>(connection)
|
||||||
|
.await
|
||||||
|
.wrap_err("deleting multiple values from Redis")?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub async fn lpush<C, D>(
|
pub async fn lpush<C, D>(connection: &mut C, key: &str, value: D) -> Result<()>
|
||||||
connection: &mut C,
|
|
||||||
key: &str,
|
|
||||||
value: D,
|
|
||||||
) -> Result<(), Error>
|
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
D: ToRedisArgs + Send + Sync + Debug,
|
D: ToRedisArgs + Send + Sync + Debug,
|
||||||
@@ -221,17 +234,19 @@ where
|
|||||||
.arg(key)
|
.arg(key)
|
||||||
.arg(value)
|
.arg(value)
|
||||||
.query_async::<()>(connection)
|
.query_async::<()>(connection)
|
||||||
.await?;
|
.await
|
||||||
|
.wrap_err("pushing to Redis list")?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub async fn incr<C>(
|
pub async fn incr<C>(connection: &mut C, key: &str) -> Result<Option<u64>>
|
||||||
connection: &mut C,
|
|
||||||
key: &str,
|
|
||||||
) -> Result<Option<u64>, Error>
|
|
||||||
where
|
where
|
||||||
C: ConnectionLike,
|
C: ConnectionLike,
|
||||||
{
|
{
|
||||||
Ok(cmd("INCR").arg(key).query_async(connection).await?)
|
cmd("INCR")
|
||||||
|
.arg(key)
|
||||||
|
.query_async(connection)
|
||||||
|
.await
|
||||||
|
.wrap_err("incrementing Redis value")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::{fmt, str::FromStr};
|
use std::{fmt, str::FromStr};
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||||
@@ -91,13 +92,11 @@ pub(crate) struct RedisPoolSize {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RedisPoolSize {
|
impl RedisPoolSize {
|
||||||
fn new(
|
fn new(name: &'static str, max: usize, min: usize) -> Result<Self> {
|
||||||
name: &'static str,
|
|
||||||
max: usize,
|
|
||||||
min: usize,
|
|
||||||
) -> Result<Self, RedisConfigError> {
|
|
||||||
if max == 0 || min > max {
|
if max == 0 || min > max {
|
||||||
return Err(RedisConfigError::InvalidPoolSize { name, max, min });
|
return Err(
|
||||||
|
RedisConfigError::InvalidPoolSize { name, max, min }.into()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self { max, min })
|
Ok(Self { max, min })
|
||||||
@@ -193,11 +192,12 @@ impl RedisConfig {
|
|||||||
blocking_pool_size: (usize, usize),
|
blocking_pool_size: (usize, usize),
|
||||||
cache_locking_strategy: CacheLockingStrategy,
|
cache_locking_strategy: CacheLockingStrategy,
|
||||||
read_replica_strategy: ReadReplicaStrategy,
|
read_replica_strategy: ReadReplicaStrategy,
|
||||||
) -> Result<Self, RedisConfigError> {
|
) -> Result<Self> {
|
||||||
if cache_locking_strategy == CacheLockingStrategy::Distributed {
|
if cache_locking_strategy == CacheLockingStrategy::Distributed {
|
||||||
return Err(RedisConfigError::UnsupportedCacheLockingStrategy {
|
return Err(RedisConfigError::UnsupportedCacheLockingStrategy {
|
||||||
strategy: cache_locking_strategy,
|
strategy: cache_locking_strategy,
|
||||||
});
|
}
|
||||||
|
.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let seed_urls = raw_urls
|
let seed_urls = raw_urls
|
||||||
@@ -208,26 +208,32 @@ impl RedisConfig {
|
|||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
if seed_urls.is_empty() {
|
if seed_urls.is_empty() {
|
||||||
return Err(RedisConfigError::MissingUrl);
|
return Err(RedisConfigError::MissingUrl.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let backend = match (mode, connection_type) {
|
let backend = match (mode, connection_type) {
|
||||||
(RedisTopology::Standalone, RedisConnectionType::Pooled) => {
|
(RedisTopology::Standalone, RedisConnectionType::Pooled) => {
|
||||||
if seed_urls.len() != 1 {
|
if seed_urls.len() != 1 {
|
||||||
return Err(RedisConfigError::MultipleStandaloneUrls);
|
return Err(RedisConfigError::MultipleStandaloneUrls.into());
|
||||||
}
|
}
|
||||||
RedisBackendConfig::StandalonePooled(RedisPoolSize::new(
|
RedisBackendConfig::StandalonePooled(
|
||||||
"standalone",
|
RedisPoolSize::new(
|
||||||
standalone_pool_size.0,
|
"standalone",
|
||||||
standalone_pool_size.1,
|
standalone_pool_size.0,
|
||||||
)?)
|
standalone_pool_size.1,
|
||||||
|
)
|
||||||
|
.wrap_err("validating standalone Redis pool size")?,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
(RedisTopology::Cluster, RedisConnectionType::Pooled) => {
|
(RedisTopology::Cluster, RedisConnectionType::Pooled) => {
|
||||||
RedisBackendConfig::ClusterPooled(RedisPoolSize::new(
|
RedisBackendConfig::ClusterPooled(
|
||||||
"cluster",
|
RedisPoolSize::new(
|
||||||
cluster_pool_size.0,
|
"cluster",
|
||||||
cluster_pool_size.1,
|
cluster_pool_size.0,
|
||||||
)?)
|
cluster_pool_size.1,
|
||||||
|
)
|
||||||
|
.wrap_err("validating clustered Redis pool size")?,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
(RedisTopology::Cluster, RedisConnectionType::Multiplexed) => {
|
(RedisTopology::Cluster, RedisConnectionType::Multiplexed) => {
|
||||||
RedisBackendConfig::ClusterMultiplexed
|
RedisBackendConfig::ClusterMultiplexed
|
||||||
@@ -236,7 +242,8 @@ impl RedisConfig {
|
|||||||
return Err(RedisConfigError::UnsupportedConnectionType {
|
return Err(RedisConfigError::UnsupportedConnectionType {
|
||||||
mode,
|
mode,
|
||||||
connection_type,
|
connection_type,
|
||||||
});
|
}
|
||||||
|
.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -249,7 +256,8 @@ impl RedisConfig {
|
|||||||
"blocking",
|
"blocking",
|
||||||
blocking_pool_size.0,
|
blocking_pool_size.0,
|
||||||
blocking_pool_size.1,
|
blocking_pool_size.1,
|
||||||
)?,
|
)
|
||||||
|
.wrap_err("validating blocking Redis pool size")?,
|
||||||
cache_locking_strategy,
|
cache_locking_strategy,
|
||||||
read_replica_strategy,
|
read_replica_strategy,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use futures::future::try_join_all;
|
use futures::future::try_join_all;
|
||||||
use prometheus::Registry;
|
use prometheus::Registry;
|
||||||
use redis::aio::ConnectionLike;
|
use redis::aio::ConnectionLike;
|
||||||
@@ -7,7 +8,6 @@ use redis::cluster_read_routing::{
|
|||||||
RandomReplicaStrategy, RoundRobinReplicaStrategy,
|
RandomReplicaStrategy, RoundRobinReplicaStrategy,
|
||||||
};
|
};
|
||||||
use redis::cluster_routing::RoutingInfo;
|
use redis::cluster_routing::RoutingInfo;
|
||||||
use thiserror::Error;
|
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::ReadReplicaStrategy;
|
use crate::ReadReplicaStrategy;
|
||||||
@@ -29,16 +29,6 @@ pub(crate) enum RedisBackend {
|
|||||||
ClusterMultiplexed(redis::cluster_async::ClusterConnection),
|
ClusterMultiplexed(redis::cluster_async::ClusterConnection),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum RedisBackendBuildError {
|
|
||||||
#[error("failed to configure Redis client: {0}")]
|
|
||||||
Redis(#[from] redis::RedisError),
|
|
||||||
#[error("failed to build Redis pool: {0}")]
|
|
||||||
PoolBuild(#[from] deadpool_redis::BuildError),
|
|
||||||
#[error("failed to establish initial Redis pool connections: {0}")]
|
|
||||||
Pool(#[from] deadpool_redis::PoolError),
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) struct RedisConnection {
|
pub(crate) struct RedisConnection {
|
||||||
inner: RedisConnectionInner,
|
inner: RedisConnectionInner,
|
||||||
}
|
}
|
||||||
@@ -58,9 +48,7 @@ pub(crate) trait RoutableConnection: ConnectionLike {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RedisBackend {
|
impl RedisBackend {
|
||||||
pub(crate) async fn new(
|
pub(crate) async fn new(config: &RedisConfig) -> Result<Self> {
|
||||||
config: &RedisConfig,
|
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
|
||||||
match config.backend() {
|
match config.backend() {
|
||||||
RedisBackendConfig::StandalonePooled(pool_size) => {
|
RedisBackendConfig::StandalonePooled(pool_size) => {
|
||||||
Self::standalone_pooled(config, pool_size).await
|
Self::standalone_pooled(config, pool_size).await
|
||||||
@@ -77,21 +65,25 @@ impl RedisBackend {
|
|||||||
async fn standalone_pooled(
|
async fn standalone_pooled(
|
||||||
config: &RedisConfig,
|
config: &RedisConfig,
|
||||||
pool_size: RedisPoolSize,
|
pool_size: RedisPoolSize,
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
) -> Result<Self> {
|
||||||
let connection_config = redis::AsyncConnectionConfig::new()
|
let connection_config = redis::AsyncConnectionConfig::new()
|
||||||
.set_connection_timeout(None)
|
.set_connection_timeout(None)
|
||||||
.set_response_timeout(None);
|
.set_response_timeout(None);
|
||||||
let manager = deadpool_redis::Manager::new_with_config(
|
let manager = deadpool_redis::Manager::new_with_config(
|
||||||
config.seed_urls()[0].clone(),
|
config.seed_urls()[0].clone(),
|
||||||
connection_config,
|
connection_config,
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("configuring standalone Redis client")?;
|
||||||
let pool = deadpool_redis::Pool::builder(manager)
|
let pool = deadpool_redis::Pool::builder(manager)
|
||||||
.max_size(pool_size.max())
|
.max_size(pool_size.max())
|
||||||
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
|
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
|
||||||
.runtime(deadpool_redis::Runtime::Tokio1)
|
.runtime(deadpool_redis::Runtime::Tokio1)
|
||||||
.build()?;
|
.build()
|
||||||
|
.wrap_err("building standalone Redis pool")?;
|
||||||
|
|
||||||
warm_standalone_pool(&pool, pool_size.min()).await?;
|
warm_standalone_pool(&pool, pool_size.min())
|
||||||
|
.await
|
||||||
|
.wrap_err("warming standalone Redis pool")?;
|
||||||
retain_standalone_pool(pool.clone());
|
retain_standalone_pool(pool.clone());
|
||||||
|
|
||||||
Ok(Self::StandalonePooled(pool))
|
Ok(Self::StandalonePooled(pool))
|
||||||
@@ -100,16 +92,18 @@ impl RedisBackend {
|
|||||||
async fn cluster_pooled(
|
async fn cluster_pooled(
|
||||||
config: &RedisConfig,
|
config: &RedisConfig,
|
||||||
pool_size: RedisPoolSize,
|
pool_size: RedisPoolSize,
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
) -> Result<Self> {
|
||||||
let manager = deadpool_redis::cluster::Manager::new(
|
let manager = deadpool_redis::cluster::Manager::new(
|
||||||
config.seed_urls().to_vec(),
|
config.seed_urls().to_vec(),
|
||||||
false,
|
false,
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("configuring clustered Redis client")?;
|
||||||
let pool = deadpool_redis::cluster::Pool::builder(manager)
|
let pool = deadpool_redis::cluster::Pool::builder(manager)
|
||||||
.max_size(pool_size.max())
|
.max_size(pool_size.max())
|
||||||
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
|
.wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms())))
|
||||||
.runtime(deadpool_redis::Runtime::Tokio1)
|
.runtime(deadpool_redis::Runtime::Tokio1)
|
||||||
.build()?;
|
.build()
|
||||||
|
.wrap_err("building clustered Redis pool")?;
|
||||||
|
|
||||||
if config.read_replica_strategy() != ReadReplicaStrategy::Primary {
|
if config.read_replica_strategy() != ReadReplicaStrategy::Primary {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -117,15 +111,15 @@ impl RedisBackend {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
warm_cluster_pool(&pool, pool_size.min()).await?;
|
warm_cluster_pool(&pool, pool_size.min())
|
||||||
|
.await
|
||||||
|
.wrap_err("warming clustered Redis pool")?;
|
||||||
retain_cluster_pool(pool.clone());
|
retain_cluster_pool(pool.clone());
|
||||||
|
|
||||||
Ok(Self::ClusterPooled(pool))
|
Ok(Self::ClusterPooled(pool))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cluster_multiplexed(
|
async fn cluster_multiplexed(config: &RedisConfig) -> Result<Self> {
|
||||||
config: &RedisConfig,
|
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
|
||||||
let mut builder = redis::cluster::ClusterClientBuilder::new(
|
let mut builder = redis::cluster::ClusterClientBuilder::new(
|
||||||
config.seed_urls().iter().map(String::as_str),
|
config.seed_urls().iter().map(String::as_str),
|
||||||
);
|
);
|
||||||
@@ -141,22 +135,31 @@ impl RedisBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let client = builder.build()?;
|
let client = builder
|
||||||
let connection = client.get_async_connection().await?;
|
.build()
|
||||||
|
.wrap_err("building multiplexed Redis client")?;
|
||||||
|
let connection = client
|
||||||
|
.get_async_connection()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting multiplexed Redis client")?;
|
||||||
|
|
||||||
Ok(Self::ClusterMultiplexed(connection))
|
Ok(Self::ClusterMultiplexed(connection))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn connect(
|
pub(crate) async fn connect(&self) -> Result<RedisConnection> {
|
||||||
&self,
|
|
||||||
) -> Result<RedisConnection, deadpool_redis::PoolError> {
|
|
||||||
let inner = match self {
|
let inner = match self {
|
||||||
Self::StandalonePooled(pool) => {
|
Self::StandalonePooled(pool) => {
|
||||||
RedisConnectionInner::StandalonePooled(pool.get().await?)
|
RedisConnectionInner::StandalonePooled(
|
||||||
}
|
pool.get()
|
||||||
Self::ClusterPooled(pool) => {
|
.await
|
||||||
RedisConnectionInner::ClusterPooled(pool.get().await?)
|
.wrap_err("fetching standalone Redis connection")?,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
Self::ClusterPooled(pool) => RedisConnectionInner::ClusterPooled(
|
||||||
|
pool.get()
|
||||||
|
.await
|
||||||
|
.wrap_err("fetching clustered Redis connection")?,
|
||||||
|
),
|
||||||
Self::ClusterMultiplexed(connection) => {
|
Self::ClusterMultiplexed(connection) => {
|
||||||
RedisConnectionInner::ClusterMultiplexed(connection.clone())
|
RedisConnectionInner::ClusterMultiplexed(connection.clone())
|
||||||
}
|
}
|
||||||
@@ -165,10 +168,7 @@ impl RedisBackend {
|
|||||||
Ok(RedisConnection { inner })
|
Ok(RedisConnection { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn register_metrics(
|
pub(crate) fn register_metrics(&self, registry: &Registry) -> Result<()> {
|
||||||
&self,
|
|
||||||
registry: &Registry,
|
|
||||||
) -> Result<(), prometheus::Error> {
|
|
||||||
register_command_pool_metrics(registry, self.clone())
|
register_command_pool_metrics(registry, self.clone())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -282,8 +282,10 @@ impl RoutableConnection for RedisConnection {
|
|||||||
async fn warm_standalone_pool(
|
async fn warm_standalone_pool(
|
||||||
pool: &deadpool_redis::Pool,
|
pool: &deadpool_redis::Pool,
|
||||||
min: usize,
|
min: usize,
|
||||||
) -> Result<(), deadpool_redis::PoolError> {
|
) -> Result<()> {
|
||||||
let connections = try_join_all((0..min).map(|_| pool.get())).await?;
|
let connections = try_join_all((0..min).map(|_| pool.get()))
|
||||||
|
.await
|
||||||
|
.wrap_err("fetching initial standalone Redis connections")?;
|
||||||
drop(connections);
|
drop(connections);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -291,8 +293,10 @@ async fn warm_standalone_pool(
|
|||||||
async fn warm_cluster_pool(
|
async fn warm_cluster_pool(
|
||||||
pool: &deadpool_redis::cluster::Pool,
|
pool: &deadpool_redis::cluster::Pool,
|
||||||
min: usize,
|
min: usize,
|
||||||
) -> Result<(), deadpool_redis::PoolError> {
|
) -> Result<()> {
|
||||||
let connections = try_join_all((0..min).map(|_| pool.get())).await?;
|
let connections = try_join_all((0..min).map(|_| pool.get()))
|
||||||
|
.await
|
||||||
|
.wrap_err("fetching initial clustered Redis connections")?;
|
||||||
drop(connections);
|
drop(connections);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
+41
-51
@@ -6,6 +6,7 @@ use std::hash::Hash;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use prometheus::Registry;
|
use prometheus::Registry;
|
||||||
use redis::aio::ConnectionLike;
|
use redis::aio::ConnectionLike;
|
||||||
use redis::{FromRedisValue, ToRedisArgs};
|
use redis::{FromRedisValue, ToRedisArgs};
|
||||||
@@ -35,25 +36,7 @@ pub use config::{
|
|||||||
RedisConfigError, RedisConnectionType, RedisTopology,
|
RedisConfigError, RedisConnectionType, RedisTopology,
|
||||||
};
|
};
|
||||||
use connection::RedisBackend;
|
use connection::RedisBackend;
|
||||||
pub use connection::RedisBackendBuildError;
|
|
||||||
pub use key::KeyBuilder;
|
pub use key::KeyBuilder;
|
||||||
use thiserror::Error as ThisError;
|
|
||||||
|
|
||||||
#[derive(Debug, ThisError)]
|
|
||||||
pub enum Error {
|
|
||||||
#[error("error while interacting with Redis: {0}")]
|
|
||||||
Redis(#[from] redis::RedisError),
|
|
||||||
#[error("Redis pool error: {0}")]
|
|
||||||
Pool(#[from] deadpool_redis::PoolError),
|
|
||||||
#[error("error while serializing a Redis cache value: {0}")]
|
|
||||||
Serialization(#[from] serde_json::Error),
|
|
||||||
#[error("Redis blocking timeout must be greater than zero")]
|
|
||||||
InvalidBlockingTimeout,
|
|
||||||
#[error(
|
|
||||||
"timeout waiting on local cache lock ({released}/{total} released)"
|
|
||||||
)]
|
|
||||||
LocalCacheTimeout { released: usize, total: usize },
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct RedisPool {
|
pub struct RedisPool {
|
||||||
@@ -75,15 +58,19 @@ impl RedisPool {
|
|||||||
meta_namespace: impl Into<Arc<str>>,
|
meta_namespace: impl Into<Arc<str>>,
|
||||||
config: RedisConfig,
|
config: RedisConfig,
|
||||||
cache_settings: CacheSettings,
|
cache_settings: CacheSettings,
|
||||||
) -> Result<Self, RedisBackendBuildError> {
|
) -> Result<Self> {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
strategy = %config.cache_locking_strategy(),
|
strategy = %config.cache_locking_strategy(),
|
||||||
"configured Redis cache locking"
|
"configured Redis cache locking"
|
||||||
);
|
);
|
||||||
|
|
||||||
let backend = RedisBackend::new(&config).await?;
|
let backend = RedisBackend::new(&config)
|
||||||
|
.await
|
||||||
|
.wrap_err("creating Redis command backend")?;
|
||||||
|
|
||||||
let blocking = blocking::RedisBlockingPool::new(&config).await?;
|
let blocking = blocking::RedisBlockingPool::new(&config)
|
||||||
|
.await
|
||||||
|
.wrap_err("creating Redis blocking pool")?;
|
||||||
let key_builder = KeyBuilder::new(meta_namespace, config.topology());
|
let key_builder = KeyBuilder::new(meta_namespace, config.topology());
|
||||||
let cache = CacheManager::new(key_builder.clone(), cache_settings);
|
let cache = CacheManager::new(key_builder.clone(), cache_settings);
|
||||||
|
|
||||||
@@ -102,9 +89,13 @@ impl RedisPool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RedisPool {
|
impl RedisPool {
|
||||||
pub async fn connect(&self) -> Result<RedisConnection, Error> {
|
pub async fn connect(&self) -> Result<RedisConnection> {
|
||||||
Ok(RedisConnection {
|
Ok(RedisConnection {
|
||||||
inner: self.backend.connect().await?,
|
inner: self
|
||||||
|
.backend
|
||||||
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting to Redis")?,
|
||||||
key_builder: self.key_builder.clone(),
|
key_builder: self.key_builder.clone(),
|
||||||
settings: self.cache.settings().clone(),
|
settings: self.cache.settings().clone(),
|
||||||
})
|
})
|
||||||
@@ -113,9 +104,13 @@ impl RedisPool {
|
|||||||
pub async fn register_and_set_metrics(
|
pub async fn register_and_set_metrics(
|
||||||
&self,
|
&self,
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
) -> Result<(), prometheus::Error> {
|
) -> Result<()> {
|
||||||
self.backend.register_metrics(registry)?;
|
self.backend
|
||||||
self.blocking.register_metrics(registry)
|
.register_metrics(registry)
|
||||||
|
.wrap_err("registering Redis command pool metrics")?;
|
||||||
|
self.blocking
|
||||||
|
.register_metrics(registry)
|
||||||
|
.wrap_err("registering Redis blocking pool metrics")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_cached_keys<F, Fut, T, K, E>(
|
pub async fn get_cached_keys<F, Fut, T, K, E>(
|
||||||
@@ -123,11 +118,11 @@ impl RedisPool {
|
|||||||
namespace: &str,
|
namespace: &str,
|
||||||
keys: &[K],
|
keys: &[K],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<Vec<T>, E>
|
) -> Result<Vec<T>>
|
||||||
where
|
where
|
||||||
F: FnOnce(Vec<K>) -> Fut,
|
F: FnOnce(Vec<K>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
K: Display
|
K: Display
|
||||||
+ Hash
|
+ Hash
|
||||||
@@ -148,11 +143,11 @@ impl RedisPool {
|
|||||||
namespace: &str,
|
namespace: &str,
|
||||||
keys: &[K],
|
keys: &[K],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<std::collections::HashMap<K, T>, E>
|
) -> Result<std::collections::HashMap<K, T>>
|
||||||
where
|
where
|
||||||
F: FnOnce(Vec<K>) -> Fut,
|
F: FnOnce(Vec<K>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
Fut: Future<Output = Result<DashMap<K, T>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
K: Display
|
K: Display
|
||||||
+ Hash
|
+ Hash
|
||||||
@@ -175,11 +170,11 @@ impl RedisPool {
|
|||||||
case_sensitive: bool,
|
case_sensitive: bool,
|
||||||
keys: &[I],
|
keys: &[I],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<Vec<T>, E>
|
) -> Result<Vec<T>>
|
||||||
where
|
where
|
||||||
F: FnOnce(Vec<I>) -> Fut,
|
F: FnOnce(Vec<I>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
||||||
K: Display
|
K: Display
|
||||||
@@ -210,11 +205,11 @@ impl RedisPool {
|
|||||||
case_sensitive: bool,
|
case_sensitive: bool,
|
||||||
keys: &[I],
|
keys: &[I],
|
||||||
closure: F,
|
closure: F,
|
||||||
) -> Result<std::collections::HashMap<K, T>, E>
|
) -> Result<std::collections::HashMap<K, T>>
|
||||||
where
|
where
|
||||||
F: FnOnce(Vec<I>) -> Fut,
|
F: FnOnce(Vec<I>) -> Fut,
|
||||||
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
Fut: Future<Output = Result<DashMap<K, (Option<S>, T)>, E>>,
|
||||||
E: From<Error>,
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
T: Serialize + DeserializeOwned,
|
T: Serialize + DeserializeOwned,
|
||||||
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
I: Display + Hash + Eq + PartialEq + Clone + Debug,
|
||||||
K: Display
|
K: Display
|
||||||
@@ -242,9 +237,7 @@ impl RedisPool {
|
|||||||
impl ConnectionProvider for RedisPool {
|
impl ConnectionProvider for RedisPool {
|
||||||
type Connection = RedisConnection;
|
type Connection = RedisConnection;
|
||||||
|
|
||||||
fn connect(
|
fn connect(&self) -> impl Future<Output = Result<Self::Connection>> + Send {
|
||||||
&self,
|
|
||||||
) -> impl Future<Output = Result<Self::Connection, Error>> + Send {
|
|
||||||
RedisPool::connect(self)
|
RedisPool::connect(self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -259,7 +252,7 @@ impl RedisConnection {
|
|||||||
key: &str,
|
key: &str,
|
||||||
data: D,
|
data: D,
|
||||||
expiry: Option<i64>,
|
expiry: Option<i64>,
|
||||||
) -> Result<(), Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
D: ToRedisArgs + Send + Sync + Debug,
|
D: ToRedisArgs + Send + Sync + Debug,
|
||||||
{
|
{
|
||||||
@@ -277,7 +270,7 @@ impl RedisConnection {
|
|||||||
key: &str,
|
key: &str,
|
||||||
data: D,
|
data: D,
|
||||||
expiry: Option<i64>,
|
expiry: Option<i64>,
|
||||||
) -> Result<(), Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
D: Serialize,
|
D: Serialize,
|
||||||
{
|
{
|
||||||
@@ -291,31 +284,28 @@ impl RedisConnection {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get(&mut self, key: &str) -> Result<Option<String>, Error> {
|
pub async fn get(&mut self, key: &str) -> Result<Option<String>> {
|
||||||
commands::get(&mut self.inner, key).await
|
commands::get(&mut self.inner, key).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_many(
|
pub async fn get_many(
|
||||||
&mut self,
|
&mut self,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<Vec<u8>>>, Error> {
|
) -> Result<Vec<Option<Vec<u8>>>> {
|
||||||
commands::get_many(&mut self.inner, keys).await
|
commands::get_many(&mut self.inner, keys).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_many_typed<R>(
|
pub async fn get_many_typed<R>(
|
||||||
&mut self,
|
&mut self,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<R>>, Error>
|
) -> Result<Vec<Option<R>>>
|
||||||
where
|
where
|
||||||
R: FromRedisValue,
|
R: FromRedisValue,
|
||||||
{
|
{
|
||||||
commands::get_many_as(&mut self.inner, keys).await
|
commands::get_many_as(&mut self.inner, keys).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_deserialized<R>(
|
pub async fn get_deserialized<R>(&mut self, key: &str) -> Result<Option<R>>
|
||||||
&mut self,
|
|
||||||
key: &str,
|
|
||||||
) -> Result<Option<R>, Error>
|
|
||||||
where
|
where
|
||||||
R: for<'a> serde::Deserialize<'a>,
|
R: for<'a> serde::Deserialize<'a>,
|
||||||
{
|
{
|
||||||
@@ -325,7 +315,7 @@ impl RedisConnection {
|
|||||||
pub async fn get_many_deserialized<R>(
|
pub async fn get_many_deserialized<R>(
|
||||||
&mut self,
|
&mut self,
|
||||||
keys: &[String],
|
keys: &[String],
|
||||||
) -> Result<Vec<Option<R>>, Error>
|
) -> Result<Vec<Option<R>>>
|
||||||
where
|
where
|
||||||
R: for<'a> serde::Deserialize<'a>,
|
R: for<'a> serde::Deserialize<'a>,
|
||||||
{
|
{
|
||||||
@@ -333,22 +323,22 @@ impl RedisConnection {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete(&mut self, key: &str) -> Result<(), Error> {
|
pub async fn delete(&mut self, key: &str) -> Result<()> {
|
||||||
commands::delete(&mut self.inner, key).await
|
commands::delete(&mut self.inner, key).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_many(&mut self, keys: &[String]) -> Result<(), Error> {
|
pub async fn delete_many(&mut self, keys: &[String]) -> Result<()> {
|
||||||
commands::delete_many(&mut self.inner, keys).await
|
commands::delete_many(&mut self.inner, keys).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn lpush<D>(&mut self, key: &str, value: D) -> Result<(), Error>
|
pub async fn lpush<D>(&mut self, key: &str, value: D) -> Result<()>
|
||||||
where
|
where
|
||||||
D: ToRedisArgs + Send + Sync + Debug,
|
D: ToRedisArgs + Send + Sync + Debug,
|
||||||
{
|
{
|
||||||
commands::lpush(&mut self.inner, key, value).await
|
commands::lpush(&mut self.inner, key, value).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn incr(&mut self, key: &str) -> Result<Option<u64>, Error> {
|
pub async fn incr(&mut self, key: &str) -> Result<Option<u64>> {
|
||||||
commands::incr(&mut self.inner, key).await
|
commands::incr(&mut self.inner, key).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use prometheus::{IntGauge, Registry};
|
use prometheus::{IntGauge, Registry};
|
||||||
|
|
||||||
const METRICS_UPDATE_INTERVAL: Duration = Duration::from_secs(5);
|
const METRICS_UPDATE_INTERVAL: Duration = Duration::from_secs(5);
|
||||||
@@ -72,7 +73,7 @@ impl RedisPoolMetrics {
|
|||||||
fn register(
|
fn register(
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
kind: RedisPoolMetricsKind,
|
kind: RedisPoolMetricsKind,
|
||||||
) -> Result<Self, prometheus::Error> {
|
) -> Result<Self> {
|
||||||
let prefix = kind.metric_prefix();
|
let prefix = kind.metric_prefix();
|
||||||
let description = kind.description();
|
let description = kind.description();
|
||||||
let max_size = IntGauge::new(
|
let max_size = IntGauge::new(
|
||||||
@@ -80,28 +81,40 @@ impl RedisPoolMetrics {
|
|||||||
format!(
|
format!(
|
||||||
"Maximum logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
|
"Maximum logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
|
||||||
),
|
),
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("creating Redis pool maximum size metric")?;
|
||||||
let size = IntGauge::new(
|
let size = IntGauge::new(
|
||||||
format!("{prefix}_size"),
|
format!("{prefix}_size"),
|
||||||
format!(
|
format!(
|
||||||
"Current logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
|
"Current logical connection count for the {description}; clustered logical connections may own multiple physical sockets"
|
||||||
),
|
),
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("creating Redis pool size metric")?;
|
||||||
let available = IntGauge::new(
|
let available = IntGauge::new(
|
||||||
format!("{prefix}_available"),
|
format!("{prefix}_available"),
|
||||||
format!("Available logical connections in the {description}"),
|
format!("Available logical connections in the {description}"),
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("creating Redis pool availability metric")?;
|
||||||
let waiting = IntGauge::new(
|
let waiting = IntGauge::new(
|
||||||
format!("{prefix}_waiting"),
|
format!("{prefix}_waiting"),
|
||||||
format!(
|
format!(
|
||||||
"Number of futures waiting for a logical connection from the {description}"
|
"Number of futures waiting for a logical connection from the {description}"
|
||||||
),
|
),
|
||||||
)?;
|
)
|
||||||
|
.wrap_err("creating Redis pool waiters metric")?;
|
||||||
|
|
||||||
registry.register(Box::new(max_size.clone()))?;
|
registry
|
||||||
registry.register(Box::new(size.clone()))?;
|
.register(Box::new(max_size.clone()))
|
||||||
registry.register(Box::new(available.clone()))?;
|
.wrap_err("registering Redis pool maximum size metric")?;
|
||||||
registry.register(Box::new(waiting.clone()))?;
|
registry
|
||||||
|
.register(Box::new(size.clone()))
|
||||||
|
.wrap_err("registering Redis pool size metric")?;
|
||||||
|
registry
|
||||||
|
.register(Box::new(available.clone()))
|
||||||
|
.wrap_err("registering Redis pool availability metric")?;
|
||||||
|
registry
|
||||||
|
.register(Box::new(waiting.clone()))
|
||||||
|
.wrap_err("registering Redis pool waiters metric")?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
max_size,
|
max_size,
|
||||||
@@ -122,7 +135,7 @@ impl RedisPoolMetrics {
|
|||||||
pub(super) fn register_command_pool_metrics<P>(
|
pub(super) fn register_command_pool_metrics<P>(
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
provider: P,
|
provider: P,
|
||||||
) -> Result<(), prometheus::Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
P: LogicalPoolStatusProvider,
|
P: LogicalPoolStatusProvider,
|
||||||
{
|
{
|
||||||
@@ -132,7 +145,7 @@ where
|
|||||||
pub(super) fn register_blocking_pool_metrics<P>(
|
pub(super) fn register_blocking_pool_metrics<P>(
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
provider: P,
|
provider: P,
|
||||||
) -> Result<(), prometheus::Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
P: LogicalPoolStatusProvider,
|
P: LogicalPoolStatusProvider,
|
||||||
{
|
{
|
||||||
@@ -143,11 +156,12 @@ fn register_pool_metrics<P>(
|
|||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
kind: RedisPoolMetricsKind,
|
kind: RedisPoolMetricsKind,
|
||||||
provider: P,
|
provider: P,
|
||||||
) -> Result<(), prometheus::Error>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
P: LogicalPoolStatusProvider,
|
P: LogicalPoolStatusProvider,
|
||||||
{
|
{
|
||||||
let metrics = RedisPoolMetrics::register(registry, kind)?;
|
let metrics = RedisPoolMetrics::register(registry, kind)
|
||||||
|
.wrap_err("registering Redis pool metrics")?;
|
||||||
metrics.set(provider.logical_pool_status());
|
metrics.set(provider.logical_pool_status());
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use eyre::{Result, WrapErr};
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use redis::ToRedisArgs;
|
use redis::ToRedisArgs;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use super::{Error, RedisPool};
|
use super::RedisPool;
|
||||||
|
|
||||||
const PUBSUB_BUFFER_SIZE: usize = 1024;
|
const PUBSUB_BUFFER_SIZE: usize = 1024;
|
||||||
const INITIAL_RECONNECT_BACKOFF: Duration = Duration::from_millis(250);
|
const INITIAL_RECONNECT_BACKOFF: Duration = Duration::from_millis(250);
|
||||||
@@ -35,21 +36,20 @@ impl RedisPool {
|
|||||||
receiver
|
receiver
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn publish<M>(
|
pub async fn publish<M>(&self, channel: &str, message: M) -> Result<()>
|
||||||
&self,
|
|
||||||
channel: &str,
|
|
||||||
message: M,
|
|
||||||
) -> Result<(), Error>
|
|
||||||
where
|
where
|
||||||
M: ToRedisArgs + Send + Sync,
|
M: ToRedisArgs + Send + Sync,
|
||||||
{
|
{
|
||||||
let mut connection = self.connect().await?;
|
let mut connection = self
|
||||||
|
.connect()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting to Redis for publishing")?;
|
||||||
let _: usize = redis::cmd("PUBLISH")
|
let _: usize = redis::cmd("PUBLISH")
|
||||||
.arg(channel)
|
.arg(channel)
|
||||||
.arg(message)
|
.arg(message)
|
||||||
.query_async(&mut connection)
|
.query_async(&mut connection)
|
||||||
.await
|
.await
|
||||||
.map_err(Error::from)?;
|
.wrap_err("publishing to Redis channel")?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,10 +111,17 @@ async fn forward_from_seed(
|
|||||||
seed_url: &str,
|
seed_url: &str,
|
||||||
channel: &'static str,
|
channel: &'static str,
|
||||||
sender: &mpsc::Sender<Vec<u8>>,
|
sender: &mpsc::Sender<Vec<u8>>,
|
||||||
) -> redis::RedisResult<SubscriptionOutcome> {
|
) -> Result<SubscriptionOutcome> {
|
||||||
let client = redis::Client::open(seed_url)?;
|
let client = redis::Client::open(seed_url)
|
||||||
let mut pubsub = client.get_async_pubsub().await?;
|
.wrap_err("configuring Redis Pub/Sub client")?;
|
||||||
pubsub.subscribe(channel).await?;
|
let mut pubsub = client
|
||||||
|
.get_async_pubsub()
|
||||||
|
.await
|
||||||
|
.wrap_err("connecting to Redis Pub/Sub")?;
|
||||||
|
pubsub
|
||||||
|
.subscribe(channel)
|
||||||
|
.await
|
||||||
|
.wrap_err("subscribing to Redis channel")?;
|
||||||
info!(channel, "Established Redis Pub/Sub subscription");
|
info!(channel, "Established Redis Pub/Sub subscription");
|
||||||
|
|
||||||
let mut stream = pubsub.into_on_message();
|
let mut stream = pubsub.into_on_message();
|
||||||
|
|||||||
Reference in New Issue
Block a user