Compare commits

..
Author SHA1 Message Date
aecsocket 16851cbd34 fmt 2026-08-02 17:28:54 +01:00
aecsocket bfc0081ee3 fix 2026-08-02 17:17:55 +01:00
aecsocket 95c2e58e0b fix shear 2026-08-02 17:10:03 +01:00
aecsocket f1930f9738 remove old parity code 2026-08-02 17:04:32 +01:00
aecsocket ef3bad811a approach parity even without explicit parity enabled 2026-08-02 16:38:22 +01:00
aecsocket c12fee2f7b fix author query 2026-08-02 15:40:02 +01:00
aecsocket 221a54a153 expand what parity does 2026-08-02 15:07:02 +01:00
aecsocket f965f64a89 more parity and perf work 2026-07-31 22:12:50 +02:00
aecsocket 3ba74a8b01 fix up search pagination 2026-07-31 22:12:49 +02:00
aecsocket d658484f6b perf 2026-07-31 22:12:49 +02:00
aecsocket 4a2388f311 (do not merge) search test branch 2026-07-31 22:12:48 +02:00
26 changed files with 1715 additions and 803 deletions
Generated
-1
View File
@@ -5490,7 +5490,6 @@ dependencies = [
"jemalloc_pprof",
"json-patch 4.1.0",
"labrinth",
"language-tags",
"lettre",
"meilisearch-sdk",
"modrinth-content-management",
-1
View File
@@ -110,7 +110,6 @@ image = { version = "0.25.8", default-features = false, features = ["rayon"] }
indexmap = "2.11.4"
indicatif = "0.18.0"
itertools = "0.14.0"
language-tags = "0.3.2"
jemalloc_pprof = "0.8.1"
json-patch = { version = "4.1.0", default-features = false }
lettre = { version = "0.11.19", default-features = false, features = [
+2 -14
View File
@@ -563,18 +563,12 @@ components:
type: array
items:
type: string
example: [fabric]
example: [fabric]
game_versions:
type: array
items:
type: string
example: ['1.18', 1.18.1]
version_types:
type: array
items:
type: string
enum: [release, alpha, beta]
example: [release]
required:
- loaders
- game_versions
@@ -618,12 +612,6 @@ components:
items:
type: string
example: ['1.18', 1.18.1]
version_types:
type: array
items:
type: string
enum: [release, alpha, beta]
example: [release]
required:
- loaders
- game_versions
@@ -2877,7 +2865,7 @@ paths:
$ref: '#/components/schemas/HashList'
/version_files/update:
post:
summary: Latest versions of multiple projects from hashes, loader(s), and game version(s)
summary: Latest versions of multiple project from hashes, loader(s), and game version(s)
description: This is the same as [`/version_file/{hash}/update`](#operation/getLatestVersionFromHash) except it accepts multiple hashes.
operationId: getLatestVersionsFromHashes
tags:
+4 -4
View File
@@ -17,14 +17,14 @@ DATABASE_URL=postgresql://labrinth:labrinth@labrinth-postgres/labrinth
DATABASE_MIN_CONNECTIONS=0
DATABASE_MAX_CONNECTIONS=16
SEARCH_BACKEND=typesense
SEARCH_BACKEND=elasticsearch
MEILISEARCH_READ_ADDR=http://localhost:7700
MEILISEARCH_WRITE_ADDRS=http://localhost:7700
MEILISEARCH_KEY=modrinth
ELASTICSEARCH_URL=http://localhost:9200
ELASTICSEARCH_URL=http://elasticsearch0:9200
ELASTICSEARCH_INDEX_PREFIX=labrinth
ELASTICSEARCH_USERNAME=elastic
ELASTICSEARCH_PASSWORD=elastic
ELASTICSEARCH_USERNAME=
ELASTICSEARCH_PASSWORD=
SEARCH_INDEX_CHUNK_SIZE=5000
SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
+2 -2
View File
@@ -17,7 +17,7 @@ DATABASE_URL=postgresql://labrinth:labrinth@localhost/labrinth
DATABASE_MIN_CONNECTIONS=0
DATABASE_MAX_CONNECTIONS=16
SEARCH_BACKEND=typesense
SEARCH_BACKEND=elasticsearch
# Meilisearch configuration
MEILISEARCH_READ_ADDR=http://localhost:7700
@@ -32,7 +32,7 @@ ELASTICSEARCH_INDEX_PREFIX=labrinth
# MEILISEARCH_READ_ADDR=http://localhost:7710
# MEILISEARCH_WRITE_ADDRS=http://localhost:7700,http://localhost:7701
SEARCH_BACKEND=typesense
SEARCH_BACKEND=elasticsearch
MEILISEARCH_KEY=modrinth
MEILISEARCH_META_NAMESPACE=
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\t\tINSERT INTO user_preferences (user_id, preferences)\n\t\t\tVALUES ($1, $2)\n\t\t\tON CONFLICT (user_id) DO UPDATE\n\t\t\tSET preferences = EXCLUDED.preferences\n\t\t\t",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "121602731987ccfa487f46bb86bb62cd913c5760c6ac0e6d1abda818673e83f6"
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\t\tINSERT INTO user_preferences (user_id, preferences)\n\t\t\tVALUES ($1, $2)\n\t\t\tON CONFLICT (user_id) DO NOTHING\n\t\t\t",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "20616b23f6948c1efdce94e44922d09daa809cb04f6de1824c3e9b924f193ba7"
}
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\t\tSELECT preferences AS \"preferences: Json<UserPreferences>\"\n\t\t\tFROM user_preferences\n\t\t\tWHERE user_id = $1\n\t\t\tFOR UPDATE\n\t\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "preferences: Json<UserPreferences>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "59034736065e7bec7ed838a870a5d389108e90adfee265792395af483b84091a"
}
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\t\tSELECT preferences AS \"preferences: Json<UserPreferences>\"\n\t\t\tFROM user_preferences\n\t\t\tWHERE user_id = $1\n\t\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "preferences: Json<UserPreferences>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "674d05dc957468ad195d8704dfc66e035f5812244a7e69d81af1fa0989ffe9eb"
}
-1
View File
@@ -71,7 +71,6 @@ image = { workspace = true, features = [
"webp",
] }
itertools = { workspace = true }
language-tags = { workspace = true }
json-patch = { workspace = true }
lettre = { workspace = true }
meilisearch-sdk = { workspace = true, features = ["reqwest"] }
@@ -1,7 +0,0 @@
CREATE TABLE user_preferences (
user_id BIGINT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
preferences JSONB NOT NULL,
CONSTRAINT user_preferences_object CHECK (
jsonb_typeof(preferences) = 'object'
)
);
-2
View File
@@ -37,7 +37,6 @@ pub mod team_item;
pub mod thread_item;
pub mod user_item;
pub mod user_limits;
pub mod user_preferences_item;
pub mod user_subscription_item;
pub mod users_compliance;
pub mod users_notifications_preferences_item;
@@ -59,7 +58,6 @@ pub use team_item::DBTeam;
pub use team_item::DBTeamMember;
pub use thread_item::{DBThread, DBThreadMessage};
pub use user_item::DBUser;
pub use user_preferences_item::DBUserPreferences;
pub use version_item::DBVersion;
pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser};
@@ -1,97 +0,0 @@
use sqlx::types::Json;
use crate::{
database::{Executor, PgTransaction},
models::v3::user_preferences::UserPreferences,
};
use super::DBUserId;
pub struct DBUserPreferences;
impl DBUserPreferences {
pub async fn get<'a, E>(
user_id: DBUserId,
exec: E,
) -> Result<Option<UserPreferences>, sqlx::Error>
where
E: Executor<'a, Database = sqlx::Postgres>,
{
let preferences = sqlx::query_scalar!(
r#"
SELECT preferences AS "preferences: Json<UserPreferences>"
FROM user_preferences
WHERE user_id = $1
"#,
user_id.0,
)
.fetch_optional(exec)
.await?
.map(|preferences| preferences.0);
Ok(preferences)
}
pub async fn upsert<'a, E>(
user_id: DBUserId,
preferences: &UserPreferences,
exec: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = sqlx::Postgres>,
{
sqlx::query!(
r#"
INSERT INTO user_preferences (user_id, preferences)
VALUES ($1, $2)
ON CONFLICT (user_id) DO UPDATE
SET preferences = EXCLUDED.preferences
"#,
user_id.0,
Json(preferences) as Json<&UserPreferences>,
)
.execute(exec)
.await?;
Ok(())
}
pub async fn insert_if_absent(
user_id: DBUserId,
preferences: &UserPreferences,
transaction: &mut PgTransaction<'_>,
) -> Result<(), sqlx::Error> {
sqlx::query!(
r#"
INSERT INTO user_preferences (user_id, preferences)
VALUES ($1, $2)
ON CONFLICT (user_id) DO NOTHING
"#,
user_id.0,
Json(preferences) as Json<&UserPreferences>,
)
.execute(&mut *transaction)
.await?;
Ok(())
}
pub async fn get_for_update(
user_id: DBUserId,
transaction: &mut PgTransaction<'_>,
) -> Result<UserPreferences, sqlx::Error> {
let preferences = sqlx::query_scalar!(
r#"
SELECT preferences AS "preferences: Json<UserPreferences>"
FROM user_preferences
WHERE user_id = $1
FOR UPDATE
"#,
user_id.0,
)
.fetch_one(&mut *transaction)
.await?;
Ok(preferences.0)
}
}
+5
View File
@@ -237,6 +237,11 @@ vars! {
SEARCH_TYPESENSE_DEFAULT_BUCKETING: Json<crate::search::backend::typesense::Bucketing> =
Json(crate::search::backend::typesense::Bucketing::Buckets(5));
SEARCH_TYPESENSE_DEFAULT_MAX_CANDIDATES: usize = 24usize;
ELASTICSEARCH_URL: String = "http://localhost:9200";
ELASTICSEARCH_INDEX_PREFIX: String = "labrinth";
ELASTICSEARCH_USERNAME: String = "";
ELASTICSEARCH_PASSWORD: String = "";
ELASTICSEARCH_BULK_BATCH_SIZE: usize = 1000usize;
// storage
STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
-1
View File
@@ -18,5 +18,4 @@ pub mod sessions;
pub mod teams;
pub mod threads;
pub mod user_limits;
pub mod user_preferences;
pub mod users;
@@ -1,327 +0,0 @@
use eyre::{Result, eyre};
use language_tags::LanguageTag;
use serde::{Deserialize, Serialize};
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(rename_all = "lowercase")]
pub enum ThemePreferenceValue {
Auto,
Light,
Dark,
Oled,
Retro,
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(rename_all = "lowercase")]
pub enum LayoutPreferenceValue {
Grid,
Rows,
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(rename_all = "lowercase")]
pub enum FriendRequestSource {
None,
Mutuals,
Everyone,
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(rename_all = "lowercase")]
pub enum SharedInstanceSource {
None,
Friends,
Everyone,
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct ThemePreference {
pub value: ThemePreferenceValue,
pub sync: bool,
}
impl Default for ThemePreference {
fn default() -> Self {
Self {
value: ThemePreferenceValue::Auto,
sync: false,
}
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct LanguagePreference {
pub value: String,
pub sync: bool,
}
impl Default for LanguagePreference {
fn default() -> Self {
Self {
value: "en-US".to_string(),
sync: false,
}
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct LayoutPreferences {
pub sync: bool,
pub mods: LayoutPreferenceValue,
pub plugins: LayoutPreferenceValue,
pub datapacks: LayoutPreferenceValue,
pub shaders: LayoutPreferenceValue,
pub resourcepacks: LayoutPreferenceValue,
pub modpacks: LayoutPreferenceValue,
pub servers: LayoutPreferenceValue,
pub users: LayoutPreferenceValue,
}
impl Default for LayoutPreferences {
fn default() -> Self {
Self {
sync: true,
mods: LayoutPreferenceValue::Rows,
plugins: LayoutPreferenceValue::Rows,
datapacks: LayoutPreferenceValue::Rows,
shaders: LayoutPreferenceValue::Grid,
resourcepacks: LayoutPreferenceValue::Grid,
modpacks: LayoutPreferenceValue::Rows,
servers: LayoutPreferenceValue::Rows,
users: LayoutPreferenceValue::Rows,
}
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct SidebarPreferences {
pub sync: bool,
pub right_aligned_search: bool,
pub left_aligned_content: bool,
}
impl Default for SidebarPreferences {
fn default() -> Self {
Self {
sync: false,
right_aligned_search: false,
left_aligned_content: false,
}
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct SocialPreferences {
pub friend_request_sources: FriendRequestSource,
pub shared_instance_sources: SharedInstanceSource,
}
impl Default for SocialPreferences {
fn default() -> Self {
Self {
friend_request_sources: FriendRequestSource::Everyone,
shared_instance_sources: SharedInstanceSource::Friends,
}
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema,
)]
#[serde(deny_unknown_fields)]
pub struct UserPreferences {
pub theme: ThemePreference,
pub language: LanguagePreference,
pub layouts: LayoutPreferences,
pub sidebars: SidebarPreferences,
pub social: SocialPreferences,
}
impl Default for UserPreferences {
fn default() -> Self {
Self {
theme: ThemePreference::default(),
language: LanguagePreference::default(),
layouts: LayoutPreferences::default(),
sidebars: SidebarPreferences::default(),
social: SocialPreferences::default(),
}
}
}
impl UserPreferences {
pub fn validate(&self) -> Result<()> {
validate_language(&self.language.value)
}
pub fn apply_patch(&mut self, patch: UserPreferencesPatch) {
if let Some(theme) = patch.theme {
if let Some(value) = theme.value {
self.theme.value = value;
}
if let Some(sync) = theme.sync {
self.theme.sync = sync;
}
}
if let Some(language) = patch.language {
if let Some(value) = language.value {
self.language.value = value;
}
if let Some(sync) = language.sync {
self.language.sync = sync;
}
}
if let Some(layouts) = patch.layouts {
if let Some(sync) = layouts.sync {
self.layouts.sync = sync;
}
if let Some(mods) = layouts.mods {
self.layouts.mods = mods;
}
if let Some(plugins) = layouts.plugins {
self.layouts.plugins = plugins;
}
if let Some(datapacks) = layouts.datapacks {
self.layouts.datapacks = datapacks;
}
if let Some(shaders) = layouts.shaders {
self.layouts.shaders = shaders;
}
if let Some(resourcepacks) = layouts.resourcepacks {
self.layouts.resourcepacks = resourcepacks;
}
if let Some(modpacks) = layouts.modpacks {
self.layouts.modpacks = modpacks;
}
if let Some(servers) = layouts.servers {
self.layouts.servers = servers;
}
if let Some(users) = layouts.users {
self.layouts.users = users;
}
}
if let Some(sidebars) = patch.sidebars {
if let Some(sync) = sidebars.sync {
self.sidebars.sync = sync;
}
if let Some(right_aligned_search) = sidebars.right_aligned_search {
self.sidebars.right_aligned_search = right_aligned_search;
}
if let Some(left_aligned_content) = sidebars.left_aligned_content {
self.sidebars.left_aligned_content = left_aligned_content;
}
}
if let Some(social) = patch.social {
if let Some(friend_request_sources) = social.friend_request_sources
{
self.social.friend_request_sources = friend_request_sources;
}
if let Some(shared_instance_sources) =
social.shared_instance_sources
{
self.social.shared_instance_sources = shared_instance_sources;
}
}
}
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct UserPreferencesPatch {
pub theme: Option<ThemePreferencePatch>,
pub language: Option<LanguagePreferencePatch>,
pub layouts: Option<LayoutPreferencesPatch>,
pub sidebars: Option<SidebarPreferencesPatch>,
pub social: Option<SocialPreferencesPatch>,
}
impl UserPreferencesPatch {
pub fn validate(&self) -> Result<()> {
if let Some(language) = self
.language
.as_ref()
.and_then(|language| language.value.as_ref())
{
validate_language(language)?;
}
Ok(())
}
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct ThemePreferencePatch {
pub value: Option<ThemePreferenceValue>,
pub sync: Option<bool>,
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct LanguagePreferencePatch {
pub value: Option<String>,
pub sync: Option<bool>,
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct LayoutPreferencesPatch {
pub sync: Option<bool>,
pub mods: Option<LayoutPreferenceValue>,
pub plugins: Option<LayoutPreferenceValue>,
pub datapacks: Option<LayoutPreferenceValue>,
pub shaders: Option<LayoutPreferenceValue>,
pub resourcepacks: Option<LayoutPreferenceValue>,
pub modpacks: Option<LayoutPreferenceValue>,
pub servers: Option<LayoutPreferenceValue>,
pub users: Option<LayoutPreferenceValue>,
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct SidebarPreferencesPatch {
pub sync: Option<bool>,
pub right_aligned_search: Option<bool>,
pub left_aligned_content: Option<bool>,
}
#[derive(Debug, Clone, Default, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct SocialPreferencesPatch {
pub friend_request_sources: Option<FriendRequestSource>,
pub shared_instance_sources: Option<SharedInstanceSource>,
}
fn validate_language(language: &str) -> Result<()> {
LanguageTag::parse(language).map(|_| ()).map_err(|_| {
eyre!("`language.value` must be a valid BCP 47 language tag")
})
}
-5
View File
@@ -22,7 +22,6 @@ pub mod statistics;
pub mod tags;
pub mod teams;
pub mod threads;
pub mod user_preferences;
pub mod users;
pub mod version_creation;
pub mod version_file;
@@ -65,7 +64,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.configure(tags::config)
.configure(teams::config)
.configure(threads::config)
.configure(user_preferences::config)
.configure(users::config)
.configure(version_file::config)
.configure(versions::config)
@@ -199,9 +197,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
threads::threads_get_route,
threads::thread_send_message_route,
threads::message_delete_route,
user_preferences::get_user_preferences,
user_preferences::put_user_preferences,
user_preferences::patch_user_preferences,
users::all_projects,
users::admin_user_email,
users::projects_list_route,
@@ -1,140 +0,0 @@
use actix_web::{HttpRequest, get, patch, put, web};
use xredis::RedisPool;
use crate::{
auth::get_user_from_headers,
database::{PgPool, ReadOnlyPgPool, models::DBUserPreferences},
models::{
pats::Scopes,
v3::user_preferences::{UserPreferences, UserPreferencesPatch},
},
queue::session::AuthQueue,
util::error::Context,
};
use super::ApiError;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(get_user_preferences)
.service(put_user_preferences)
.service(patch_user_preferences);
}
#[utoipa::path(
tag = "users",
responses((status = OK, body = UserPreferences))
)]
#[get("/user/preferences")]
pub async fn get_user_preferences(
req: HttpRequest,
pool: web::Data<PgPool>,
ro_pool: web::Data<ReadOnlyPgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
) -> Result<web::Json<UserPreferences>, ApiError> {
let user = get_user_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Scopes::USER_READ,
)
.await?
.1;
let preferences = DBUserPreferences::get(user.id.into(), &***ro_pool)
.await
.wrap_internal_err("failed to fetch user preferences")?
.unwrap_or_default();
Ok(web::Json(preferences))
}
#[utoipa::path(
tag = "users",
request_body = UserPreferences,
responses((status = OK, body = UserPreferences))
)]
#[put("/user/preferences")]
pub async fn put_user_preferences(
req: HttpRequest,
web::Json(preferences): web::Json<UserPreferences>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
) -> Result<web::Json<UserPreferences>, ApiError> {
preferences.validate().map_err(ApiError::Request)?;
let user = get_user_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Scopes::USER_WRITE,
)
.await?
.1;
DBUserPreferences::upsert(user.id.into(), &preferences, &**pool)
.await
.wrap_internal_err("failed to store user preferences")?;
Ok(web::Json(preferences))
}
#[utoipa::path(
tag = "users",
request_body = UserPreferencesPatch,
responses((status = OK, body = UserPreferences))
)]
#[patch("/user/preferences")]
pub async fn patch_user_preferences(
req: HttpRequest,
web::Json(patch): web::Json<UserPreferencesPatch>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
) -> Result<web::Json<UserPreferences>, ApiError> {
patch.validate().map_err(ApiError::Request)?;
let user = get_user_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Scopes::USER_WRITE,
)
.await?
.1;
let user_id = user.id.into();
let mut transaction = pool
.begin()
.await
.wrap_internal_err("failed to begin user preferences transaction")?;
DBUserPreferences::insert_if_absent(
user_id,
&UserPreferences::default(),
&mut transaction,
)
.await
.wrap_internal_err("failed to initialize user preferences")?;
let mut preferences =
DBUserPreferences::get_for_update(user_id, &mut transaction)
.await
.wrap_internal_err("failed to fetch user preferences for update")?;
preferences.apply_patch(patch);
DBUserPreferences::upsert(user_id, &preferences, &mut transaction)
.await
.wrap_internal_err("failed to store user preferences")?;
transaction
.commit()
.await
.wrap_internal_err("failed to commit user preferences transaction")?;
Ok(web::Json(preferences))
}
@@ -0,0 +1,390 @@
use eyre::{Result, eyre};
use serde_json::{Value, json};
use crate::search::filter::{
FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
FilterPredicate,
};
use crate::search::indexing::normalize_for_search;
const MAX_DNF_CLAUSES: usize = 64;
const MAX_FILTER_DEPTH: usize = 64;
const MAX_FILTER_NODES: usize = 1024;
#[derive(Clone, Copy, PartialEq, Eq)]
enum FilterScope {
Project,
Version,
Mixed,
}
pub(super) struct ElasticsearchFilter {
pub query: Value,
pub has_version_filter: bool,
}
pub(super) fn serialize_filter(
filter: &FilterExpr,
) -> Result<ElasticsearchFilter> {
let (nodes, depth) = filter_complexity(filter);
if nodes > MAX_FILTER_NODES {
return Err(eyre!("search filter has too many expressions"));
}
if depth > MAX_FILTER_DEPTH {
return Err(eyre!("search filter is nested too deeply"));
}
let mut inner_hits_index = 0;
let query = plan(filter, &mut inner_hits_index)?;
Ok(ElasticsearchFilter {
query,
has_version_filter: inner_hits_index != 0,
})
}
fn plan(filter: &FilterExpr, inner_hits_index: &mut usize) -> Result<Value> {
match filter_scope(filter) {
FilterScope::Project => lower(filter),
FilterScope::Version => version_query(lower(filter)?, inner_hits_index),
FilterScope::Mixed => plan_mixed(filter, inner_hits_index),
}
}
fn plan_mixed(
filter: &FilterExpr,
inner_hits_index: &mut usize,
) -> Result<Value> {
match filter {
FilterExpr::Or(expressions) => expressions
.iter()
.map(|expression| plan(expression, inner_hits_index))
.collect::<Result<Vec<_>>>()
.map(or_query),
FilterExpr::And(expressions)
if expressions.iter().all(|expression| {
filter_scope(expression) != FilterScope::Mixed
}) =>
{
plan_partitioned_and(expressions, inner_hits_index)
}
_ => {
let clauses = to_dnf(filter)?;
clauses
.into_iter()
.map(|clause| plan_clause(clause, inner_hits_index))
.collect::<Result<Vec<_>>>()
.map(or_query)
}
}
}
fn plan_partitioned_and(
expressions: &[FilterExpr],
inner_hits_index: &mut usize,
) -> Result<Value> {
let mut project = Vec::new();
let mut version = Vec::new();
for expression in expressions {
match filter_scope(expression) {
FilterScope::Project => project.push(lower(expression)?),
FilterScope::Version => version.push(lower(expression)?),
FilterScope::Mixed => {
return Err(eyre!("could not partition mixed search filter"));
}
}
}
if !version.is_empty() {
project.push(version_query(and_query(version), inner_hits_index)?);
}
Ok(and_query(project))
}
fn plan_clause(
predicates: Vec<&FilterPredicate>,
inner_hits_index: &mut usize,
) -> Result<Value> {
let mut project = Vec::new();
let mut version = Vec::new();
for predicate in predicates {
let query = predicate_query(predicate)?;
if is_version_filter_field(predicate.field.as_str()) {
version.push(query);
} else {
project.push(query);
}
}
if !version.is_empty() {
project.push(version_query(and_query(version), inner_hits_index)?);
}
Ok(and_query(project))
}
fn lower(filter: &FilterExpr) -> Result<Value> {
match filter {
FilterExpr::And(expressions) => expressions
.iter()
.map(lower)
.collect::<Result<Vec<_>>>()
.map(and_query),
FilterExpr::Or(expressions) => expressions
.iter()
.map(lower)
.collect::<Result<Vec<_>>>()
.map(or_query),
FilterExpr::Predicate(predicate) => predicate_query(predicate),
FilterExpr::Not(_) => {
Err(eyre!("search filter contains an unnormalized negation"))
}
}
}
fn predicate_query(predicate: &FilterPredicate) -> Result<Value> {
let source_field = predicate.field.as_str();
let field = exact_field(source_field);
match &predicate.condition {
FilterCondition::Compare { comparison, value } => {
let value = literal_value(source_field, value)?;
Ok(match comparison {
FilterComparison::Equal => {
json!({"term": {(field): {"value": value}}})
}
FilterComparison::NotEqual => not_query(json!({
"term": {(field): {"value": value}}
})),
FilterComparison::GreaterThan => {
json!({"range": {(field): {"gt": value}}})
}
FilterComparison::GreaterThanOrEqual => {
json!({"range": {(field): {"gte": value}}})
}
FilterComparison::LessThan => {
json!({"range": {(field): {"lt": value}}})
}
FilterComparison::LessThanOrEqual => {
json!({"range": {(field): {"lte": value}}})
}
})
}
FilterCondition::In { values, negated } => {
let values = values
.iter()
.map(|value| literal_value(source_field, value))
.collect::<Result<Vec<_>>>()?;
let query = json!({"terms": {(field): values}});
Ok(if *negated { not_query(query) } else { query })
}
FilterCondition::Exists { negated } => {
let query = json!({"exists": {"field": field}});
Ok(if *negated { not_query(query) } else { query })
}
}
}
fn literal_value(field: &str, literal: &FilterLiteral) -> Result<Value> {
match literal {
FilterLiteral::String(value) if field == "author" => {
Ok(Value::String(normalize_for_search(value)))
}
FilterLiteral::String(value) => Ok(Value::String(value.clone())),
FilterLiteral::Number(value) => serde_json::from_str(value)
.map_err(|error| eyre!("invalid numeric filter literal: {error}")),
FilterLiteral::Bool(value) => Ok(Value::Bool(*value)),
}
}
fn exact_field(field: &str) -> &str {
match field {
"name" => "name.keyword",
"author" => "indexed_author.keyword",
"summary" => "summary.keyword",
"slug" => "slug.keyword",
_ => field,
}
}
fn version_query(query: Value, inner_hits_index: &mut usize) -> Result<Value> {
let name = format!("matching_versions_{}", *inner_hits_index);
*inner_hits_index += 1;
Ok(json!({
"has_child": {
"type": "version",
"score_mode": "none",
"query": query,
"inner_hits": {
"name": name,
"size": 1,
"_source": ["version_id", "version_published_timestamp"],
"sort": [
{"version_published_timestamp": {"order": "desc"}},
{"version_id": {"order": "desc"}}
]
}
}
}))
}
fn and_query(queries: Vec<Value>) -> Value {
match queries.len() {
0 => json!({"match_all": {}}),
1 => queries.into_iter().next().unwrap_or_default(),
_ => json!({"bool": {"filter": queries}}),
}
}
fn or_query(queries: Vec<Value>) -> Value {
match queries.len() {
0 => json!({"match_none": {}}),
1 => queries.into_iter().next().unwrap_or_default(),
_ => json!({
"bool": {
"should": queries,
"minimum_should_match": 1
}
}),
}
}
fn not_query(query: Value) -> Value {
json!({
"bool": {
"must": [{"match_all": {}}],
"must_not": [query]
}
})
}
fn filter_scope(filter: &FilterExpr) -> FilterScope {
match filter {
FilterExpr::Predicate(predicate) => {
if is_version_filter_field(predicate.field.as_str()) {
FilterScope::Version
} else {
FilterScope::Project
}
}
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
let mut scopes = expressions.iter().map(filter_scope);
let Some(first) = scopes.next() else {
return FilterScope::Project;
};
if scopes.all(|scope| scope == first) {
first
} else {
FilterScope::Mixed
}
}
FilterExpr::Not(expression) => filter_scope(expression),
}
}
fn is_version_filter_field(field: &str) -> bool {
matches!(
field,
"categories"
| "project_types"
| "environment"
| "game_versions"
| "client_side"
| "server_side"
)
}
fn to_dnf(filter: &FilterExpr) -> Result<Vec<Vec<&FilterPredicate>>> {
match filter {
FilterExpr::Predicate(predicate) => Ok(vec![vec![predicate]]),
FilterExpr::Or(expressions) => {
let mut clauses = Vec::new();
for expression in expressions.iter() {
clauses.extend(to_dnf(expression)?);
if clauses.len() > MAX_DNF_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
}
Ok(clauses)
}
FilterExpr::And(expressions) => {
let mut clauses = vec![Vec::new()];
for expression in expressions.iter() {
let right = to_dnf(expression)?;
if clauses.len().saturating_mul(right.len()) > MAX_DNF_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
clauses = clauses
.into_iter()
.flat_map(|left| {
right.iter().map(move |right| {
let mut clause = left.clone();
clause.extend(right);
clause
})
})
.collect();
}
Ok(clauses)
}
FilterExpr::Not(_) => {
Err(eyre!("search filter contains an unnormalized negation"))
}
}
}
fn filter_complexity(filter: &FilterExpr) -> (usize, usize) {
match filter {
FilterExpr::Predicate(_) => (1, 1),
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
expressions.iter().map(filter_complexity).fold(
(1, 1),
|(nodes, depth), (child_nodes, child_depth)| {
(nodes + child_nodes, depth.max(child_depth + 1))
},
)
}
FilterExpr::Not(expression) => {
let (nodes, depth) = filter_complexity(expression);
(nodes + 1, depth + 1)
}
}
}
#[cfg(test)]
mod tests {
use super::serialize_filter;
use crate::search::filter::{normalize, parse_expression};
use serde_json::Value;
fn serialize(input: &str) -> Value {
let filter = normalize(parse_expression(input).unwrap());
serialize_filter(&filter).unwrap().query
}
#[test]
fn correlated_version_filters_use_one_join() {
let query = serialize("categories = fabric AND game_versions = 1.21");
assert_eq!(query.to_string().matches("has_child").count(), 1);
}
#[test]
fn project_filters_do_not_use_a_join() {
let query = serialize("license = MIT");
assert_eq!(query.to_string().matches("has_child").count(), 0);
assert_eq!(query["term"]["license"]["value"], "MIT");
}
#[test]
fn author_filters_use_the_normalized_exact_field() {
let query = serialize("author = User");
assert_eq!(query["term"]["indexed_author.keyword"]["value"], "user");
}
#[test]
fn mixed_boolean_filters_preserve_version_correlation() {
let query = serialize(
"(license = MIT OR categories = fabric) AND game_versions = 1.21",
);
assert_eq!(query.to_string().matches("has_child").count(), 2);
}
}
File diff suppressed because it is too large Load Diff
+2
View File
@@ -1,8 +1,10 @@
mod common;
pub mod elasticsearch;
pub mod typesense;
pub use common::{
ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters,
parse_search_index, parse_search_request,
};
pub use elasticsearch::{Elasticsearch, ElasticsearchConfig};
pub use typesense::{Typesense, TypesenseConfig};
+1 -1
View File
@@ -45,7 +45,7 @@ struct PartialProject {
components: exp::ProjectSerial,
}
fn normalize_for_search(s: &str) -> String {
pub(crate) fn normalize_for_search(s: &str) -> String {
static SPECIAL_CHARS_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[^a-zA-Z0-9-.\s]").expect("valid regex"));
+6
View File
@@ -189,6 +189,7 @@ pub enum TasksCancelFilter {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SearchBackendKind {
Typesense,
Elasticsearch,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumIter)]
@@ -224,6 +225,7 @@ impl FromStr for SearchBackendKind {
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s {
"typesense" => SearchBackendKind::Typesense,
"elasticsearch" => SearchBackendKind::Elasticsearch,
_ => return Err(InvalidSearchBackendKind),
})
}
@@ -438,5 +440,9 @@ pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
let config = backend::TypesenseConfig::new(meta_namespace);
Box::new(backend::Typesense::new(config))
}
SearchBackendKind::Elasticsearch => {
let config = backend::ElasticsearchConfig::new(meta_namespace);
Box::new(backend::Elasticsearch::new(config))
}
}
}
+99 -1
View File
@@ -32,6 +32,102 @@ services:
interval: 3s
timeout: 5s
retries: 3
elasticsearch0:
image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4
container_name: labrinth-elasticsearch0
restart: on-failure
networks:
- elasticsearch-mesh
ports:
- '127.0.0.1:9200:9200'
volumes:
- elasticsearch0-data:/usr/share/elasticsearch/data
environment:
node.name: elasticsearch0
cluster.name: labrinth-elasticsearch
discovery.seed_hosts: elasticsearch0,elasticsearch1,elasticsearch2
cluster.initial_master_nodes: elasticsearch0,elasticsearch1,elasticsearch2
bootstrap.memory_lock: 'true'
xpack.security.enabled: 'false'
xpack.security.enrollment.enabled: 'false'
ES_JAVA_OPTS: -Xms512m -Xmx512m
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
'CMD-SHELL',
'curl --fail http://localhost:9200/_cluster/health?wait_for_status=yellow',
]
interval: 5s
timeout: 5s
retries: 30
elasticsearch1:
image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4
container_name: labrinth-elasticsearch1
restart: on-failure
networks:
- elasticsearch-mesh
ports:
- '127.0.0.1:9201:9200'
volumes:
- elasticsearch1-data:/usr/share/elasticsearch/data
environment:
node.name: elasticsearch1
cluster.name: labrinth-elasticsearch
discovery.seed_hosts: elasticsearch0,elasticsearch1,elasticsearch2
cluster.initial_master_nodes: elasticsearch0,elasticsearch1,elasticsearch2
bootstrap.memory_lock: 'true'
xpack.security.enabled: 'false'
xpack.security.enrollment.enabled: 'false'
ES_JAVA_OPTS: -Xms512m -Xmx512m
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
'CMD-SHELL',
'curl --fail http://localhost:9200/_cluster/health?wait_for_status=yellow',
]
interval: 5s
timeout: 5s
retries: 30
elasticsearch2:
image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4
container_name: labrinth-elasticsearch2
restart: on-failure
networks:
- elasticsearch-mesh
ports:
- '127.0.0.1:9202:9200'
volumes:
- elasticsearch2-data:/usr/share/elasticsearch/data
environment:
node.name: elasticsearch2
cluster.name: labrinth-elasticsearch
discovery.seed_hosts: elasticsearch0,elasticsearch1,elasticsearch2
cluster.initial_master_nodes: elasticsearch0,elasticsearch1,elasticsearch2
bootstrap.memory_lock: 'true'
xpack.security.enabled: 'false'
xpack.security.enrollment.enabled: 'false'
ES_JAVA_OPTS: -Xms512m -Xmx512m
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
'CMD-SHELL',
'curl --fail http://localhost:9200/_cluster/health?wait_for_status=yellow',
]
interval: 5s
timeout: 5s
retries: 30
meilisearch0:
image: getmeili/meilisearch:v1.12.0
container_name: labrinth-meilisearch0
@@ -398,7 +494,7 @@ services:
depends_on:
postgres_db:
condition: service_healthy
meilisearch:
meilisearch0:
condition: service_healthy
elasticsearch0:
condition: service_healthy
@@ -481,6 +577,8 @@ services:
volumes:
- ./apps/labrinth/nginx/meili-lb.conf:/etc/nginx/conf.d/default.conf:ro
networks:
elasticsearch-mesh:
driver: bridge
meilisearch-mesh:
driver: bridge
redis-cluster-mesh:
@@ -111,33 +111,7 @@
<span class="text-lg font-bold text-primary">
{{ formatMessage(messages.authProvidersLabel) }}
</span>
<ul class="flex flex-col gap-1 pl-4 leading-normal m-0">
<li v-for="provider in user.auth_providers ?? []" :key="provider">
<span>{{ authProviderNames[provider] ?? provider }}</span>
<span v-if="provider === 'discord' && user.discord_id" class="ml-1">
({{ user.discord_id }})
</span>
<template v-else-if="provider === 'github' && user.github_id">
<span class="ml-1">(</span>
<button
type="button"
class="m-0 appearance-none border-0 bg-transparent p-0 font-[inherit] text-link disabled:cursor-wait disabled:opacity-70"
:disabled="isLoadingGithubProfile"
@click="openGithubProfile"
>
{{
isLoadingGithubProfile
? formatMessage(messages.loadingGithubProfileLabel)
: formatMessage(messages.viewGithubProfileLabel)
}}
</button>
<span>)</span>
</template>
<span v-else-if="provider === 'steam' && user.steam_id" class="ml-1">
({{ user.steam_id }})
</span>
</li>
</ul>
<span>{{ user.auth_providers?.join(', ') || '—' }}</span>
</div>
<div v-if="isAdminViewing" class="flex flex-col gap-1">
@@ -486,13 +460,7 @@ import ProjectCardList from '#ui/components/project/ProjectCardList.vue'
import UserBadges from '#ui/components/user/UserBadges.vue'
import UserPageHeader from '#ui/components/user/UserPageHeader.vue'
import { defineMessages, useVIntl } from '#ui/composables'
import {
injectAuth,
injectModrinthClient,
injectNotificationManager,
injectPageContext,
injectTags,
} from '#ui/providers'
import { injectAuth, injectNotificationManager, injectPageContext, injectTags } from '#ui/providers'
import { commonMessages, getProjectTypeTitleMessage } from '#ui/utils'
import { blockedUsersQueryKey, injectUserProfile } from './providers'
@@ -552,7 +520,6 @@ const auth = injectAuth()
const tags = injectTags(null)
const pageContext = injectPageContext()
const notificationManager = injectNotificationManager()
const client = injectModrinthClient()
const queryClient = useQueryClient()
const route = useRoute()
const router = useRouter()
@@ -600,26 +567,6 @@ const messages = defineMessages({
id: 'profile.details.label.auth-providers',
defaultMessage: 'Auth providers',
},
viewGithubProfileLabel: {
id: 'profile.details.label.view-github-profile',
defaultMessage: 'View profile',
},
loadingGithubProfileLabel: {
id: 'profile.details.label.loading-github-profile',
defaultMessage: 'Loading...',
},
githubProfileErrorTitle: {
id: 'profile.details.error.github-profile-title',
defaultMessage: 'Unable to open GitHub profile',
},
githubProfileErrorMessage: {
id: 'profile.details.error.github-profile-message',
defaultMessage: 'The GitHub profile could not be retrieved. Please try again.',
},
githubPopupBlockedMessage: {
id: 'profile.details.error.github-popup-blocked',
defaultMessage: 'Allow pop-ups for Modrinth, then try again.',
},
paymentMethodsLabel: {
id: 'profile.details.label.payment-methods',
defaultMessage: 'Payment methods',
@@ -906,17 +853,6 @@ const showCollectionsEmptyState = computed(
const normalizedSiteUrl = computed(() => props.siteUrl.replace(/\/$/, ''))
const editProfileLink = computed(() => props.editProfileLink ?? linkTarget('/settings/profile'))
const authProviderNames = {
github: 'GitHub',
discord: 'Discord',
microsoft: 'Microsoft',
gitlab: 'GitLab',
google: 'Google',
steam: 'Steam',
paypal: 'PayPal',
}
const isLoadingGithubProfile = ref(false)
function externalUrl(path: string): string {
return `${normalizedSiteUrl.value}${path.startsWith('/') ? path : `/${path}`}`
}
@@ -960,50 +896,6 @@ async function copyPermalink(): Promise<void> {
}
}
async function openGithubProfile() {
const githubId = user.value?.github_id
if (!githubId || isLoadingGithubProfile.value) return
const profileWindow = window.open('about:blank', '_blank')
if (!profileWindow) {
notificationManager.addNotification({
type: 'error',
title: formatMessage(messages.githubProfileErrorTitle),
text: formatMessage(messages.githubPopupBlockedMessage),
})
return
}
profileWindow.opener = null
isLoadingGithubProfile.value = true
try {
const githubUser = await client.request<{ login?: string }>(`/${githubId}`, {
api: 'https://api.github.com',
version: 'user',
method: 'GET',
headers: { 'Content-Type': '' },
skipAuth: true,
})
if (!githubUser?.login) {
throw new Error('GitHub user response did not include a login')
}
profileWindow.location.replace(`https://github.com/${encodeURIComponent(githubUser.login)}`)
} catch (error) {
profileWindow.close()
console.error('Failed to retrieve GitHub profile:', error)
notificationManager.addNotification({
type: 'error',
title: formatMessage(messages.githubProfileErrorTitle),
text: formatMessage(messages.githubProfileErrorMessage),
})
} finally {
isLoadingGithubProfile.value = false
}
}
function reportProfile(): void {
if (!user.value) return
const reportPath = `/report?item=user&itemID=${encodeURIComponent(user.value.id)}`
-15
View File
@@ -2921,15 +2921,6 @@
"profile.collection.projects-count": {
"defaultMessage": "{count, plural, one {# project} other {# projects}}"
},
"profile.details.error.github-popup-blocked": {
"defaultMessage": "Allow pop-ups for Modrinth, then try again."
},
"profile.details.error.github-profile-message": {
"defaultMessage": "The GitHub profile could not be retrieved. Please try again."
},
"profile.details.error.github-profile-title": {
"defaultMessage": "Unable to open GitHub profile"
},
"profile.details.label.auth-providers": {
"defaultMessage": "Auth providers"
},
@@ -2942,15 +2933,9 @@
"profile.details.label.has-totp": {
"defaultMessage": "Has TOTP"
},
"profile.details.label.loading-github-profile": {
"defaultMessage": "Loading..."
},
"profile.details.label.payment-methods": {
"defaultMessage": "Payment methods"
},
"profile.details.label.view-github-profile": {
"defaultMessage": "View profile"
},
"profile.details.title": {
"defaultMessage": "User details"
},