mirror of
https://github.com/modrinth/code.git
synced 2026-08-31 12:05:53 +00:00
aecsocket
b3b0b85691
* use ro_pool for search indexing again * adjust incremental indexing operations
491 lines
16 KiB
Rust
491 lines
16 KiB
Rust
use crate::models::exp;
|
|
use crate::models::exp::minecraft::JavaServerPing;
|
|
use crate::models::ids::{ProjectId, VersionId};
|
|
use crate::models::projects::DependencyType;
|
|
use crate::queue::server_ping;
|
|
use crate::routes::ApiError;
|
|
use crate::util::error::ApiContext as _;
|
|
use crate::util::error::Context as _;
|
|
use crate::{database::PgPool, env::ENV};
|
|
use ariadne::ids::base62_impl::parse_base62;
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::Value;
|
|
use std::{collections::HashMap, str::FromStr, sync::Arc};
|
|
use thiserror::Error;
|
|
use utoipa::ToSchema;
|
|
use xredis::RedisPool;
|
|
|
|
pub mod backend;
|
|
pub mod filter;
|
|
pub mod incremental;
|
|
pub mod indexing;
|
|
|
|
#[derive(Clone)]
|
|
pub struct SearchState {
|
|
pub backend: Arc<dyn SearchBackend>,
|
|
pub queue: incremental::IncrementalSearchQueue,
|
|
}
|
|
|
|
/// Search parameters which can fit in a URL query string.
|
|
///
|
|
/// Used with `GET /*/search` endpoints.
|
|
///
|
|
/// Can be converted into a [`SearchRequest`] using [`From`].
|
|
#[derive(Serialize, Deserialize, Debug)]
|
|
pub struct SearchQuery {
|
|
pub query: Option<String>,
|
|
pub offset: Option<String>,
|
|
pub index: Option<String>,
|
|
pub limit: Option<String>,
|
|
|
|
pub new_filters: Option<String>,
|
|
|
|
// TODO: Deprecated values below. WILL BE REMOVED V3!
|
|
pub facets: Option<String>,
|
|
pub filters: Option<String>,
|
|
pub version: Option<String>,
|
|
}
|
|
|
|
/// Search parameters which are more complicated and more suitable for a POST
|
|
/// request body.
|
|
///
|
|
/// Used with `POST /*/search` endpoints.
|
|
///
|
|
/// Can be converted from a [`SearchQuery`] using [`From`].
|
|
#[derive(Serialize, Deserialize, Debug)]
|
|
pub struct SearchRequest {
|
|
pub query: Option<String>,
|
|
pub offset: Option<String>,
|
|
pub index: Option<String>,
|
|
pub limit: Option<String>,
|
|
#[serde(default)]
|
|
pub show_metadata: bool,
|
|
#[serde(default)]
|
|
pub typesense_config: backend::typesense::RequestConfig,
|
|
|
|
pub new_filters: Option<String>,
|
|
|
|
pub facets: Option<String>,
|
|
pub filters: Option<String>,
|
|
pub version: Option<String>,
|
|
}
|
|
|
|
impl From<SearchQuery> for SearchRequest {
|
|
fn from(query: SearchQuery) -> Self {
|
|
Self {
|
|
query: query.query,
|
|
offset: query.offset,
|
|
index: query.index,
|
|
limit: query.limit,
|
|
show_metadata: false,
|
|
typesense_config: backend::typesense::RequestConfig::default(),
|
|
new_filters: query.new_filters,
|
|
facets: query.facets,
|
|
filters: query.filters,
|
|
version: query.version,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
pub trait SearchBackend: Send + Sync {
|
|
async fn search_for_project(
|
|
&self,
|
|
info: &SearchRequest,
|
|
redis: &RedisPool,
|
|
) -> Result<SearchResults, ApiError> {
|
|
let mut results = self
|
|
.search_for_project_raw(info)
|
|
.await
|
|
.wrap_api_err("searching projects")?;
|
|
hydrate_search_results(&mut results.hits, redis)
|
|
.await
|
|
.wrap_internal_err("hydrating search results from database")?;
|
|
Ok(results)
|
|
}
|
|
|
|
async fn search_for_project_raw(
|
|
&self,
|
|
info: &SearchRequest,
|
|
) -> Result<SearchResults, ApiError>;
|
|
|
|
async fn rebuild_index(
|
|
&self,
|
|
ro_pool: PgPool,
|
|
redis: RedisPool,
|
|
) -> eyre::Result<()>;
|
|
|
|
async fn apply_update(
|
|
&self,
|
|
update: SearchIndexUpdate<'_>,
|
|
) -> eyre::Result<()>;
|
|
|
|
async fn tasks(&self) -> eyre::Result<Value>;
|
|
|
|
async fn tasks_cancel(
|
|
&self,
|
|
filter: &TasksCancelFilter,
|
|
) -> eyre::Result<()>;
|
|
}
|
|
|
|
async fn hydrate_search_results(
|
|
hits: &mut [ResultSearchProject],
|
|
redis_pool: &RedisPool,
|
|
) -> eyre::Result<()> {
|
|
// Minecraft Java servers should fetch the latest player count that we have
|
|
// from Redis, rather than the (pretty stale) data from search backend
|
|
// TODO: this block should be made generic over the component type,
|
|
// for now we can hardcode MC java servers tho
|
|
|
|
let project_ids = hits
|
|
.iter()
|
|
.filter(|hit| hit.components.minecraft_java_server.is_some())
|
|
.filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId))
|
|
.collect::<Vec<_>>();
|
|
|
|
let pings_by_project_id = if project_ids.is_empty() {
|
|
HashMap::new()
|
|
} else {
|
|
let mut redis = redis_pool.connect().await?;
|
|
let ping_keys = project_ids
|
|
.iter()
|
|
.map(|project_id| {
|
|
redis_pool
|
|
.key()
|
|
.entity(server_ping::REDIS_NAMESPACE, project_id)
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let ping_results = redis
|
|
.get_many_deserialized::<JavaServerPing>(&ping_keys)
|
|
.await?;
|
|
|
|
ping_results
|
|
.into_iter()
|
|
.enumerate()
|
|
.filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping)))
|
|
.collect::<HashMap<_, _>>()
|
|
};
|
|
|
|
for hit in hits {
|
|
let Some(java_server) = hit.components.minecraft_java_server.as_mut()
|
|
else {
|
|
continue;
|
|
};
|
|
if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) {
|
|
java_server.ping = pings_by_project_id.get(&project_id).cloned();
|
|
} else {
|
|
java_server.ping = None;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Deserialize, Serialize, ToSchema)]
|
|
#[serde(tag = "type", rename_all = "snake_case")]
|
|
pub enum TasksCancelFilter {
|
|
All,
|
|
AllEnqueued,
|
|
Indexes { indexes: Vec<String> },
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
|
pub enum SearchBackendKind {
|
|
Typesense,
|
|
Elasticsearch,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumIter)]
|
|
pub enum SearchField {
|
|
Categories,
|
|
Name,
|
|
Author,
|
|
License,
|
|
ProjectTypes,
|
|
AllProjectTypes,
|
|
ProjectId,
|
|
OpenSource,
|
|
Environment,
|
|
GameVersions,
|
|
ClientSide,
|
|
ServerSide,
|
|
MinecraftServerRegion,
|
|
MinecraftServerLanguages,
|
|
MinecraftJavaServerContentKind,
|
|
MinecraftJavaServerContentSupportedGameVersions,
|
|
MinecraftJavaServerPingData,
|
|
DependencyProjectIds,
|
|
CompatibleDependencyProjectIds,
|
|
DisclosureTypes,
|
|
RequiredDependencyProjectIds,
|
|
OptionalDependencyProjectIds,
|
|
EmbeddedDependencyProjectIds,
|
|
IncompatibleDependencyProjectIds,
|
|
}
|
|
|
|
#[derive(Debug, Error)]
|
|
#[error("invalid search backend kind")]
|
|
pub struct InvalidSearchBackendKind;
|
|
|
|
impl FromStr for SearchBackendKind {
|
|
type Err = InvalidSearchBackendKind;
|
|
|
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
|
Ok(match s {
|
|
"typesense" => SearchBackendKind::Typesense,
|
|
"elasticsearch" => SearchBackendKind::Elasticsearch,
|
|
_ => return Err(InvalidSearchBackendKind),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Nullable fields in Typesense-bound documents should use
|
|
/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
|
|
/// serialized as `null`.
|
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
|
pub struct UploadSearchProject {
|
|
/// ID of the most recently published version.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub version_id: Option<String>,
|
|
pub project_id: String,
|
|
//
|
|
pub project_types: Vec<String>,
|
|
#[serde(default)]
|
|
pub all_project_types: Vec<String>,
|
|
pub slug: Option<String>,
|
|
pub author: String,
|
|
pub author_id: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub organization: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub organization_id: Option<String>,
|
|
pub indexed_author: String,
|
|
pub name: String,
|
|
pub indexed_name: String,
|
|
pub summary: String,
|
|
pub categories: Vec<String>,
|
|
pub project_categories: Vec<String>,
|
|
pub display_categories: Vec<String>,
|
|
pub follows: i32,
|
|
pub downloads: i32,
|
|
pub log_downloads: f64,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub icon_url: Option<String>,
|
|
pub license: String,
|
|
pub gallery: Vec<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub featured_gallery: Option<String>,
|
|
/// RFC 3339 formatted creation date of the project
|
|
pub date_created: DateTime<Utc>,
|
|
/// Unix timestamp of the creation date of the project
|
|
pub created_timestamp: i64,
|
|
/// RFC 3339 formatted date/time of last major modification (update)
|
|
pub date_modified: DateTime<Utc>,
|
|
/// Unix timestamp of the last major modification
|
|
pub modified_timestamp: i64,
|
|
/// Unix timestamp of the most recently published version.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub version_published_timestamp: Option<i64>,
|
|
pub open_source: bool,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub color: Option<u32>,
|
|
#[serde(default)]
|
|
pub dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub compatible_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub required_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub optional_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub embedded_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub incompatible_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub dependencies: Vec<SearchProjectDependency>,
|
|
#[serde(default)]
|
|
pub disclosure_types: Vec<String>,
|
|
|
|
// Hidden fields to get the Project model out of the search results.
|
|
pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
|
|
pub project_loader_fields: HashMap<String, Vec<serde_json::Value>>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model.
|
|
|
|
#[serde(flatten)]
|
|
pub components: exp::ProjectQuery,
|
|
#[serde(flatten)]
|
|
pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
|
pub struct UploadSearchVersion {
|
|
pub version_id: String,
|
|
pub project_id: String,
|
|
pub categories: Vec<String>,
|
|
pub project_types: Vec<String>,
|
|
pub version_published_timestamp: i64,
|
|
#[serde(flatten)]
|
|
pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
pub struct SearchDocumentBatch {
|
|
pub projects: Vec<UploadSearchProject>,
|
|
pub versions: Vec<UploadSearchVersion>,
|
|
}
|
|
|
|
/// A logical search index mutation. Removals are applied before replacements,
|
|
/// so a document may be present in both a removed and replacement field.
|
|
#[derive(Debug, Clone, Copy, Default)]
|
|
pub struct SearchIndexUpdate<'a> {
|
|
pub projects: &'a [UploadSearchProject],
|
|
pub versions: &'a [UploadSearchVersion],
|
|
/// Projects and all of their version documents to remove.
|
|
pub removed_projects: &'a [ProjectId],
|
|
pub removed_versions: &'a [VersionId],
|
|
}
|
|
|
|
/// Nullable fields in Typesense-bound documents should use
|
|
/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of
|
|
/// serialized as `null`.
|
|
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
|
pub struct SearchProjectDependency {
|
|
pub project_id: String,
|
|
pub dependency_type: DependencyType,
|
|
pub name: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub slug: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub icon_url: Option<String>,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Debug, ToSchema)]
|
|
pub struct SearchResults {
|
|
pub hits: Vec<ResultSearchProject>,
|
|
pub page: usize,
|
|
pub hits_per_page: usize,
|
|
pub total_hits: usize,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
|
|
pub struct ResultSearchProject {
|
|
/// ID of the most recently published version.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub version_id: Option<String>,
|
|
pub project_id: String,
|
|
pub project_types: Vec<String>,
|
|
#[serde(default)]
|
|
pub all_project_types: Vec<String>,
|
|
pub slug: Option<String>,
|
|
pub author: String,
|
|
#[serde(default)]
|
|
pub author_id: Option<String>,
|
|
#[serde(default)]
|
|
pub organization: Option<String>,
|
|
#[serde(default)]
|
|
pub organization_id: Option<String>,
|
|
pub name: String,
|
|
pub summary: String,
|
|
pub categories: Vec<String>,
|
|
pub display_categories: Vec<String>,
|
|
pub downloads: i32,
|
|
pub follows: i32,
|
|
pub icon_url: Option<String>,
|
|
/// RFC 3339 formatted creation date of the project
|
|
pub date_created: String,
|
|
/// RFC 3339 formatted modification date of the project
|
|
pub date_modified: String,
|
|
pub license: String,
|
|
pub gallery: Vec<String>,
|
|
pub featured_gallery: Option<String>,
|
|
pub color: Option<u32>,
|
|
#[serde(default)]
|
|
pub dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub compatible_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub required_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub optional_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub embedded_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub incompatible_dependency_project_ids: Vec<String>,
|
|
#[serde(default)]
|
|
pub dependencies: Vec<SearchProjectDependency>,
|
|
#[serde(default)]
|
|
pub disclosure_types: Vec<String>,
|
|
|
|
// Hidden fields to get the Project model out of the search results.
|
|
pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
|
|
pub project_loader_fields: HashMap<String, Vec<serde_json::Value>>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model.
|
|
|
|
#[serde(flatten)]
|
|
pub components: exp::ProjectQuery,
|
|
#[serde(flatten)]
|
|
pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub search_metadata: Option<Value>,
|
|
}
|
|
|
|
impl From<UploadSearchProject> for ResultSearchProject {
|
|
fn from(source: UploadSearchProject) -> Self {
|
|
Self {
|
|
version_id: source.version_id,
|
|
project_id: source.project_id,
|
|
project_types: source.project_types,
|
|
all_project_types: source.all_project_types,
|
|
slug: source.slug,
|
|
author: source.author,
|
|
author_id: Some(source.author_id),
|
|
organization: source.organization,
|
|
organization_id: source.organization_id,
|
|
name: source.name,
|
|
summary: source.summary,
|
|
categories: source.categories,
|
|
display_categories: source.display_categories,
|
|
downloads: source.downloads,
|
|
follows: source.follows,
|
|
icon_url: source.icon_url,
|
|
date_created: source.date_created.to_rfc3339(),
|
|
date_modified: source.date_modified.to_rfc3339(),
|
|
license: source.license,
|
|
gallery: source.gallery,
|
|
featured_gallery: source.featured_gallery,
|
|
color: source.color,
|
|
dependency_project_ids: source.dependency_project_ids,
|
|
compatible_dependency_project_ids: source
|
|
.compatible_dependency_project_ids,
|
|
required_dependency_project_ids: source
|
|
.required_dependency_project_ids,
|
|
optional_dependency_project_ids: source
|
|
.optional_dependency_project_ids,
|
|
embedded_dependency_project_ids: source
|
|
.embedded_dependency_project_ids,
|
|
incompatible_dependency_project_ids: source
|
|
.incompatible_dependency_project_ids,
|
|
dependencies: source.dependencies,
|
|
disclosure_types: source.disclosure_types,
|
|
loaders: source.loaders,
|
|
project_loader_fields: source.project_loader_fields,
|
|
components: source.components,
|
|
loader_fields: source.loader_fields,
|
|
search_metadata: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
|
|
match ENV.SEARCH_BACKEND {
|
|
SearchBackendKind::Typesense => {
|
|
let config = backend::TypesenseConfig::new(meta_namespace);
|
|
Box::new(backend::Typesense::new(config))
|
|
}
|
|
SearchBackendKind::Elasticsearch => {
|
|
let config = backend::ElasticsearchConfig::new(meta_namespace);
|
|
Box::new(backend::Elasticsearch::new(config))
|
|
}
|
|
}
|
|
}
|