use crate::models::exp; use crate::models::exp::minecraft::JavaServerPing; use crate::models::ids::{ProjectId, VersionId}; use crate::models::projects::DependencyType; use crate::queue::server_ping; use crate::routes::ApiError; use crate::util::error::ApiContext as _; use crate::util::error::Context as _; use crate::{database::PgPool, env::ENV}; use ariadne::ids::base62_impl::parse_base62; use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{collections::HashMap, str::FromStr, sync::Arc}; use thiserror::Error; use utoipa::ToSchema; use xredis::RedisPool; pub mod backend; pub mod filter; pub mod incremental; pub mod indexing; #[derive(Clone)] pub struct SearchState { pub backend: Arc, pub queue: incremental::IncrementalSearchQueue, } /// Search parameters which can fit in a URL query string. /// /// Used with `GET /*/search` endpoints. /// /// Can be converted into a [`SearchRequest`] using [`From`]. #[derive(Serialize, Deserialize, Debug)] pub struct SearchQuery { pub query: Option, pub offset: Option, pub index: Option, pub limit: Option, pub new_filters: Option, // TODO: Deprecated values below. WILL BE REMOVED V3! pub facets: Option, pub filters: Option, pub version: Option, } /// Search parameters which are more complicated and more suitable for a POST /// request body. /// /// Used with `POST /*/search` endpoints. /// /// Can be converted from a [`SearchQuery`] using [`From`]. #[derive(Serialize, Deserialize, Debug)] pub struct SearchRequest { pub query: Option, pub offset: Option, pub index: Option, pub limit: Option, #[serde(default)] pub show_metadata: bool, #[serde(default)] pub typesense_config: backend::typesense::RequestConfig, pub new_filters: Option, pub facets: Option, pub filters: Option, pub version: Option, } impl From for SearchRequest { fn from(query: SearchQuery) -> Self { Self { query: query.query, offset: query.offset, index: query.index, limit: query.limit, show_metadata: false, typesense_config: backend::typesense::RequestConfig::default(), new_filters: query.new_filters, facets: query.facets, filters: query.filters, version: query.version, } } } #[async_trait] pub trait SearchBackend: Send + Sync { async fn search_for_project( &self, info: &SearchRequest, redis: &RedisPool, ) -> Result { let mut results = self .search_for_project_raw(info) .await .wrap_api_err("searching projects")?; hydrate_search_results(&mut results.hits, redis) .await .wrap_internal_err("hydrating search results from database")?; Ok(results) } async fn search_for_project_raw( &self, info: &SearchRequest, ) -> Result; async fn rebuild_index( &self, ro_pool: PgPool, redis: RedisPool, ) -> eyre::Result<()>; async fn apply_update( &self, update: SearchIndexUpdate<'_>, ) -> eyre::Result<()>; async fn tasks(&self) -> eyre::Result; async fn tasks_cancel( &self, filter: &TasksCancelFilter, ) -> eyre::Result<()>; } async fn hydrate_search_results( hits: &mut [ResultSearchProject], redis_pool: &RedisPool, ) -> eyre::Result<()> { // Minecraft Java servers should fetch the latest player count that we have // from Redis, rather than the (pretty stale) data from search backend // TODO: this block should be made generic over the component type, // for now we can hardcode MC java servers tho let project_ids = hits .iter() .filter(|hit| hit.components.minecraft_java_server.is_some()) .filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId)) .collect::>(); let pings_by_project_id = if project_ids.is_empty() { HashMap::new() } else { let mut redis = redis_pool.connect().await?; let ping_keys = project_ids .iter() .map(|project_id| { redis_pool .key() .entity(server_ping::REDIS_NAMESPACE, project_id) }) .collect::>(); let ping_results = redis .get_many_deserialized::(&ping_keys) .await?; ping_results .into_iter() .enumerate() .filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping))) .collect::>() }; for hit in hits { let Some(java_server) = hit.components.minecraft_java_server.as_mut() else { continue; }; if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) { java_server.ping = pings_by_project_id.get(&project_id).cloned(); } else { java_server.ping = None; } } Ok(()) } #[derive(Deserialize, Serialize, ToSchema)] #[serde(tag = "type", rename_all = "snake_case")] pub enum TasksCancelFilter { All, AllEnqueued, Indexes { indexes: Vec }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum SearchBackendKind { Typesense, Elasticsearch, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumIter)] pub enum SearchField { Categories, Name, Author, License, ProjectTypes, AllProjectTypes, ProjectId, OpenSource, Environment, GameVersions, ClientSide, ServerSide, MinecraftServerRegion, MinecraftServerLanguages, MinecraftJavaServerContentKind, MinecraftJavaServerContentSupportedGameVersions, MinecraftJavaServerPingData, DependencyProjectIds, CompatibleDependencyProjectIds, DisclosureTypes, RequiredDependencyProjectIds, OptionalDependencyProjectIds, EmbeddedDependencyProjectIds, IncompatibleDependencyProjectIds, } #[derive(Debug, Error)] #[error("invalid search backend kind")] pub struct InvalidSearchBackendKind; impl FromStr for SearchBackendKind { type Err = InvalidSearchBackendKind; fn from_str(s: &str) -> Result { Ok(match s { "typesense" => SearchBackendKind::Typesense, "elasticsearch" => SearchBackendKind::Elasticsearch, _ => return Err(InvalidSearchBackendKind), }) } } /// Nullable fields in Typesense-bound documents should use /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of /// serialized as `null`. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchProject { /// ID of the most recently published version. #[serde(skip_serializing_if = "Option::is_none")] pub version_id: Option, pub project_id: String, // pub project_types: Vec, #[serde(default)] pub all_project_types: Vec, pub slug: Option, pub author: String, pub author_id: String, #[serde(skip_serializing_if = "Option::is_none")] pub organization: Option, #[serde(skip_serializing_if = "Option::is_none")] pub organization_id: Option, pub indexed_author: String, pub name: String, pub indexed_name: String, pub summary: String, pub categories: Vec, pub project_categories: Vec, pub display_categories: Vec, pub follows: i32, pub downloads: i32, pub log_downloads: f64, #[serde(skip_serializing_if = "Option::is_none")] pub icon_url: Option, pub license: String, pub gallery: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub featured_gallery: Option, /// RFC 3339 formatted creation date of the project pub date_created: DateTime, /// Unix timestamp of the creation date of the project pub created_timestamp: i64, /// RFC 3339 formatted date/time of last major modification (update) pub date_modified: DateTime, /// Unix timestamp of the last major modification pub modified_timestamp: i64, /// Unix timestamp of the most recently published version. #[serde(skip_serializing_if = "Option::is_none")] pub version_published_timestamp: Option, pub open_source: bool, #[serde(skip_serializing_if = "Option::is_none")] pub color: Option, #[serde(default)] pub dependency_project_ids: Vec, #[serde(default)] pub compatible_dependency_project_ids: Vec, #[serde(default)] pub required_dependency_project_ids: Vec, #[serde(default)] pub optional_dependency_project_ids: Vec, #[serde(default)] pub embedded_dependency_project_ids: Vec, #[serde(default)] pub incompatible_dependency_project_ids: Vec, #[serde(default)] pub dependencies: Vec, #[serde(default)] pub disclosure_types: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. pub project_loader_fields: HashMap>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model. #[serde(flatten)] pub components: exp::ProjectQuery, #[serde(flatten)] pub loader_fields: HashMap>, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchVersion { pub version_id: String, pub project_id: String, pub categories: Vec, pub project_types: Vec, pub version_published_timestamp: i64, #[serde(flatten)] pub loader_fields: HashMap>, } #[derive(Debug, Default)] pub struct SearchDocumentBatch { pub projects: Vec, pub versions: Vec, } /// A logical search index mutation. Removals are applied before replacements, /// so a document may be present in both a removed and replacement field. #[derive(Debug, Clone, Copy, Default)] pub struct SearchIndexUpdate<'a> { pub projects: &'a [UploadSearchProject], pub versions: &'a [UploadSearchVersion], /// Projects and all of their version documents to remove. pub removed_projects: &'a [ProjectId], pub removed_versions: &'a [VersionId], } /// Nullable fields in Typesense-bound documents should use /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of /// serialized as `null`. #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] pub struct SearchProjectDependency { pub project_id: String, pub dependency_type: DependencyType, pub name: String, #[serde(skip_serializing_if = "Option::is_none")] pub slug: Option, #[serde(skip_serializing_if = "Option::is_none")] pub icon_url: Option, } #[derive(Serialize, Deserialize, Debug, ToSchema)] pub struct SearchResults { pub hits: Vec, pub page: usize, pub hits_per_page: usize, pub total_hits: usize, } #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] pub struct ResultSearchProject { /// ID of the most recently published version. #[serde(default, skip_serializing_if = "Option::is_none")] pub version_id: Option, pub project_id: String, pub project_types: Vec, #[serde(default)] pub all_project_types: Vec, pub slug: Option, pub author: String, #[serde(default)] pub author_id: Option, #[serde(default)] pub organization: Option, #[serde(default)] pub organization_id: Option, pub name: String, pub summary: String, pub categories: Vec, pub display_categories: Vec, pub downloads: i32, pub follows: i32, pub icon_url: Option, /// RFC 3339 formatted creation date of the project pub date_created: String, /// RFC 3339 formatted modification date of the project pub date_modified: String, pub license: String, pub gallery: Vec, pub featured_gallery: Option, pub color: Option, #[serde(default)] pub dependency_project_ids: Vec, #[serde(default)] pub compatible_dependency_project_ids: Vec, #[serde(default)] pub required_dependency_project_ids: Vec, #[serde(default)] pub optional_dependency_project_ids: Vec, #[serde(default)] pub embedded_dependency_project_ids: Vec, #[serde(default)] pub incompatible_dependency_project_ids: Vec, #[serde(default)] pub dependencies: Vec, #[serde(default)] pub disclosure_types: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. pub project_loader_fields: HashMap>, // Aggregation of loader_fields from all versions of the project, allowing for reconstruction of the Project model. #[serde(flatten)] pub components: exp::ProjectQuery, #[serde(flatten)] pub loader_fields: HashMap>, #[serde(skip_serializing_if = "Option::is_none")] pub search_metadata: Option, } impl From for ResultSearchProject { fn from(source: UploadSearchProject) -> Self { Self { version_id: source.version_id, project_id: source.project_id, project_types: source.project_types, all_project_types: source.all_project_types, slug: source.slug, author: source.author, author_id: Some(source.author_id), organization: source.organization, organization_id: source.organization_id, name: source.name, summary: source.summary, categories: source.categories, display_categories: source.display_categories, downloads: source.downloads, follows: source.follows, icon_url: source.icon_url, date_created: source.date_created.to_rfc3339(), date_modified: source.date_modified.to_rfc3339(), license: source.license, gallery: source.gallery, featured_gallery: source.featured_gallery, color: source.color, dependency_project_ids: source.dependency_project_ids, compatible_dependency_project_ids: source .compatible_dependency_project_ids, required_dependency_project_ids: source .required_dependency_project_ids, optional_dependency_project_ids: source .optional_dependency_project_ids, embedded_dependency_project_ids: source .embedded_dependency_project_ids, incompatible_dependency_project_ids: source .incompatible_dependency_project_ids, dependencies: source.dependencies, disclosure_types: source.disclosure_types, loaders: source.loaders, project_loader_fields: source.project_loader_fields, components: source.components, loader_fields: source.loader_fields, search_metadata: None, } } } pub fn backend(meta_namespace: Option) -> Box { match ENV.SEARCH_BACKEND { SearchBackendKind::Typesense => { let config = backend::TypesenseConfig::new(meta_namespace); Box::new(backend::Typesense::new(config)) } SearchBackendKind::Elasticsearch => { let config = backend::ElasticsearchConfig::new(meta_namespace); Box::new(backend::Elasticsearch::new(config)) } } }