mirror of
https://github.com/modrinth/code.git
synced 2026-08-31 12:05:53 +00:00
standardise fn names
This commit is contained in:
@@ -177,7 +177,7 @@ pub async fn index_search(
|
||||
search_backend: web::Data<dyn SearchBackend>,
|
||||
) -> eyre::Result<()> {
|
||||
info!("Indexing local database");
|
||||
search_backend.index_projects(ro_pool, redis_pool).await
|
||||
search_backend.rebuild_index(ro_pool, redis_pool).await
|
||||
}
|
||||
|
||||
pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {
|
||||
|
||||
@@ -181,8 +181,9 @@ impl ServerPingQueue {
|
||||
None,
|
||||
&self.redis,
|
||||
);
|
||||
let queue_search =
|
||||
self.incremental_search_queue.push(*project_id);
|
||||
let queue_search = self
|
||||
.incremental_search_queue
|
||||
.push_project_change(*project_id);
|
||||
|
||||
let (clear_cache_result, _) =
|
||||
join(clear_cache, queue_search).await;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::queue::analytics::AnalyticsQueue;
|
||||
use crate::queue::session::AuthQueue;
|
||||
use crate::routes::ApiError;
|
||||
use crate::search::SearchBackend;
|
||||
use crate::search::incremental::consume::reindex_project;
|
||||
use crate::search::incremental::consume::reindex_project_document;
|
||||
use crate::util::date::get_current_tenths_of_ms;
|
||||
use crate::util::error::Context;
|
||||
use crate::util::guards::admin_key_guard;
|
||||
@@ -330,7 +330,7 @@ pub async fn force_reindex(
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let redis = redis.get_ref();
|
||||
search_backend
|
||||
.index_projects(pool.as_ref().clone(), redis.clone())
|
||||
.rebuild_index(pool.as_ref().clone(), redis.clone())
|
||||
.await
|
||||
.wrap_internal_err("failed to index projects")?;
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
@@ -355,7 +355,7 @@ pub async fn force_reindex_project(
|
||||
search_backend: web::Data<dyn SearchBackend>,
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let (project_id,) = path.into_inner();
|
||||
reindex_project(
|
||||
reindex_project_document(
|
||||
pool.as_ref(),
|
||||
redis.as_ref(),
|
||||
search_backend.as_ref(),
|
||||
|
||||
@@ -86,7 +86,10 @@ pub async fn clear_project_cache_and_queue_search(
|
||||
)
|
||||
.await?;
|
||||
|
||||
search_state.queue.push(project_id.into()).await;
|
||||
search_state
|
||||
.queue
|
||||
.push_project_change(project_id.into())
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1180,7 +1183,7 @@ pub async fn project_edit_internal(
|
||||
.await?;
|
||||
search_state
|
||||
.queue
|
||||
.push_versions(
|
||||
.push_version_changes(
|
||||
project_item.inner.id.into(),
|
||||
project_item.versions.iter().copied().map(VersionId::from),
|
||||
)
|
||||
|
||||
@@ -20,8 +20,8 @@ use crate::models::notifications::NotificationBody;
|
||||
use crate::models::pack::PackFileHash;
|
||||
use crate::models::pats::Scopes;
|
||||
use crate::models::projects::{
|
||||
Dependency, FileType, Loader, ProjectStatus, Version, VersionFile,
|
||||
VersionStatus, VersionType,
|
||||
Dependency, FileType, Loader, Version, VersionFile, VersionStatus,
|
||||
VersionType,
|
||||
};
|
||||
use crate::models::projects::{DependencyType, skip_nulls};
|
||||
use crate::models::teams::ProjectPermissions;
|
||||
@@ -190,7 +190,10 @@ pub async fn version_create(
|
||||
.await?;
|
||||
search_state
|
||||
.queue
|
||||
.push_versions((*project_id).into(), [VersionId::from(*version_id)])
|
||||
.push_version_changes(
|
||||
(*project_id).into(),
|
||||
[VersionId::from(*version_id)],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -567,18 +570,6 @@ async fn version_create_inner(
|
||||
}
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE mods
|
||||
SET queued = NOW()
|
||||
WHERE id = $1 AND status = $2
|
||||
",
|
||||
project_id as models::DBProjectId,
|
||||
ProjectStatus::Processing.as_str(),
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
|
||||
Ok((
|
||||
HttpResponse::Ok().json(response),
|
||||
project_id,
|
||||
@@ -685,7 +676,7 @@ pub async fn upload_file_to_version(
|
||||
.await?;
|
||||
search_state
|
||||
.queue
|
||||
.push_versions((*project_id).into(), [version_id])
|
||||
.push_version_changes((*project_id).into(), [version_id])
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
@@ -861,7 +861,7 @@ pub async fn version_edit_helper(
|
||||
.await?;
|
||||
search_state
|
||||
.queue
|
||||
.push_versions(
|
||||
.push_version_changes(
|
||||
version_item.inner.project_id.into(),
|
||||
[VersionId::from(version_item.inner.id)],
|
||||
)
|
||||
@@ -1249,7 +1249,7 @@ pub async fn version_delete(
|
||||
.await?;
|
||||
search_state
|
||||
.queue
|
||||
.push_versions(
|
||||
.push_version_changes(
|
||||
version.inner.project_id.into(),
|
||||
[VersionId::from(version.inner.id)],
|
||||
)
|
||||
|
||||
@@ -1167,7 +1167,7 @@ impl SearchBackend for Typesense {
|
||||
})
|
||||
}
|
||||
|
||||
async fn index_projects(
|
||||
async fn rebuild_index(
|
||||
&self,
|
||||
ro_pool: PgPool,
|
||||
redis: RedisPool,
|
||||
|
||||
@@ -36,11 +36,11 @@ impl IncrementalSearchQueue {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn push(&self, project_id: ProjectId) {
|
||||
pub async fn push_project_change(&self, project_id: ProjectId) {
|
||||
self.operations.lock().await.push_project_change(project_id);
|
||||
}
|
||||
|
||||
pub async fn push_versions(
|
||||
pub async fn push_version_changes(
|
||||
&self,
|
||||
project_id: ProjectId,
|
||||
version_ids: impl IntoIterator<Item = VersionId>,
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::{
|
||||
SearchBackend, SearchDocumentBatch, SearchIndexUpdate,
|
||||
UploadSearchProject,
|
||||
incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC,
|
||||
indexing::{index_project_documents, index_project_version_documents},
|
||||
indexing::{build_project_documents, build_version_change_documents},
|
||||
},
|
||||
util::kafka::{
|
||||
INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL,
|
||||
@@ -219,14 +219,25 @@ async fn consume_batch(
|
||||
|
||||
if !project_ids_with_version_changes.is_empty() {
|
||||
let operation_start = Instant::now();
|
||||
let changed_documents = build_changed_project_versions(
|
||||
let changed_documents = build_version_change_documents(
|
||||
ro_pool,
|
||||
redis_pool,
|
||||
&project_ids_with_version_changes,
|
||||
&version_ids_to_change,
|
||||
)
|
||||
.instrument(info_span!(
|
||||
"index",
|
||||
batch_size = project_ids_with_version_changes.len(),
|
||||
version_count = version_ids_to_change.len()
|
||||
))
|
||||
.await
|
||||
.wrap_err("failed to build changed project versions")?;
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to build search documents for {} projects and {} versions",
|
||||
project_ids_with_version_changes.len(),
|
||||
version_ids_to_change.len()
|
||||
)
|
||||
})?;
|
||||
documents.projects.extend(changed_documents.projects);
|
||||
documents.versions.extend(changed_documents.versions);
|
||||
info!(
|
||||
@@ -244,9 +255,13 @@ async fn consume_batch(
|
||||
"Building changed projects"
|
||||
);
|
||||
documents.projects.extend(
|
||||
build_changed_projects(ro_pool, redis_pool, &project_ids_to_change)
|
||||
.await
|
||||
.wrap_err("failed to build changed projects")?,
|
||||
build_changed_project_documents(
|
||||
ro_pool,
|
||||
redis_pool,
|
||||
&project_ids_to_change,
|
||||
)
|
||||
.await
|
||||
.wrap_err("failed to build changed projects")?,
|
||||
);
|
||||
info!(
|
||||
project_count = project_ids_to_change.len(),
|
||||
@@ -290,16 +305,22 @@ async fn consume_batch(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reindex_project(
|
||||
pub async fn reindex_project_document(
|
||||
ro_pool: &PgPool,
|
||||
redis_pool: &RedisPool,
|
||||
search_backend: &dyn SearchBackend,
|
||||
project_id: ProjectId,
|
||||
) -> eyre::Result<()> {
|
||||
reindex_projects(ro_pool, redis_pool, search_backend, &[project_id]).await
|
||||
reindex_project_documents(
|
||||
ro_pool,
|
||||
redis_pool,
|
||||
search_backend,
|
||||
&[project_id],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn reindex_projects(
|
||||
pub async fn reindex_project_documents(
|
||||
ro_pool: &PgPool,
|
||||
redis_pool: &RedisPool,
|
||||
search_backend: &dyn SearchBackend,
|
||||
@@ -307,7 +328,8 @@ pub async fn reindex_projects(
|
||||
) -> eyre::Result<()> {
|
||||
info!("Creating project documents");
|
||||
let projects =
|
||||
build_changed_projects(ro_pool, redis_pool, project_ids).await?;
|
||||
build_changed_project_documents(ro_pool, redis_pool, project_ids)
|
||||
.await?;
|
||||
search_backend
|
||||
.apply_update(SearchIndexUpdate {
|
||||
projects: &projects,
|
||||
@@ -318,12 +340,12 @@ pub async fn reindex_projects(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_changed_projects(
|
||||
async fn build_changed_project_documents(
|
||||
ro_pool: &PgPool,
|
||||
redis_pool: &RedisPool,
|
||||
project_ids: &[ProjectId],
|
||||
) -> eyre::Result<Vec<UploadSearchProject>> {
|
||||
let documents = index_project_documents(ro_pool, redis_pool, project_ids)
|
||||
let documents = build_project_documents(ro_pool, redis_pool, project_ids)
|
||||
.instrument(info_span!("index", batch_size = project_ids.len()))
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
@@ -337,35 +359,6 @@ async fn build_changed_projects(
|
||||
Ok(documents)
|
||||
}
|
||||
|
||||
async fn build_changed_project_versions(
|
||||
ro_pool: &PgPool,
|
||||
redis_pool: &RedisPool,
|
||||
project_ids: &[ProjectId],
|
||||
version_ids: &[VersionId],
|
||||
) -> eyre::Result<SearchDocumentBatch> {
|
||||
let documents = index_project_version_documents(
|
||||
ro_pool,
|
||||
redis_pool,
|
||||
project_ids,
|
||||
version_ids,
|
||||
)
|
||||
.instrument(info_span!(
|
||||
"index",
|
||||
batch_size = project_ids.len(),
|
||||
version_count = version_ids.len()
|
||||
))
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to build search documents for {} projects and {} versions",
|
||||
project_ids.len(),
|
||||
version_ids.len()
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(documents)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum SearchProjectIndexQueueEvent {
|
||||
|
||||
@@ -121,7 +121,7 @@ pub async fn index_local(
|
||||
Ok((documents, *largest))
|
||||
}
|
||||
|
||||
pub async fn index_project_documents(
|
||||
pub async fn build_project_documents(
|
||||
pool: &PgPool,
|
||||
redis: &RedisPool,
|
||||
project_ids: &[ProjectId],
|
||||
@@ -171,14 +171,14 @@ pub async fn index_project_documents(
|
||||
.projects)
|
||||
}
|
||||
|
||||
pub async fn index_project_version_documents(
|
||||
pub async fn build_version_change_documents(
|
||||
pool: &PgPool,
|
||||
redis: &RedisPool,
|
||||
project_ids: &[ProjectId],
|
||||
version_ids: &[VersionId],
|
||||
) -> eyre::Result<SearchDocumentBatch> {
|
||||
let projects =
|
||||
index_project_document_batch(pool, redis, project_ids).await?;
|
||||
build_search_document_batch(pool, redis, project_ids).await?;
|
||||
let version_ids = version_ids
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
@@ -193,7 +193,7 @@ pub async fn index_project_version_documents(
|
||||
})
|
||||
}
|
||||
|
||||
async fn index_project_document_batch(
|
||||
async fn build_search_document_batch(
|
||||
pool: &PgPool,
|
||||
redis: &RedisPool,
|
||||
project_ids: &[ProjectId],
|
||||
@@ -359,7 +359,7 @@ async fn build_search_documents(
|
||||
.await?;
|
||||
|
||||
info!("Indexing local versions!");
|
||||
let mut versions = index_versions(pool, project_ids.clone()).await?;
|
||||
let mut versions = load_project_versions(pool, project_ids.clone()).await?;
|
||||
|
||||
info!("Indexing local org owners!");
|
||||
|
||||
@@ -780,7 +780,7 @@ struct PartialVersion {
|
||||
date_published: DateTime<Utc>,
|
||||
}
|
||||
|
||||
async fn index_versions(
|
||||
async fn load_project_versions(
|
||||
pool: &PgPool,
|
||||
project_ids: Vec<i64>,
|
||||
) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>> {
|
||||
|
||||
@@ -105,7 +105,7 @@ pub trait SearchBackend: Send + Sync {
|
||||
info: &SearchRequest,
|
||||
) -> Result<SearchResults, ApiError>;
|
||||
|
||||
async fn index_projects(
|
||||
async fn rebuild_index(
|
||||
&self,
|
||||
ro_pool: PgPool,
|
||||
redis: RedisPool,
|
||||
|
||||
Reference in New Issue
Block a user