fix: scan logic file whitelist + rescan route (#6585)

* explicitly only scan jar/zip/disabled files for attributions

* add route to force scan file

* prepare

apps/labrinth/src/queue/file_scan.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 use chrono::Utc;
 use eyre::{Result, eyre};
 use hex::ToHex;
+use serde::Serialize;
 use sha1::Digest;
 use tokio::task::{spawn, spawn_blocking};
 use tracing::{Instrument, info, info_span, warn};
@@ -42,6 +43,16 @@ struct PendingFileScan {
     project_id: DBProjectId,
 }
 
+#[derive(Debug, Default, Serialize, utoipa::ToSchema)]
+pub struct FileScanSummary {
+    /// Number of attribution groups newly created by the scan.
+    pub new_attribution_groups: u64,
+    /// Number of attribution files newly created by the scan.
+    pub new_attribution_files: u64,
+    /// Override file paths found and scanned in the file archive.
+    pub scanned_file_names: Vec<String>,
+}
+
 /// Attribution enforcement is version/project-scoped, not file-hash-scoped.
 ///
 /// Versions or projects listed in `attributions_exemptions` predate this
@@ -277,7 +288,7 @@ pub async fn scan_file(
     project_id: DBProjectId,
     file_id: DBFileId,
     file_url: &str,
-) -> Result<()> {
+) -> Result<FileScanSummary> {
     let result =
         scan_file_inner(txn, redis, file_host, project_id, file_id, file_url)
             .await;
@@ -296,7 +307,7 @@ async fn scan_file_inner(
     project_id: DBProjectId,
     file_id: DBFileId,
     file_url: &str,
-) -> Result<()> {
+) -> Result<FileScanSummary> {
     let overrides =
         extract_override_files_from_storage(file_host, file_id, file_url)
             .await
@@ -304,7 +315,16 @@ async fn scan_file_inner(
                 eyre!("extracting overrides for file {file_id:?}")
             })?;
 
+    let scanned_file_names =
+        overrides.iter().map(|file| file.path.clone()).collect();
+    let mut summary = FileScanSummary {
+        scanned_file_names,
+        ..Default::default()
+    };
+
     if !overrides.is_empty() {
+        let before = count_project_attributions(project_id, txn).await?;
+
         let resolved = resolve_overrides(&overrides, redis, txn)
             .await
             .wrap_err_with(|| {
@@ -318,15 +338,58 @@ async fn scan_file_inner(
         .wrap_err_with(|| {
             eyre!("persisting attribution results for file {file_id:?}")
         })?;
+
+        let after = count_project_attributions(project_id, txn).await?;
+        summary.new_attribution_groups =
+            after.groups.saturating_sub(before.groups);
+        summary.new_attribution_files =
+            after.files.saturating_sub(before.files);
+
         log_marked_override_projects(&resolved);
     }
 
-    Ok(())
+    Ok(summary)
 }
 
-fn file_scan_result(result: &Result<()>) -> FileScanResult<'static> {
+struct ProjectAttributionCounts {
+    groups: u64,
+    files: u64,
+}
+
+async fn count_project_attributions(
+    project_id: DBProjectId,
+    txn: &mut PgTransaction<'_>,
+) -> Result<ProjectAttributionCounts> {
+    let row = sqlx::query!(
+        r#"
+        select
+            (
+                select count(*)
+                from project_attribution_groups
+                where project_id = $1
+            ) as "groups!",
+            (
+                select count(*)
+                from project_attribution_files paf
+                inner join project_attribution_groups pag on pag.id = paf.group_id
+                where pag.project_id = $1
+            ) as "files!"
+        "#,
+        project_id as DBProjectId,
+    )
+    .fetch_one(&mut *txn)
+    .await
+    .wrap_err("counting project attributions")?;
+
+    Ok(ProjectAttributionCounts {
+        groups: row.groups as u64,
+        files: row.files as u64,
+    })
+}
+
+fn file_scan_result<T>(result: &Result<T>) -> FileScanResult<'static> {
     match result {
-        Ok(()) => Ok(()),
+        Ok(_) => Ok(()),
         Err(err) => Err(ApiError {
             error: "internal_error",
             description: format!("{err:#}"),
@@ -480,6 +543,7 @@ const OVERRIDE_PREFIXES: &[&str] = &[
 ];
 
 fn should_scan(name: &str) -> bool {
+    let name = name.to_lowercase();
     let should_skip = name.starts_with("mods/.connector/")
         || name.starts_with(".sable/natives/")
         || name.starts_with("local/crash_assistant/")
@@ -491,8 +555,10 @@ fn should_scan(name: &str) -> bool {
         || name.starts_with("essential/")
         || name.ends_with(".rpo")
         || name.ends_with(".txt");
-    let is_archive = name.contains(".jar") || name.contains(".zip");
-
+    let is_archive = name.ends_with(".jar")
+        || name.ends_with(".zip")
+        || name.ends_with(".jar.disabled")
+        || name.ends_with(".zip.disabled");
     is_archive && !should_skip
 }
 

apps/labrinth/src/routes/internal/attribution.rs

@@ -3,17 +3,18 @@ use chrono::{DateTime, Utc};
 use eyre::eyre;
 use serde::{Deserialize, Serialize};
 
-use crate::auth::get_user_from_headers;
+use crate::auth::{check_is_moderator_from_headers, get_user_from_headers};
 use crate::database::PgPool;
 use crate::database::models::{
-    DBOrganization, DBTeamMember, DBVersion,
+    DBFileId, DBOrganization, DBTeamMember, DBVersion,
     ids::{
         DBAttributionGroupId, DBProjectId, DBVersionId,
         generate_attribution_group_id,
     },
 };
 use crate::database::redis::RedisPool;
-use crate::models::ids::{ProjectId, VersionId};
+use crate::file_hosting::FileHost;
+use crate::models::ids::{FileId, ProjectId, VersionId};
 use crate::models::pats::Scopes;
 use crate::models::projects::{
     AttributionModerationStatusKind, AttributionResolution,
@@ -21,6 +22,7 @@ use crate::models::projects::{
 };
 use crate::models::teams::ProjectPermissions;
 use crate::models::users::User;
+use crate::queue::file_scan::{FileScanSummary, scan_file};
 use crate::queue::moderation::ApprovalType;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
@@ -30,6 +32,7 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
     cfg.service(list)
         .service(update_group)
         .service(scan)
+        .service(force_scan_file)
         .service(assign)
         .service(split);
 }
@@ -201,6 +204,80 @@ async fn scan(
     }))
 }
 
+#[utoipa::path]
+#[post("/file/{file_id}/scan")]
+async fn force_scan_file(
+    req: HttpRequest,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    file_host: web::Data<dyn FileHost>,
+    path: web::Path<FileId>,
+) -> Result<web::Json<FileScanSummary>, ApiError> {
+    check_is_moderator_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::PROJECT_READ,
+    )
+    .await?;
+
+    let file_id: DBFileId = path.into_inner().into();
+    let file = sqlx::query!(
+        r#"
+		select
+			f.url,
+			f.version_id as "version_id: DBVersionId",
+			v.mod_id as "project_id: DBProjectId"
+		from files f
+		inner join versions v on v.id = f.version_id
+		where f.id = $1
+		"#,
+        file_id as DBFileId,
+    )
+    .fetch_optional(pool.as_ref())
+    .await
+    .wrap_internal_err("failed to fetch attribution scan file")?
+    .ok_or(ApiError::NotFound)?;
+
+    let mut transaction = pool.begin().await.wrap_internal_err(
+        "failed to begin attribution file scan transaction",
+    )?;
+
+    sqlx::query!(
+        r#"
+		delete from attributions_exemptions
+		where version_id = $1
+		"#,
+        file.version_id as DBVersionId,
+    )
+    .execute(&mut transaction)
+    .await
+    .wrap_internal_err("failed to remove attribution scan exemption")?;
+
+    let scan_summary = scan_file(
+        &mut transaction,
+        redis.as_ref(),
+        &**file_host,
+        file.project_id,
+        file_id,
+        &file.url,
+    )
+    .await
+    .wrap_internal_err("failed to scan file for attributions")?;
+
+    transaction.commit().await.wrap_internal_err(
+        "failed to commit attribution file scan transaction",
+    )?;
+
+    DBVersion::clear_cache_ids(&[file.version_id], redis.as_ref())
+        .await
+        .wrap_internal_err("failed to clear version cache")?;
+
+    Ok(web::Json(scan_summary))
+}
+
 #[utoipa::path]
 #[get("/{project_id}")]
 async fn list(