update scan file filtering

apps/labrinth/.sqlx/query-502a0fcf14a874beb2709d5518fb7f4dd3ec2d110bd6e75caa1fbe3014d8e412.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        select\n            fa.file_id as \"file_id: DBFileId\",\n            f.url,\n            v.mod_id as \"project_id: DBProjectId\"\n        from file_scans fa\n        inner join files f on f.id = fa.file_id\n        inner join attribution_enforced_versions aev on aev.id = f.version_id\n        inner join versions v on v.id = f.version_id\n        where fa.attributions_scanned_at is null\n        ",
+  "query": "\n        select\n            fa.file_id as \"file_id: DBFileId\",\n            f.url,\n            v.mod_id as \"project_id: DBProjectId\"\n        from file_scans fa\n        inner join files f on f.id = fa.file_id\n        inner join attribution_enforced_versions aev on aev.id = f.version_id\n        inner join versions v on v.id = f.version_id\n        where fa.attributions_scanned_at is null\n        order by fa.file_id\n        limit $1\n        ",
   "describe": {
     "columns": [
       {
@@ -20,7 +20,9 @@
       }
     ],
     "parameters": {
-      "Left": []
+      "Left": [
+        "Int8"
+      ]
     },
     "nullable": [
       false,
@@ -28,5 +30,5 @@
       false
     ]
   },
-  "hash": "0dda0265b39d4c8b019eb1ea6d164ee639a6deb0f47c1dcdd83a5816308faab0"
+  "hash": "502a0fcf14a874beb2709d5518fb7f4dd3ec2d110bd6e75caa1fbe3014d8e412"
 }

apps/labrinth/.sqlx/query-eaafc6b2e29562112ae323e68955aa99e70f8ecbdc1b2500f7d7e89af8fc6e29.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        select count(*) as \"count!\" from file_scans\n        where attributions_scanned_at is null\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "eaafc6b2e29562112ae323e68955aa99e70f8ecbdc1b2500f7d7e89af8fc6e29"
+}

apps/labrinth/src/background_task.rs

@@ -8,7 +8,7 @@ use crate::models::notifications::NotificationBody;
 use crate::queue::analytics::cache::cache_analytics;
 use crate::queue::billing::{index_billing, index_subscriptions};
 use crate::queue::email::EmailQueue;
-use crate::queue::file_scan::scan_all_files;
+use crate::queue::file_scan::scan_all_pending_files;
 use crate::queue::payouts::{
     PayoutsQueue, index_payouts_notifications,
     insert_bank_balances_and_webhook, process_affiliate_payouts,
@@ -43,7 +43,7 @@ pub enum BackgroundTask {
     /// Finds files of versions which have not been scanned for attributions
     /// yet, extracts them to find file overrides, and finds any overrides which
     /// require attribution from the creator.
-    ScanFiles,
+    ScanPendingFiles,
     /// Queues Discord Creator Club role claim emails for newly eligible users.
     DiscordRoleEmailCampaign,
 }
@@ -119,7 +119,14 @@ impl BackgroundTask {
                 )
                 .await
             }
-            ScanFiles => scan_all_files(&pool, &redis_pool, &**file_host).await,
+            ScanPendingFiles => {
+                scan_all_pending_files(
+                    &pool,
+                    &redis_pool,
+                    file_host.into_inner(),
+                )
+                .await
+            }
             DiscordRoleEmailCampaign => {
                 discord_role_email_campaign(pool, redis_pool).await
             }

apps/labrinth/src/env.rs

@@ -166,6 +166,7 @@ vars! {
 
     // storage
     STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
+    FILE_SCAN_CONCURRENCY: i64 = 8i64;
 
     // s3
     S3_PUBLIC_BUCKET_NAME: String = "";

apps/labrinth/src/queue/file_scan.rs

@@ -1,11 +1,12 @@
 use std::collections::HashMap;
 use std::io::{Cursor, Read};
+use std::sync::Arc;
 
 use chrono::Utc;
 use eyre::{Result, eyre};
 use hex::ToHex;
 use sha1::Digest;
-use tokio::task::spawn_blocking;
+use tokio::task::{spawn, spawn_blocking};
 use tracing::{Instrument, info, info_span, warn};
 use zip::ZipArchive;
 
@@ -29,6 +30,15 @@ use crate::queue::moderation::{
 use crate::util::error::Context;
 use crate::util::http::HTTP_CLIENT;
 
+const PENDING_FILE_SCAN_BATCH_SIZE: i64 = 100;
+
+#[derive(Clone)]
+struct PendingFileScan {
+    file_id: DBFileId,
+    url: String,
+    project_id: DBProjectId,
+}
+
 /// Attribution enforcement is version/project-scoped, not file-hash-scoped.
 ///
 /// Versions or projects listed in `attributions_exemptions` predate this
@@ -39,11 +49,50 @@ use crate::util::http::HTTP_CLIENT;
 /// versions must go through the `attribution_enforced_versions` view so
 /// grandfathered versions and projects are ignored without making the SHA1
 /// itself exempt.
-pub async fn scan_all_files(
+pub async fn scan_all_pending_files(
     db: &PgPool,
     redis: &RedisPool,
-    file_host: &dyn FileHost,
+    file_host: Arc<dyn FileHost>,
 ) -> Result<()> {
+    let scan_concurrency = ENV.FILE_SCAN_CONCURRENCY.max(1);
+
+    let total_to_scan = sqlx::query_scalar!(
+        r#"
+        select count(*) as "count!" from file_scans
+        where attributions_scanned_at is null
+        "#,
+    )
+    .fetch_one(db)
+    .await
+    .wrap_err("fetching number of files to scan")?;
+
+    info!(
+        "Found {total_to_scan} total pending files to scan, running in batches of {PENDING_FILE_SCAN_BATCH_SIZE} with concurrency {scan_concurrency}"
+    );
+
+    loop {
+        let scanned_count = scan_pending_files_batch(
+            db,
+            redis,
+            file_host.clone(),
+            scan_concurrency * PENDING_FILE_SCAN_BATCH_SIZE,
+        )
+        .await?;
+
+        if scanned_count == 0 {
+            break;
+        }
+    }
+
+    Ok(())
+}
+
+async fn scan_pending_files_batch(
+    db: &PgPool,
+    redis: &RedisPool,
+    file_host: Arc<dyn FileHost>,
+    scan_limit: i64,
+) -> Result<usize> {
     let files_to_scan = sqlx::query!(
         r#"
         select
@@ -55,14 +104,61 @@ pub async fn scan_all_files(
         inner join attribution_enforced_versions aev on aev.id = f.version_id
         inner join versions v on v.id = f.version_id
         where fa.attributions_scanned_at is null
-        "#
+        order by fa.file_id
+        limit $1
+        "#,
+        scan_limit,
     )
     .fetch_all(db)
     .await
     .wrap_err("fetching files to scan")?;
 
-    info!("Found {} files to scan", files_to_scan.len());
+    info!(
+        "Found {} pending files to scan, splitting into jobs of {PENDING_FILE_SCAN_BATCH_SIZE}",
+        files_to_scan.len(),
+    );
 
+    let files_to_scan: Vec<_> = files_to_scan
+        .into_iter()
+        .map(|row| PendingFileScan {
+            file_id: row.file_id,
+            url: row.url,
+            project_id: row.project_id,
+        })
+        .collect();
+
+    let mut tasks = Vec::new();
+    for chunk in files_to_scan.chunks(PENDING_FILE_SCAN_BATCH_SIZE as usize) {
+        let db = db.clone();
+        let redis = redis.clone();
+        let file_host = file_host.clone();
+        let chunk = chunk.to_vec();
+
+        tasks.push(spawn(async move {
+            scan_pending_files_chunk(&db, &redis, &*file_host, chunk).await
+        }));
+    }
+
+    let mut scanned_count = 0;
+    for task in tasks {
+        scanned_count += task
+            .await
+            .wrap_err("joining file scan task")?
+            .wrap_err("scanning pending file chunk")?;
+    }
+
+    info!("Marked {} files as scanned", scanned_count);
+
+    Ok(scanned_count)
+}
+
+async fn scan_pending_files_chunk(
+    db: &PgPool,
+    redis: &RedisPool,
+    file_host: &dyn FileHost,
+    files_to_scan: Vec<PendingFileScan>,
+) -> Result<usize> {
+    info!("Scanning {} files", files_to_scan.len());
     let mut scanned_count = 0;
 
     for row in files_to_scan {
@@ -72,10 +168,6 @@ pub async fn scan_all_files(
             info!("Scanning file");
 
             let file_id = row.file_id;
-            let mut txn = db
-                .begin()
-                .await
-                .wrap_err("beginning file scan transaction")?;
 
             let overrides = extract_override_files_from_storage(
                 file_host, file_id, &row.url,
@@ -87,29 +179,35 @@ pub async fn scan_all_files(
 
             if overrides.is_empty() {
                 info!("Found no overrides");
-            } else {
-                info!("Found {} overrides", overrides.len());
+                return Ok(());
+            }
 
-                let resolved = resolve_overrides(&overrides, redis, &mut txn)
-                    .await
-                    .wrap_err_with(|| {
-                        eyre!("resolving overrides for file {file_id:?}")
-                    })?;
-                info!("Resolved: {resolved:#?}");
+            info!("Found {} overrides", overrides.len());
 
-                persist_attribution_results(
-                    row.project_id,
-                    file_id,
-                    &overrides,
-                    &resolved,
-                    redis,
-                    &mut txn,
-                )
+            let mut txn = db
+                .begin()
+                .await
+                .wrap_err("beginning file scan transaction")?;
+
+            let resolved = resolve_overrides(&overrides, redis, &mut txn)
                 .await
                 .wrap_err_with(|| {
-                    eyre!("persisting attribution results for file {file_id:?}")
+                    eyre!("resolving overrides for file {file_id:?}")
                 })?;
-            }
+            info!("Resolved: {resolved:#?}");
+
+            persist_attribution_results(
+                row.project_id,
+                file_id,
+                &overrides,
+                &resolved,
+                redis,
+                &mut txn,
+            )
+            .await
+            .wrap_err_with(|| {
+                eyre!("persisting attribution results for file {file_id:?}")
+            })?;
 
             let now = Utc::now();
             sqlx::query!(
@@ -137,9 +235,7 @@ pub async fn scan_all_files(
         scanned_count += 1;
     }
 
-    info!("Marked {} files as scanned", scanned_count);
-
-    Ok(())
+    Ok(scanned_count)
 }
 
 pub async fn scan_file(
@@ -276,10 +372,19 @@ fn extract_override_files(data: &[u8]) -> Result<Vec<OverrideFile>> {
             continue;
         }
 
-        let should_scan_file = name.contains(".jar")
-            || (name.contains(".zip") && !name.ends_with(".zip.txt"));
-
-        if name.matches('/').count() > 2 || !should_scan_file {
+        let should_skip = name.starts_with("mods/.connector/")
+            || name.starts_with(".sable/natives/")
+            || name.starts_with("local/crash_assistant/")
+            || name.starts_with("mods/mcef-libraries/")
+            || name.starts_with("mods/mcef-cache/")
+            || name.starts_with("config/super_resolution/libraries/")
+            || name.starts_with("config/Veinminer/update/")
+            || name.starts_with("config/epicfight/native/")
+            || name.starts_with("essential/")
+            || name.ends_with(".rpo")
+            || name.ends_with(".txt");
+        let should_scan = name.contains(".jar") || name.contains(".zip");
+        if should_scan && !should_skip {
             continue;
         }
 

apps/labrinth/src/queue/moderation.rs

@@ -1,6 +1,4 @@
-use crate::auth::checks::filter_visible_versions;
 use crate::database;
-use crate::database::models::DBUserId;
 use crate::database::models::notification_item::NotificationBuilder;
 use crate::database::models::thread_item::ThreadMessageBuilder;
 use crate::database::redis::RedisPool;
@@ -8,20 +6,15 @@ use crate::database::{PgPool, ReadOnlyPgPool};
 use crate::env::ENV;
 use crate::models::ids::ProjectId;
 use crate::models::notifications::NotificationBody;
-use crate::models::pack::{PackFile, PackFileHash, PackFormat};
 use crate::models::projects::ProjectStatus;
 use crate::models::threads::MessageBody;
 use crate::routes::ApiError;
 use dashmap::DashSet;
-use hex::ToHex;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
-use sha1::Digest;
 use std::collections::HashMap;
 use std::fmt::Write;
-use std::io::{Cursor, Read};
 use std::time::Duration;
-use zip::ZipArchive;
 
 pub const AUTOMOD_ID: i64 = 0;
 
@@ -293,343 +286,7 @@ impl AutomatedModerationQueue {
                             for version in versions {
                                 let primary_file = version.files.iter().find_or_first(|x| x.primary);
 
-                                if let Some(primary_file) = primary_file {
-                                    let data = reqwest::get(&primary_file.url).await?.bytes().await?;
-
-                                    let reader = Cursor::new(data);
-                                    let mut zip = ZipArchive::new(reader)?;
-
-                                    let pack: PackFormat = {
-                                        let Ok(mut file) = zip.by_name("modrinth.index.json") else {
-                                                continue;
-                                            };
-
-                                        let mut contents = String::new();
-                                        file.read_to_string(&mut contents)?;
-
-                                        serde_json::from_str(&contents)?
-                                    };
-
-                                    // sha1, pack file, file path, murmur
-                                    let mut hashes: Vec<(
-                                        String,
-                                        Option<PackFile>,
-                                        String,
-                                        Option<u32>,
-                                    )> = pack
-                                        .files
-                                        .clone()
-                                        .into_iter()
-                                        .filter_map(|x| {
-                                            let hash = x.hashes.get(&PackFileHash::Sha1);
-
-                                            if let Some(hash) = hash {
-                                                let path = x.path.to_string();
-                                                Some((hash.clone(), Some(x), path, None))
-                                            } else {
-                                                None
-                                            }
-                                        })
-                                        .collect();
-
-                                    for i in 0..zip.len() {
-                                        let mut file = zip.by_index(i)?;
-
-                                        if file.name().starts_with("overrides/mods")
-                                            || file.name().starts_with("client-overrides/mods")
-                                            || file.name().starts_with("server-overrides/mods")
-                                            || file.name().starts_with("overrides/shaderpacks")
-                                            || file.name().starts_with("client-overrides/shaderpacks")
-                                            || file.name().starts_with("overrides/resourcepacks")
-                                            || file.name().starts_with("client-overrides/resourcepacks")
-                                        {
-                                            if file.name().matches('/').count() > 2 || file.name().ends_with(".txt") || file.name().ends_with(".rpo") {
-                                                continue;
-                                            }
-
-                                            let mut contents = Vec::new();
-                                            file.read_to_end(&mut contents)?;
-
-                                            let hash = sha1::Sha1::digest(&contents).encode_hex::<String>();
-                                            let murmur = hash_flame_murmur32(contents);
-
-                                            hashes.push((
-                                                hash,
-                                                None,
-                                                file.name().to_string(),
-                                                Some(murmur),
-                                            ));
-                                        }
-                                    }
-
-                                    let files = database::models::DBVersion::get_files_from_hash(
-                                        "sha1".to_string(),
-                                        &hashes.iter().map(|x| x.0.clone()).collect::<Vec<_>>(),
-                                        &pool,
-                                        &redis,
-                                    )
-                                        .await?;
-
-                                    let version_ids =
-                                        files.iter().map(|x| x.version_id).collect::<Vec<_>>();
-                                    let versions_data = filter_visible_versions(
-                                        database::models::DBVersion::get_many(
-                                            &version_ids,
-                                            &*ro_pool,
-                                            &redis,
-                                        )
-                                            .await?,
-                                        &None,
-                                        &pool,
-                                        &ro_pool,
-                                        &redis,
-                                    )
-                                        .await?;
-
-                                    let mut final_hashes = HashMap::new();
-
-                                    for version in versions_data {
-                                        for file in
-                                        files.iter().filter(|x| x.version_id == version.id.into())
-                                        {
-                                            if let Some(hash) = file.hashes.get("sha1")
-                                                && let Some((index, (sha1, _, file_name, _))) = hashes
-                                                    .iter()
-                                                    .enumerate()
-                                                    .find(|(_, (value, _, _, _))| value == hash)
-                                                {
-                                                    final_hashes
-                                                        .insert(sha1.clone(), IdentifiedFile { status: ApprovalType::Yes, file_name: file_name.clone() });
-
-                                                    hashes.remove(index);
-                                                }
-                                        }
-                                    }
-
-                                    // All files are on Modrinth, so we don't send any messages
-                                    if hashes.is_empty() {
-                                        sqlx::query!(
-                                            "
-                                            UPDATE files
-                                            SET metadata = $1
-                                            WHERE id = $2
-                                            ",
-                                            serde_json::to_value(&MissingMetadata {
-                                                identified: final_hashes,
-                                                flame_files: HashMap::new(),
-                                                unknown_files: HashMap::new(),
-                                            })?,
-                                            primary_file.id.0
-                                        )
-                                            .execute(&pool)
-                                            .await?;
-
-                                        continue;
-                                    }
-
-                                    let rows = sqlx::query!(
-                                        "
-                                        SELECT encode(mef.sha1, 'escape') sha1, mel.status status
-                                        FROM moderation_external_files mef
-                                        INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
-                                        WHERE mef.sha1 = ANY($1)
-                                        ",
-                                        &hashes.iter().map(|x| x.0.as_bytes().to_vec()).collect::<Vec<_>>()
-                                    )
-                                        .fetch_all(&pool)
-                                        .await?;
-
-                                    for row in rows {
-                                        if let Some(sha1) = row.sha1
-                                            && let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == &sha1) {
-                                                final_hashes.insert(sha1.clone(), IdentifiedFile { file_name: file_name.clone(), status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified) });
-                                                hashes.remove(index);
-                                            }
-                                    }
-
-                                    if hashes.is_empty() {
-                                        let metadata = MissingMetadata {
-                                            identified: final_hashes,
-                                            flame_files: HashMap::new(),
-                                            unknown_files: HashMap::new(),
-                                        };
-
-                                        sqlx::query!(
-                                            "
-                                            UPDATE files
-                                            SET metadata = $1
-                                            WHERE id = $2
-                                            ",
-                                            serde_json::to_value(&metadata)?,
-                                            primary_file.id.0
-                                        )
-                                            .execute(&pool)
-                                            .await?;
-
-                                        if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
-                                            let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
-                                            val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
-                                        }
-                                        continue;
-                                    }
-
-                                    let client = reqwest::Client::new();
-                                    let res = client
-                                        .post(format!("{}/v1/fingerprints", ENV.FLAME_ANVIL_URL))
-                                        .json(&serde_json::json!({
-                                        "fingerprints": hashes.iter().filter_map(|x| x.3).collect::<Vec<u32>>()
-                                    }))
-                                        .send()
-                                        .await?.text()
-                                        .await?;
-
-                                    let flame_hashes = serde_json::from_str::<FlameResponse<FingerprintResponse>>(&res)?
-                                        .data
-                                        .exact_matches
-                                        .into_iter()
-                                        .map(|x| x.file)
-                                        .collect::<Vec<_>>();
-
-                                    let mut flame_files = Vec::new();
-
-                                    for file in flame_hashes {
-                                        let hash = file
-                                            .hashes
-                                            .iter()
-                                            .find(|x| x.algo == 1)
-                                            .map(|x| x.value.clone());
-
-                                        if let Some(hash) = hash  {
-                                            flame_files.push((hash, file.mod_id))
-                                        }
-                                    }
-
-                                    let rows = sqlx::query!(
-                                        "
-                                        SELECT mel.id, mel.flame_project_id, mel.status status
-                                        FROM moderation_external_licenses mel
-                                        WHERE mel.flame_project_id = ANY($1)
-                                        ",
-                                        &flame_files.iter().map(|x| x.1 as i32).collect::<Vec<_>>()
-                                    )
-                                        .fetch_all(&pool).await?;
-
-                                    let mut insert_hashes = Vec::new();
-                                    let mut insert_filenames = Vec::new();
-                                    let mut insert_ids = Vec::new();
-
-                                    for row in rows {
-                                        if let Some((curse_index, (hash, _flame_id))) = flame_files.iter().enumerate().find(|(_, x)| Some(x.1 as i32) == row.flame_project_id)
-                                            && let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == hash) {
-                                                final_hashes.insert(sha1.clone(), IdentifiedFile {
-                                                    file_name: file_name.clone(),
-                                                    status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified),
-                                                });
-
-                                                insert_hashes.push(hash.clone().as_bytes().to_vec());
-                                                insert_filenames.push(Some(file_name.clone()));
-                                                insert_ids.push(row.id);
-
-                                                hashes.remove(index);
-                                                flame_files.remove(curse_index);
-                                            }
-                                    }
-
-                                    if !insert_ids.is_empty() && !insert_hashes.is_empty() {
-                                        crate::database::models::moderation_external_item::ExternalLicense::insert_files(
-                                            &pool,
-                                            &insert_hashes,
-                                            &insert_filenames,
-                                            &insert_ids,
-                                            DBUserId(0),
-                                        )
-                                            .await?;
-                                    }
-
-                                    if hashes.is_empty() {
-                                        let metadata = MissingMetadata {
-                                            identified: final_hashes,
-                                            flame_files: HashMap::new(),
-                                            unknown_files: HashMap::new(),
-                                        };
-
-                                        sqlx::query!(
-                                            "
-                                            UPDATE files
-                                            SET metadata = $1
-                                            WHERE id = $2
-                                            ",
-                                            serde_json::to_value(&metadata)?,
-                                            primary_file.id.0
-                                        )
-                                            .execute(&pool)
-                                            .await?;
-
-                                        if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
-                                            let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
-                                            val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
-                                        }
-
-                                        continue;
-                                    }
-
-                                        let flame_projects  = if flame_files.is_empty() {
-                                            Vec::new()
-                                        } else {
-                                            let res = client
-                                                .post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL))
-                                            .json(&serde_json::json!({
-                                                "modIds": flame_files.iter().map(|x| x.1).collect::<Vec<_>>()
-                                            }))
-                                            .send()
-                                            .await?
-                                            .text()
-                                            .await?;
-
-                                        serde_json::from_str::<FlameResponse<Vec<FlameProjectResponse>>>(&res)?.data
-                                    };
-
-                                    let mut missing_metadata = MissingMetadata {
-                                        identified: final_hashes,
-                                        flame_files: HashMap::new(),
-                                        unknown_files: HashMap::new(),
-                                    };
-
-                                    for (sha1, _pack_file, file_name, _mumur2) in hashes {
-                                        let flame_file = flame_files.iter().find(|x| x.0 == sha1);
-
-                                        if let Some((_, flame_project_id)) = flame_file
-                                            && let Some(project) = flame_projects.iter().find(|x| &x.id == flame_project_id) {
-                                                missing_metadata.flame_files.insert(sha1, MissingMetadataFlame {
-                                                    title: project.name.clone(),
-                                                    file_name,
-                                                    url: project.links.website_url.clone(),
-                                                    id: *flame_project_id,
-                                                });
-
-                                                continue;
-                                            }
-
-                                        missing_metadata.unknown_files.insert(sha1, file_name);
-                                    }
-
-                                    sqlx::query!(
-                                        "
-                                        UPDATE files
-                                        SET metadata = $1
-                                        WHERE id = $2
-                                        ",
-                                        serde_json::to_value(&missing_metadata)?,
-                                        primary_file.id.0
-                                    )
-                                        .execute(&pool)
-                                        .await?;
-
-                                    if missing_metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
-                                        let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
-                                        val.push(ModerationMessage::PackFilesNotAllowed {files: missing_metadata.identified, incomplete: true });
-                                    }
-                                } else {
+                                if primary_file.is_none() {
                                     let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
                                     val.push(ModerationMessage::NoPrimaryFile);
                                 }
@@ -922,13 +579,3 @@ pub struct FlameLogo {
 pub struct FlameLinks {
     pub website_url: String,
 }
-
-fn hash_flame_murmur32(input: Vec<u8>) -> u32 {
-    murmur2::murmur2(
-        &input
-            .into_iter()
-            .filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32)
-            .collect::<Vec<u8>>(),
-        1,
-    )
-}