mirror of
https://github.com/modrinth/code.git
synced 2026-07-31 21:26:40 +00:00
Batch pending file scanning ops (#6491)
* Batch pending file scanning ops
* prepare
* move some stuff out of the txn
* scan concurrency
* prepr
* 🦀 automod attribution is dead, long live automod
* fix ci
* update scan file filtering
This commit is contained in:
aecsocket
+5
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_scans fa\n inner join files f on f.id = fa.file_id\n inner join attribution_enforced_versions aev on aev.id = f.version_id\n inner join versions v on v.id = f.version_id\n where fa.attributions_scanned_at is null\n ",
|
||||
"query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_scans fa\n inner join files f on f.id = fa.file_id\n inner join attribution_enforced_versions aev on aev.id = f.version_id\n inner join versions v on v.id = f.version_id\n where fa.attributions_scanned_at is null\n order by fa.file_id\n limit $1\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -20,7 +20,9 @@
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
"Left": [
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
@@ -28,5 +30,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "0dda0265b39d4c8b019eb1ea6d164ee639a6deb0f47c1dcdd83a5816308faab0"
|
||||
"hash": "502a0fcf14a874beb2709d5518fb7f4dd3ec2d110bd6e75caa1fbe3014d8e412"
|
||||
}
|
||||
Generated
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n select count(*) as \"count!\" from file_scans\n where attributions_scanned_at is null\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "eaafc6b2e29562112ae323e68955aa99e70f8ecbdc1b2500f7d7e89af8fc6e29"
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use crate::models::notifications::NotificationBody;
|
||||
use crate::queue::analytics::cache::cache_analytics;
|
||||
use crate::queue::billing::{index_billing, index_subscriptions};
|
||||
use crate::queue::email::EmailQueue;
|
||||
use crate::queue::file_scan::scan_all_files;
|
||||
use crate::queue::file_scan::scan_all_pending_files;
|
||||
use crate::queue::payouts::{
|
||||
PayoutsQueue, index_payouts_notifications,
|
||||
insert_bank_balances_and_webhook, process_affiliate_payouts,
|
||||
@@ -43,7 +43,7 @@ pub enum BackgroundTask {
|
||||
/// Finds files of versions which have not been scanned for attributions
|
||||
/// yet, extracts them to find file overrides, and finds any overrides which
|
||||
/// require attribution from the creator.
|
||||
ScanFiles,
|
||||
ScanPendingFiles,
|
||||
/// Queues Discord Creator Club role claim emails for newly eligible users.
|
||||
DiscordRoleEmailCampaign,
|
||||
}
|
||||
@@ -119,7 +119,14 @@ impl BackgroundTask {
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScanFiles => scan_all_files(&pool, &redis_pool, &**file_host).await,
|
||||
ScanPendingFiles => {
|
||||
scan_all_pending_files(
|
||||
&pool,
|
||||
&redis_pool,
|
||||
file_host.into_inner(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
DiscordRoleEmailCampaign => {
|
||||
discord_role_email_campaign(pool, redis_pool).await
|
||||
}
|
||||
|
||||
@@ -166,6 +166,7 @@ vars! {
|
||||
|
||||
// storage
|
||||
STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
|
||||
FILE_SCAN_CONCURRENCY: i64 = 8i64;
|
||||
|
||||
// s3
|
||||
S3_PUBLIC_BUCKET_NAME: String = "";
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Cursor, Read};
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::Utc;
|
||||
use eyre::{Result, eyre};
|
||||
use hex::ToHex;
|
||||
use sha1::Digest;
|
||||
use tokio::task::spawn_blocking;
|
||||
use tokio::task::{spawn, spawn_blocking};
|
||||
use tracing::{Instrument, info, info_span, warn};
|
||||
use zip::ZipArchive;
|
||||
|
||||
@@ -29,6 +30,15 @@ use crate::queue::moderation::{
|
||||
use crate::util::error::Context;
|
||||
use crate::util::http::HTTP_CLIENT;
|
||||
|
||||
const PENDING_FILE_SCAN_BATCH_SIZE: i64 = 100;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PendingFileScan {
|
||||
file_id: DBFileId,
|
||||
url: String,
|
||||
project_id: DBProjectId,
|
||||
}
|
||||
|
||||
/// Attribution enforcement is version/project-scoped, not file-hash-scoped.
|
||||
///
|
||||
/// Versions or projects listed in `attributions_exemptions` predate this
|
||||
@@ -39,11 +49,50 @@ use crate::util::http::HTTP_CLIENT;
|
||||
/// versions must go through the `attribution_enforced_versions` view so
|
||||
/// grandfathered versions and projects are ignored without making the SHA1
|
||||
/// itself exempt.
|
||||
pub async fn scan_all_files(
|
||||
pub async fn scan_all_pending_files(
|
||||
db: &PgPool,
|
||||
redis: &RedisPool,
|
||||
file_host: &dyn FileHost,
|
||||
file_host: Arc<dyn FileHost>,
|
||||
) -> Result<()> {
|
||||
let scan_concurrency = ENV.FILE_SCAN_CONCURRENCY.max(1);
|
||||
|
||||
let total_to_scan = sqlx::query_scalar!(
|
||||
r#"
|
||||
select count(*) as "count!" from file_scans
|
||||
where attributions_scanned_at is null
|
||||
"#,
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.wrap_err("fetching number of files to scan")?;
|
||||
|
||||
info!(
|
||||
"Found {total_to_scan} total pending files to scan, running in batches of {PENDING_FILE_SCAN_BATCH_SIZE} with concurrency {scan_concurrency}"
|
||||
);
|
||||
|
||||
loop {
|
||||
let scanned_count = scan_pending_files_batch(
|
||||
db,
|
||||
redis,
|
||||
file_host.clone(),
|
||||
scan_concurrency * PENDING_FILE_SCAN_BATCH_SIZE,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if scanned_count == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn scan_pending_files_batch(
|
||||
db: &PgPool,
|
||||
redis: &RedisPool,
|
||||
file_host: Arc<dyn FileHost>,
|
||||
scan_limit: i64,
|
||||
) -> Result<usize> {
|
||||
let files_to_scan = sqlx::query!(
|
||||
r#"
|
||||
select
|
||||
@@ -55,14 +104,61 @@ pub async fn scan_all_files(
|
||||
inner join attribution_enforced_versions aev on aev.id = f.version_id
|
||||
inner join versions v on v.id = f.version_id
|
||||
where fa.attributions_scanned_at is null
|
||||
"#
|
||||
order by fa.file_id
|
||||
limit $1
|
||||
"#,
|
||||
scan_limit,
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.wrap_err("fetching files to scan")?;
|
||||
|
||||
info!("Found {} files to scan", files_to_scan.len());
|
||||
info!(
|
||||
"Found {} pending files to scan, splitting into jobs of {PENDING_FILE_SCAN_BATCH_SIZE}",
|
||||
files_to_scan.len(),
|
||||
);
|
||||
|
||||
let files_to_scan: Vec<_> = files_to_scan
|
||||
.into_iter()
|
||||
.map(|row| PendingFileScan {
|
||||
file_id: row.file_id,
|
||||
url: row.url,
|
||||
project_id: row.project_id,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for chunk in files_to_scan.chunks(PENDING_FILE_SCAN_BATCH_SIZE as usize) {
|
||||
let db = db.clone();
|
||||
let redis = redis.clone();
|
||||
let file_host = file_host.clone();
|
||||
let chunk = chunk.to_vec();
|
||||
|
||||
tasks.push(spawn(async move {
|
||||
scan_pending_files_chunk(&db, &redis, &*file_host, chunk).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut scanned_count = 0;
|
||||
for task in tasks {
|
||||
scanned_count += task
|
||||
.await
|
||||
.wrap_err("joining file scan task")?
|
||||
.wrap_err("scanning pending file chunk")?;
|
||||
}
|
||||
|
||||
info!("Marked {} files as scanned", scanned_count);
|
||||
|
||||
Ok(scanned_count)
|
||||
}
|
||||
|
||||
async fn scan_pending_files_chunk(
|
||||
db: &PgPool,
|
||||
redis: &RedisPool,
|
||||
file_host: &dyn FileHost,
|
||||
files_to_scan: Vec<PendingFileScan>,
|
||||
) -> Result<usize> {
|
||||
info!("Scanning {} files", files_to_scan.len());
|
||||
let mut scanned_count = 0;
|
||||
|
||||
for row in files_to_scan {
|
||||
@@ -72,10 +168,6 @@ pub async fn scan_all_files(
|
||||
info!("Scanning file");
|
||||
|
||||
let file_id = row.file_id;
|
||||
let mut txn = db
|
||||
.begin()
|
||||
.await
|
||||
.wrap_err("beginning file scan transaction")?;
|
||||
|
||||
let overrides = extract_override_files_from_storage(
|
||||
file_host, file_id, &row.url,
|
||||
@@ -87,29 +179,35 @@ pub async fn scan_all_files(
|
||||
|
||||
if overrides.is_empty() {
|
||||
info!("Found no overrides");
|
||||
} else {
|
||||
info!("Found {} overrides", overrides.len());
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let resolved = resolve_overrides(&overrides, redis, &mut txn)
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
eyre!("resolving overrides for file {file_id:?}")
|
||||
})?;
|
||||
info!("Resolved: {resolved:#?}");
|
||||
info!("Found {} overrides", overrides.len());
|
||||
|
||||
persist_attribution_results(
|
||||
row.project_id,
|
||||
file_id,
|
||||
&overrides,
|
||||
&resolved,
|
||||
redis,
|
||||
&mut txn,
|
||||
)
|
||||
let mut txn = db
|
||||
.begin()
|
||||
.await
|
||||
.wrap_err("beginning file scan transaction")?;
|
||||
|
||||
let resolved = resolve_overrides(&overrides, redis, &mut txn)
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
eyre!("persisting attribution results for file {file_id:?}")
|
||||
eyre!("resolving overrides for file {file_id:?}")
|
||||
})?;
|
||||
}
|
||||
info!("Resolved: {resolved:#?}");
|
||||
|
||||
persist_attribution_results(
|
||||
row.project_id,
|
||||
file_id,
|
||||
&overrides,
|
||||
&resolved,
|
||||
redis,
|
||||
&mut txn,
|
||||
)
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
eyre!("persisting attribution results for file {file_id:?}")
|
||||
})?;
|
||||
|
||||
let now = Utc::now();
|
||||
sqlx::query!(
|
||||
@@ -137,9 +235,7 @@ pub async fn scan_all_files(
|
||||
scanned_count += 1;
|
||||
}
|
||||
|
||||
info!("Marked {} files as scanned", scanned_count);
|
||||
|
||||
Ok(())
|
||||
Ok(scanned_count)
|
||||
}
|
||||
|
||||
pub async fn scan_file(
|
||||
@@ -276,10 +372,19 @@ fn extract_override_files(data: &[u8]) -> Result<Vec<OverrideFile>> {
|
||||
continue;
|
||||
}
|
||||
|
||||
let should_scan_file = name.contains(".jar")
|
||||
|| (name.contains(".zip") && !name.ends_with(".zip.txt"));
|
||||
|
||||
if name.matches('/').count() > 2 || !should_scan_file {
|
||||
let should_skip = name.starts_with("mods/.connector/")
|
||||
|| name.starts_with(".sable/natives/")
|
||||
|| name.starts_with("local/crash_assistant/")
|
||||
|| name.starts_with("mods/mcef-libraries/")
|
||||
|| name.starts_with("mods/mcef-cache/")
|
||||
|| name.starts_with("config/super_resolution/libraries/")
|
||||
|| name.starts_with("config/Veinminer/update/")
|
||||
|| name.starts_with("config/epicfight/native/")
|
||||
|| name.starts_with("essential/")
|
||||
|| name.ends_with(".rpo")
|
||||
|| name.ends_with(".txt");
|
||||
let should_scan = name.contains(".jar") || name.contains(".zip");
|
||||
if should_scan && !should_skip {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
use crate::auth::checks::filter_visible_versions;
|
||||
use crate::database;
|
||||
use crate::database::models::DBUserId;
|
||||
use crate::database::models::notification_item::NotificationBuilder;
|
||||
use crate::database::models::thread_item::ThreadMessageBuilder;
|
||||
use crate::database::redis::RedisPool;
|
||||
@@ -8,20 +6,15 @@ use crate::database::{PgPool, ReadOnlyPgPool};
|
||||
use crate::env::ENV;
|
||||
use crate::models::ids::ProjectId;
|
||||
use crate::models::notifications::NotificationBody;
|
||||
use crate::models::pack::{PackFile, PackFileHash, PackFormat};
|
||||
use crate::models::projects::ProjectStatus;
|
||||
use crate::models::threads::MessageBody;
|
||||
use crate::routes::ApiError;
|
||||
use dashmap::DashSet;
|
||||
use hex::ToHex;
|
||||
use itertools::Itertools;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha1::Digest;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write;
|
||||
use std::io::{Cursor, Read};
|
||||
use std::time::Duration;
|
||||
use zip::ZipArchive;
|
||||
|
||||
pub const AUTOMOD_ID: i64 = 0;
|
||||
|
||||
@@ -293,343 +286,7 @@ impl AutomatedModerationQueue {
|
||||
for version in versions {
|
||||
let primary_file = version.files.iter().find_or_first(|x| x.primary);
|
||||
|
||||
if let Some(primary_file) = primary_file {
|
||||
let data = reqwest::get(&primary_file.url).await?.bytes().await?;
|
||||
|
||||
let reader = Cursor::new(data);
|
||||
let mut zip = ZipArchive::new(reader)?;
|
||||
|
||||
let pack: PackFormat = {
|
||||
let Ok(mut file) = zip.by_name("modrinth.index.json") else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut contents = String::new();
|
||||
file.read_to_string(&mut contents)?;
|
||||
|
||||
serde_json::from_str(&contents)?
|
||||
};
|
||||
|
||||
// sha1, pack file, file path, murmur
|
||||
let mut hashes: Vec<(
|
||||
String,
|
||||
Option<PackFile>,
|
||||
String,
|
||||
Option<u32>,
|
||||
)> = pack
|
||||
.files
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter_map(|x| {
|
||||
let hash = x.hashes.get(&PackFileHash::Sha1);
|
||||
|
||||
if let Some(hash) = hash {
|
||||
let path = x.path.to_string();
|
||||
Some((hash.clone(), Some(x), path, None))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
for i in 0..zip.len() {
|
||||
let mut file = zip.by_index(i)?;
|
||||
|
||||
if file.name().starts_with("overrides/mods")
|
||||
|| file.name().starts_with("client-overrides/mods")
|
||||
|| file.name().starts_with("server-overrides/mods")
|
||||
|| file.name().starts_with("overrides/shaderpacks")
|
||||
|| file.name().starts_with("client-overrides/shaderpacks")
|
||||
|| file.name().starts_with("overrides/resourcepacks")
|
||||
|| file.name().starts_with("client-overrides/resourcepacks")
|
||||
{
|
||||
if file.name().matches('/').count() > 2 || file.name().ends_with(".txt") || file.name().ends_with(".rpo") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut contents = Vec::new();
|
||||
file.read_to_end(&mut contents)?;
|
||||
|
||||
let hash = sha1::Sha1::digest(&contents).encode_hex::<String>();
|
||||
let murmur = hash_flame_murmur32(contents);
|
||||
|
||||
hashes.push((
|
||||
hash,
|
||||
None,
|
||||
file.name().to_string(),
|
||||
Some(murmur),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let files = database::models::DBVersion::get_files_from_hash(
|
||||
"sha1".to_string(),
|
||||
&hashes.iter().map(|x| x.0.clone()).collect::<Vec<_>>(),
|
||||
&pool,
|
||||
&redis,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let version_ids =
|
||||
files.iter().map(|x| x.version_id).collect::<Vec<_>>();
|
||||
let versions_data = filter_visible_versions(
|
||||
database::models::DBVersion::get_many(
|
||||
&version_ids,
|
||||
&*ro_pool,
|
||||
&redis,
|
||||
)
|
||||
.await?,
|
||||
&None,
|
||||
&pool,
|
||||
&ro_pool,
|
||||
&redis,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut final_hashes = HashMap::new();
|
||||
|
||||
for version in versions_data {
|
||||
for file in
|
||||
files.iter().filter(|x| x.version_id == version.id.into())
|
||||
{
|
||||
if let Some(hash) = file.hashes.get("sha1")
|
||||
&& let Some((index, (sha1, _, file_name, _))) = hashes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find(|(_, (value, _, _, _))| value == hash)
|
||||
{
|
||||
final_hashes
|
||||
.insert(sha1.clone(), IdentifiedFile { status: ApprovalType::Yes, file_name: file_name.clone() });
|
||||
|
||||
hashes.remove(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All files are on Modrinth, so we don't send any messages
|
||||
if hashes.is_empty() {
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE files
|
||||
SET metadata = $1
|
||||
WHERE id = $2
|
||||
",
|
||||
serde_json::to_value(&MissingMetadata {
|
||||
identified: final_hashes,
|
||||
flame_files: HashMap::new(),
|
||||
unknown_files: HashMap::new(),
|
||||
})?,
|
||||
primary_file.id.0
|
||||
)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
let rows = sqlx::query!(
|
||||
"
|
||||
SELECT encode(mef.sha1, 'escape') sha1, mel.status status
|
||||
FROM moderation_external_files mef
|
||||
INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
|
||||
WHERE mef.sha1 = ANY($1)
|
||||
",
|
||||
&hashes.iter().map(|x| x.0.as_bytes().to_vec()).collect::<Vec<_>>()
|
||||
)
|
||||
.fetch_all(&pool)
|
||||
.await?;
|
||||
|
||||
for row in rows {
|
||||
if let Some(sha1) = row.sha1
|
||||
&& let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == &sha1) {
|
||||
final_hashes.insert(sha1.clone(), IdentifiedFile { file_name: file_name.clone(), status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified) });
|
||||
hashes.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
if hashes.is_empty() {
|
||||
let metadata = MissingMetadata {
|
||||
identified: final_hashes,
|
||||
flame_files: HashMap::new(),
|
||||
unknown_files: HashMap::new(),
|
||||
};
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE files
|
||||
SET metadata = $1
|
||||
WHERE id = $2
|
||||
",
|
||||
serde_json::to_value(&metadata)?,
|
||||
primary_file.id.0
|
||||
)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
|
||||
if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
|
||||
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
|
||||
val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let res = client
|
||||
.post(format!("{}/v1/fingerprints", ENV.FLAME_ANVIL_URL))
|
||||
.json(&serde_json::json!({
|
||||
"fingerprints": hashes.iter().filter_map(|x| x.3).collect::<Vec<u32>>()
|
||||
}))
|
||||
.send()
|
||||
.await?.text()
|
||||
.await?;
|
||||
|
||||
let flame_hashes = serde_json::from_str::<FlameResponse<FingerprintResponse>>(&res)?
|
||||
.data
|
||||
.exact_matches
|
||||
.into_iter()
|
||||
.map(|x| x.file)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut flame_files = Vec::new();
|
||||
|
||||
for file in flame_hashes {
|
||||
let hash = file
|
||||
.hashes
|
||||
.iter()
|
||||
.find(|x| x.algo == 1)
|
||||
.map(|x| x.value.clone());
|
||||
|
||||
if let Some(hash) = hash {
|
||||
flame_files.push((hash, file.mod_id))
|
||||
}
|
||||
}
|
||||
|
||||
let rows = sqlx::query!(
|
||||
"
|
||||
SELECT mel.id, mel.flame_project_id, mel.status status
|
||||
FROM moderation_external_licenses mel
|
||||
WHERE mel.flame_project_id = ANY($1)
|
||||
",
|
||||
&flame_files.iter().map(|x| x.1 as i32).collect::<Vec<_>>()
|
||||
)
|
||||
.fetch_all(&pool).await?;
|
||||
|
||||
let mut insert_hashes = Vec::new();
|
||||
let mut insert_filenames = Vec::new();
|
||||
let mut insert_ids = Vec::new();
|
||||
|
||||
for row in rows {
|
||||
if let Some((curse_index, (hash, _flame_id))) = flame_files.iter().enumerate().find(|(_, x)| Some(x.1 as i32) == row.flame_project_id)
|
||||
&& let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == hash) {
|
||||
final_hashes.insert(sha1.clone(), IdentifiedFile {
|
||||
file_name: file_name.clone(),
|
||||
status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified),
|
||||
});
|
||||
|
||||
insert_hashes.push(hash.clone().as_bytes().to_vec());
|
||||
insert_filenames.push(Some(file_name.clone()));
|
||||
insert_ids.push(row.id);
|
||||
|
||||
hashes.remove(index);
|
||||
flame_files.remove(curse_index);
|
||||
}
|
||||
}
|
||||
|
||||
if !insert_ids.is_empty() && !insert_hashes.is_empty() {
|
||||
crate::database::models::moderation_external_item::ExternalLicense::insert_files(
|
||||
&pool,
|
||||
&insert_hashes,
|
||||
&insert_filenames,
|
||||
&insert_ids,
|
||||
DBUserId(0),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if hashes.is_empty() {
|
||||
let metadata = MissingMetadata {
|
||||
identified: final_hashes,
|
||||
flame_files: HashMap::new(),
|
||||
unknown_files: HashMap::new(),
|
||||
};
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE files
|
||||
SET metadata = $1
|
||||
WHERE id = $2
|
||||
",
|
||||
serde_json::to_value(&metadata)?,
|
||||
primary_file.id.0
|
||||
)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
|
||||
if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
|
||||
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
|
||||
val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
let flame_projects = if flame_files.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
let res = client
|
||||
.post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL))
|
||||
.json(&serde_json::json!({
|
||||
"modIds": flame_files.iter().map(|x| x.1).collect::<Vec<_>>()
|
||||
}))
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
|
||||
serde_json::from_str::<FlameResponse<Vec<FlameProjectResponse>>>(&res)?.data
|
||||
};
|
||||
|
||||
let mut missing_metadata = MissingMetadata {
|
||||
identified: final_hashes,
|
||||
flame_files: HashMap::new(),
|
||||
unknown_files: HashMap::new(),
|
||||
};
|
||||
|
||||
for (sha1, _pack_file, file_name, _mumur2) in hashes {
|
||||
let flame_file = flame_files.iter().find(|x| x.0 == sha1);
|
||||
|
||||
if let Some((_, flame_project_id)) = flame_file
|
||||
&& let Some(project) = flame_projects.iter().find(|x| &x.id == flame_project_id) {
|
||||
missing_metadata.flame_files.insert(sha1, MissingMetadataFlame {
|
||||
title: project.name.clone(),
|
||||
file_name,
|
||||
url: project.links.website_url.clone(),
|
||||
id: *flame_project_id,
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
missing_metadata.unknown_files.insert(sha1, file_name);
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE files
|
||||
SET metadata = $1
|
||||
WHERE id = $2
|
||||
",
|
||||
serde_json::to_value(&missing_metadata)?,
|
||||
primary_file.id.0
|
||||
)
|
||||
.execute(&pool)
|
||||
.await?;
|
||||
|
||||
if missing_metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
|
||||
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
|
||||
val.push(ModerationMessage::PackFilesNotAllowed {files: missing_metadata.identified, incomplete: true });
|
||||
}
|
||||
} else {
|
||||
if primary_file.is_none() {
|
||||
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
|
||||
val.push(ModerationMessage::NoPrimaryFile);
|
||||
}
|
||||
@@ -922,13 +579,3 @@ pub struct FlameLogo {
|
||||
pub struct FlameLinks {
|
||||
pub website_url: String,
|
||||
}
|
||||
|
||||
fn hash_flame_murmur32(input: Vec<u8>) -> u32 {
|
||||
murmur2::murmur2(
|
||||
&input
|
||||
.into_iter()
|
||||
.filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32)
|
||||
.collect::<Vec<u8>>(),
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user