sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-08-25 00:55:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: attribution scanning details (#6651)

Browse Source 
* key override file sources on file path, allow deleting groups

* dont scan modrinth files

* delete attr group button

* return what override files already exist on platform

* intl:extract

* uppercase queries

* fix dupe migration key
This commit is contained in:
aecsocket
2026-07-12 09:04:54 +00:00
committed by GitHub
parent 1741d12b7b
commit e1556a8ee3
15 changed files with 346 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/labrinth/.sqlx/query-00ef039799c162e21a8fa5ba7aec2a9044a3028f1cb34e8c9c565a670956d938.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tSELECT DISTINCT f.version_id AS \"version_id: DBVersionId\"\n\t\tFROM project_attribution_files paf\n\t\tINNER JOIN project_attribution_groups pag ON pag.id = paf.group_id\n\t\tINNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n\t\tINNER JOIN files f ON f.id = ofs.file_id\n\t\tINNER JOIN versions v ON v.id = f.version_id\n\t\tWHERE paf.group_id = $1\n\t\t\tAND pag.project_id = v.mod_id\n\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_id: DBVersionId",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "00ef039799c162e21a8fa5ba7aec2a9044a3028f1cb34e8c9c565a670956d938"
}
apps/labrinth/.sqlx/query-079846a1e6a6b080e0e7f2e3efbb53b6526332433faf66de8716bc5cd2b12afd.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n select id\n from mods\n where id = any($1)\n and status not in ('rejected', 'draft', 'withheld', 'withdrawn')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8Array"
]
},
"nullable": [
false
]
},
"hash": "079846a1e6a6b080e0e7f2e3efbb53b6526332433faf66de8716bc5cd2b12afd"
}
apps/labrinth/.sqlx/query-827534d191739cd2ea502142a65f62a052715c7441952ee4ee182afde07c44d7.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tDELETE FROM project_attribution_groups\n\t\tWHERE id = $1\n\t\t",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "827534d191739cd2ea502142a65f62a052715c7441952ee4ee182afde07c44d7"
}
apps/labrinth/.sqlx/query-9a0a10cd75d0c9317ea599d21e7935c4ec289b1fc6417eb0bdb2feddd94f175e.json
Generated
+46
View File
@@ -0,0 +1,46 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\t\tSELECT\n\t\t\t\tofs.file_path AS \"file_path!\",\n\t\t\t\tCONVERT_FROM(ofs.sha1, 'UTF8') AS \"sha1!\",\n\t\t\t\tsource_file.version_id AS \"version_id: DBVersionId\",\n\t\t\t\tplatform_file.version_id AS \"platform_version_id: DBVersionId\",\n\t\t\t\tplatform_version.mod_id AS \"platform_project_id: DBProjectId\"\n\t\t\tFROM files source_file\n\t\t\tINNER JOIN versions source_version\n\t\t\t\tON source_version.id = source_file.version_id\n\t\t\tINNER JOIN override_file_sources ofs\n\t\t\t\tON ofs.file_id = source_file.id\n\t\t\tINNER JOIN hashes h\n\t\t\t\tON h.algorithm = 'sha1' AND h.hash = ofs.sha1\n\t\t\tINNER JOIN files platform_file\n\t\t\t\tON platform_file.id = h.file_id\n\t\t\tINNER JOIN versions platform_version\n\t\t\t\tON platform_version.id = platform_file.version_id\n\t\t\tWHERE source_version.mod_id = $1\n\t\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "file_path!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "sha1!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "version_id: DBVersionId",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "platform_version_id: DBVersionId",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "platform_project_id: DBProjectId",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
null,
false,
false,
false
]
},
"hash": "9a0a10cd75d0c9317ea599d21e7935c4ec289b1fc6417eb0bdb2feddd94f175e"
}
apps/labrinth/.sqlx/query-b69317d234e934aa6bdf8be4d35bc7475308bde8b7f34a4ef3c7f3ef351f46a8.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tDELETE FROM project_attribution_files\n\t\tWHERE group_id = $1\n\t\t",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "b69317d234e934aa6bdf8be4d35bc7475308bde8b7f34a4ef3c7f3ef351f46a8"
}
apps/labrinth/.sqlx/query-c1482e5ded6536a24937a9396a3f141270b3acf96ec832a50b5f7c572e255947.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO override_file_sources (sha1, file_id, file_path)\n SELECT source.sha1, $2, source.file_path\n FROM UNNEST($1::BYTEA[], $3::TEXT[]) AS source(sha1, file_path)\n ON CONFLICT DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"ByteaArray",
"Int8",
"TextArray"
]
},
"nullable": []
},
"hash": "c1482e5ded6536a24937a9396a3f141270b3acf96ec832a50b5f7c572e255947"
}
apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n insert into override_file_sources (sha1, file_id)\n select unnest($1::bytea[]), $2\n on conflict do nothing\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"ByteaArray",
"Int8"
]
},
"nullable": []
},
"hash": "e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb"
}
apps/labrinth/migrations/20260707120001_override_file_sources_file_path.sql
+4
View File
@@ -0,0 +1,4 @@
alter table override_file_sources
add column file_path text not null default '',
drop constraint override_file_sources_pkey,
add primary key (sha1, file_id, file_path);
apps/labrinth/src/queue/file_scan.rs
+12 -68
View File
@@ -231,7 +231,7 @@ async fn scan_pending_files_chunk(
.await
.wrap_err("beginning file scan transaction")?;
let resolved = resolve_overrides(&overrides, redis, &mut txn)
let resolved = resolve_overrides(&overrides, &mut txn)
.await
.wrap_err_with(|| {
eyre!("resolving overrides for file {file_id:?}")
@@ -325,9 +325,8 @@ async fn scan_file_inner(
if !overrides.is_empty() {
let before = count_project_attributions(project_id, txn).await?;
let resolved = resolve_overrides(&overrides, redis, txn)
.await
.wrap_err_with(|| {
let resolved =
resolve_overrides(&overrides, txn).await.wrap_err_with(|| {
eyre!("resolving overrides for file {file_id:?}")
})?;
@@ -494,7 +493,6 @@ pub struct OverrideFile {
#[derive(Debug)]
pub enum OverrideResolution {
OnModrinth,
ExternalLicense {
id: i64,
status: ApprovalType,
@@ -518,8 +516,7 @@ fn log_marked_override_projects(
| OverrideResolution::Flame(flame_project) => {
Some(format!("{} ({})", flame_project.title, flame_project.id))
}
OverrideResolution::OnModrinth
| OverrideResolution::ExternalLicense { .. }
OverrideResolution::ExternalLicense { .. }
| OverrideResolution::Unknown => None,
})
.collect::<Vec<_>>();
@@ -671,7 +668,6 @@ async fn persist_attribution_results(
}
match resolved.get(&file.sha1) {
Some(OverrideResolution::OnModrinth) => continue,
Some(OverrideResolution::ExternalLicense {
id,
status,
@@ -864,14 +860,19 @@ async fn persist_attribution_results(
}
if !all_sha1s.is_empty() {
let file_paths: Vec<String> =
overrides.iter().map(|f| f.path.clone()).collect();
sqlx::query!(
"
insert into override_file_sources (sha1, file_id)
select unnest($1::bytea[]), $2
on conflict do nothing
INSERT INTO override_file_sources (sha1, file_id, file_path)
SELECT source.sha1, $2, source.file_path
FROM UNNEST($1::BYTEA[], $3::TEXT[]) AS source(sha1, file_path)
ON CONFLICT DO NOTHING
",
&all_sha1s,
file_id as DBFileId,
&file_paths,
)
.execute(&mut *txn)
.await
@@ -937,7 +938,6 @@ fn default_external_license_attribution(
async fn resolve_overrides(
overrides: &[OverrideFile],
redis: &RedisPool,
txn: &mut PgTransaction<'_>,
) -> Result<HashMap<String, OverrideResolution>> {
let mut results: HashMap<String, OverrideResolution> = HashMap::new();
@@ -947,62 +947,6 @@ async fn resolve_overrides(
return Ok(results);
}
let hashes: Vec<String> =
overrides.iter().map(|x| x.sha1.clone()).collect();
let files = DBVersion::get_files_from_hash(
"sha1".to_string(),
&hashes,
&mut *txn,
redis,
)
.await
.wrap_err("fetching files on platform by hash")?;
let matching_project_ids: Vec<_> =
files.iter().map(|file| file.project_id.0).collect();
let valid_project_ids = sqlx::query_scalar!(
r#"
select id
from mods
where id = any($1)
and status not in ('rejected', 'draft', 'withheld', 'withdrawn')
"#,
&matching_project_ids,
)
.fetch_all(&mut *txn)
.await
.wrap_err("fetching matched file project statuses")?;
let version_ids: Vec<_> = files.iter().map(|x| x.version_id).collect();
let versions_data = DBVersion::get_many(&version_ids, &mut *txn, redis)
.await
.wrap_err("fetching versions")?;
for file in &files {
if !valid_project_ids.contains(&file.project_id.0) {
continue;
}
if !versions_data.iter().any(|v| v.inner.id == file.version_id) {
continue;
}
if let Some(hash) = file.hashes.get("sha1")
&& let Some(pos) =
remaining.iter().position(|i| overrides[*i].sha1 == *hash)
{
let idx = remaining.remove(pos);
results.insert(
overrides[idx].sha1.clone(),
OverrideResolution::OnModrinth,
);
}
}
if remaining.is_empty() {
return Ok(results);
}
let rows = sqlx::query!(
"
SELECT encode(mef.sha1, 'escape') sha1, mel.id, mel.status status, mel.link
apps/labrinth/src/routes/internal/attribution.rs
+146 -1
View File
@@ -1,4 +1,4 @@
use actix_web::{HttpRequest, get, patch, post, web};
use actix_web::{HttpRequest, delete, get, patch, post, web};
use chrono::{DateTime, Utc};
use eyre::eyre;
use serde::{Deserialize, Serialize};
@@ -31,6 +31,7 @@ use crate::util::error::Context;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(list)
.service(update_group)
.service(delete_group)
.service(scan)
.service(force_scan_file)
.service(assign)
@@ -46,6 +47,7 @@ struct AttributionGroupResponse {
attributed_by: Option<ariadne::ids::UserId>,
files: Vec<AttributionFileResponse>,
versions: Vec<VersionInfo>,
override_files_on_platform: Vec<OverrideFileOnPlatformResponse>,
}
#[derive(Clone, Serialize, utoipa::ToSchema)]
@@ -67,6 +69,15 @@ struct AttributionFileResponse {
moderation_external_license: Option<ModerationExternalLicenseResponse>,
}
#[derive(Clone, Serialize, utoipa::ToSchema)]
struct OverrideFileOnPlatformResponse {
file_path: String,
sha1: String,
version_id: VersionId,
platform_version_id: VersionId,
platform_project_id: ProjectId,
}
#[derive(Clone, Serialize, utoipa::ToSchema)]
struct ModerationExternalLicenseResponse {
id: i64,
@@ -358,6 +369,48 @@ pub async fn list(
.wrap_internal_err("failed to fetch attribution group files")?
};
let override_files_on_platform = if requester_is_mod {
sqlx::query!(
r#"
SELECT
ofs.file_path AS "file_path!",
CONVERT_FROM(ofs.sha1, 'UTF8') AS "sha1!",
source_file.version_id AS "version_id: DBVersionId",
platform_file.version_id AS "platform_version_id: DBVersionId",
platform_version.mod_id AS "platform_project_id: DBProjectId"
FROM files source_file
INNER JOIN versions source_version
ON source_version.id = source_file.version_id
INNER JOIN override_file_sources ofs
ON ofs.file_id = source_file.id
INNER JOIN hashes h
ON h.algorithm = 'sha1' AND h.hash = ofs.sha1
INNER JOIN files platform_file
ON platform_file.id = h.file_id
INNER JOIN versions platform_version
ON platform_version.id = platform_file.version_id
WHERE source_version.mod_id = $1
"#,
project_id as DBProjectId,
)
.fetch_all(pool.as_ref())
.await
.wrap_internal_err(
"failed to fetch override files already on platform",
)?
.into_iter()
.map(|row| OverrideFileOnPlatformResponse {
file_path: row.file_path,
sha1: row.sha1,
version_id: row.version_id.into(),
platform_version_id: row.platform_version_id.into(),
platform_project_id: row.platform_project_id.into(),
})
.collect::<Vec<_>>()
} else {
Vec::new()
};
let moderation_external_licenses = if requester_is_mod {
let mut ids: Vec<i64> = files
.iter()
@@ -497,6 +550,15 @@ pub async fn list(
.filter(|version| group_version_ids.contains(&version.id))
.cloned()
.collect();
let group_file_sha1s = group_files
.iter()
.map(|file| file.sha1.as_str())
.collect::<std::collections::HashSet<_>>();
let group_override_files_on_platform = override_files_on_platform
.iter()
.filter(|file| group_file_sha1s.contains(file.sha1.as_str()))
.cloned()
.collect();
let mut attribution = group.attribution.and_then(|v| {
serde_json::from_value::<AttributionResolution>(v).ok()
@@ -530,6 +592,7 @@ pub async fn list(
attributed_by,
files: group_files,
versions: group_versions,
override_files_on_platform: group_override_files_on_platform,
});
}
@@ -620,6 +683,88 @@ pub async fn update_group(
Ok(())
}
/// Delete an attribution group and all files inside it.
#[utoipa::path(
context_path = "/attribution",
tag = "attribution",
responses((status = NO_CONTENT))
)]
#[delete("/group/{group_id}")]
pub async fn delete_group(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
path: web::Path<i64>,
) -> Result<(), ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Scopes::PROJECT_READ,
)
.await?;
let mut txn = pool.begin().await.wrap_internal_err(
"failed to begin attribution group deletion transaction",
)?;
let group_id = path.into_inner();
let version_ids = sqlx::query_scalar!(
r#"
SELECT DISTINCT f.version_id AS "version_id: DBVersionId"
FROM project_attribution_files paf
INNER JOIN project_attribution_groups pag ON pag.id = paf.group_id
INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1
INNER JOIN files f ON f.id = ofs.file_id
INNER JOIN versions v ON v.id = f.version_id
WHERE paf.group_id = $1
AND pag.project_id = v.mod_id
"#,
group_id,
)
.fetch_all(&mut txn)
.await
.wrap_internal_err("failed to fetch attribution group versions")?;
sqlx::query!(
"
DELETE FROM project_attribution_files
WHERE group_id = $1
",
group_id,
)
.execute(&mut txn)
.await
.wrap_internal_err("failed to delete attribution group files")?;
let result = sqlx::query!(
"
DELETE FROM project_attribution_groups
WHERE id = $1
",
group_id,
)
.execute(&mut txn)
.await
.wrap_internal_err("failed to delete attribution group")?;
if result.rows_affected() == 0 {
return Err(ApiError::NotFound);
}
txn.commit().await.wrap_internal_err(
"failed to commit attribution group deletion transaction",
)?;
DBVersion::clear_cache_ids(&version_ids, redis.as_ref())
.await
.wrap_internal_err("failed to clear version attribution cache")?;
Ok(())
}
#[derive(Deserialize, utoipa::ToSchema)]
struct AssignBody {
sha1: String,
apps/labrinth/src/routes/internal/mod.rs
+1
View File
@@ -142,6 +142,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
attribution::scan,
attribution::list,
attribution::update_group,
attribution::delete_group,
attribution::assign,
attribution::split,
billing::products,