feat: new modpack permissions system (#6005)

* Begin external projects moderator database frontend

* add copy link button

* begin project page permissions settings

* MEL database backend routes

* include filename in external files

* wip: when uploading a version file, fetch its overrides as a list

* wip: override license checks

* improve FileHost ref counting

* file host read capability

* scan files when inserting version file

* add dependency sha1 field

* clean up version files

* wip: attributions

* update s3 file host

* attribution scanning basic works

* works

* insert attribution info after resolving

* add routes

* remove dep sha1 stuff

* prepr

* wip: override file sources

* add files_missing_attributions to versions

* return extended version info + attributed at/by

* hook up frontend to backend (mostly)

* expose version date published

* withholding version visibility

* frontend work

* prepr

* use api-client for img upload

* moar frontend

* prepr

* Add schema to attribution resolution and Flame project results

* sqlx prepare

* changes

* remove feature flag, fix optional proof images

* fix schema

* fmt

* fix deletion and file fetch

* prepare

* fix admonition

* update frontend stuff to new schema

* prepr

* attribution on dependencies

* fixes

* sqlx prepare

* fixes

* routes

* fix routes

* Version grandfathering

* prepare

* wip: bulk routes

* pushing what i've got rn

* include link in NoPermission

* change hash insert to bulk route

* query flame even if entry in MEL

* delete file with weird name

* Prioritise putting override files in existing groups even with ExternalLicense

* fix how hex bytes are handled in route

* feat: coolbot moderation changes (#6215)

* Update moderator checklist

* move permissions stage order

* Updated nagContext.versions to v3, added nag for permissions

* Update permissions.vue default messages

* prepr

---------

Co-authored-by: coolbot100s <76798835+coolbot100s@users.noreply.github.com>

* QA

* prepr

* should group by project

* return attribution resolution correctly

* updated by moderator info

* Track what moderator reviewed an attribution moderation status

* default deser FMA field

* new version page

* clean up fetching + add a couple missing features

* qa items

* prepr

* provide moderation package stuff with DI

* format?

* don't redact moderated_at

* move supplementary resources

* Reorganize moderation messages.

* Quick replies for external content permissions.

* prepare

* QA

* allow exempting projects

* Ignore Flame projects which 404

* fix ci

* fix cross project attribution stuff

* Fix permission error

* change what files get cscanned

* add more logging

* QA Jun 22

* fix

* idempotency

* Expose route for rescanning

* update blog link

---------

Co-authored-by: aecsocket <aecsocket@tutanota.com>
Co-authored-by: coolbot100s <76798835+coolbot100s@users.noreply.github.com>
Co-authored-by: aecsocket <43144841+aecsocket@users.noreply.github.com>

apps/labrinth/src/database/models/ids.rs

@@ -1,12 +1,13 @@
 use super::DatabaseError;
 use crate::database::PgTransaction;
 use crate::models::ids::{
-    AffiliateCodeId, AnalyticsEventId, CampaignDonationId, ChargeId,
-    CollectionId, FileId, ImageId, NotificationId, OAuthAccessTokenId,
-    OAuthClientAuthorizationId, OAuthClientId, OAuthRedirectUriId,
-    OrganizationId, PatId, PayoutId, ProductId, ProductPriceId, ProjectId,
-    ReportId, SessionId, SharedInstanceId, SharedInstanceVersionId, TeamId,
-    TeamMemberId, ThreadId, ThreadMessageId, UserSubscriptionId, VersionId,
+    AffiliateCodeId, AnalyticsEventId, AttributionGroupId, CampaignDonationId,
+    ChargeId, CollectionId, FileId, ImageId, NotificationId,
+    OAuthAccessTokenId, OAuthClientAuthorizationId, OAuthClientId,
+    OAuthRedirectUriId, OrganizationId, PatId, PayoutId, ProductId,
+    ProductPriceId, ProjectId, ReportId, SessionId, SharedInstanceId,
+    SharedInstanceVersionId, TeamId, TeamMemberId, ThreadId, ThreadMessageId,
+    UserSubscriptionId, VersionId,
 };
 use ariadne::ids::base62_impl::to_base62;
 use ariadne::ids::{UserId, random_base62_rng, random_base62_rng_range};
@@ -172,6 +173,10 @@ db_id_interface!(
     CollectionId,
     generator: generate_collection_id @ "collections",
 );
+db_id_interface!(
+    AttributionGroupId,
+    generator: generate_attribution_group_id @ "project_attribution_groups",
+);
 db_id_interface!(
     FileId,
     generator: generate_file_id @ "files",

apps/labrinth/src/database/models/project_item.rs

@@ -6,6 +6,7 @@ use super::{DBUser, ids::*};
 use crate::database::models::DatabaseError;
 use crate::database::redis::RedisPool;
 use crate::database::{PgTransaction, models};
+use crate::file_hosting::FileHost;
 use crate::models::exp;
 use crate::models::ids::ProjectId;
 use crate::models::projects::{
@@ -187,6 +188,8 @@ impl ProjectBuilder {
     pub async fn insert(
         self,
         transaction: &mut PgTransaction<'_>,
+        redis: &RedisPool,
+        file_host: &dyn FileHost,
         http: &reqwest::Client,
     ) -> Result<DBProjectId, DatabaseError> {
         let project_struct = DBProject {
@@ -235,7 +238,7 @@ impl ProjectBuilder {
 
         for mut version in self.initial_versions {
             version.project_id = self.project_id;
-            version.insert(&mut *transaction, http).await?;
+            version.insert(transaction, redis, file_host, http).await?;
         }
 
         LinkUrl::insert_many_projects(

apps/labrinth/src/database/models/version_item.rs

@@ -6,8 +6,11 @@ use crate::database::models::loader_fields::{
     QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
 };
 use crate::database::redis::RedisPool;
+use crate::file_hosting::FileHost;
 use crate::models::exp;
+
 use crate::models::projects::{FileType, VersionStatus};
+use crate::queue::file_scan::scan_file;
 use crate::routes::internal::delphi::DelphiRunParameters;
 use chrono::{DateTime, Utc};
 use dashmap::{DashMap, DashSet};
@@ -17,10 +20,31 @@ use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::iter;
+use tracing::error;
 
 pub const VERSIONS_NAMESPACE: &str = "versions";
 const VERSION_FILES_NAMESPACE: &str = "versions_files";
 
+pub async fn cleanup_empty_attribution_groups(
+    transaction: &mut PgTransaction<'_>,
+) -> Result<(), DatabaseError> {
+    sqlx::query!(
+        "
+        DELETE FROM project_attribution_groups g
+        WHERE NOT EXISTS (
+            SELECT 1
+            FROM project_attribution_files paf
+            INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1
+            WHERE paf.group_id = g.id
+        )
+        ",
+    )
+    .execute(&mut *transaction)
+    .await?;
+
+    Ok(())
+}
+
 #[derive(Clone)]
 pub struct VersionBuilder {
     pub version_id: DBVersionId,
@@ -134,7 +158,10 @@ impl VersionFileBuilder {
     pub async fn insert(
         self,
         version_id: DBVersionId,
+        project_id: DBProjectId,
         transaction: &mut PgTransaction<'_>,
+        redis: &RedisPool,
+        file_host: &dyn FileHost,
         http: &reqwest::Client,
     ) -> Result<DBFileId, DatabaseError> {
         let file_id = generate_file_id(&mut *transaction).await?;
@@ -169,6 +196,22 @@ impl VersionFileBuilder {
             .await?;
         }
 
+        let attribution_scan = sqlx::query!(
+            "
+            INSERT INTO file_scans (file_id)
+            SELECT $1
+            WHERE EXISTS (
+                SELECT 1
+                FROM attribution_enforced_versions
+                WHERE id = $2
+            )
+            ",
+            file_id as DBFileId,
+            version_id as DBVersionId,
+        )
+        .execute(&mut *transaction)
+        .await?;
+
         if let Err(err) = crate::routes::internal::delphi::run(
             &mut *transaction,
             DelphiRunParameters {
@@ -178,7 +221,21 @@ impl VersionFileBuilder {
         )
         .await
         {
-            tracing::error!("Error submitting new file to Delphi: {err}");
+            error!("Error submitting new file to Delphi: {err:?}");
+        }
+
+        if attribution_scan.rows_affected() > 0
+            && let Err(err) = scan_file(
+                &mut *transaction,
+                redis,
+                file_host,
+                project_id,
+                file_id,
+                &self.url,
+            )
+            .await
+        {
+            error!("Error scanning new file {file_id:?}: {err:?}");
         }
 
         Ok(file_id)
@@ -195,6 +252,8 @@ impl VersionBuilder {
     pub async fn insert(
         self,
         transaction: &mut PgTransaction<'_>,
+        redis: &RedisPool,
+        file_host: &dyn FileHost,
         http: &reqwest::Client,
     ) -> Result<DBVersionId, DatabaseError> {
         let version = DBVersion {
@@ -236,7 +295,15 @@ impl VersionBuilder {
         } = self;
 
         for file in files {
-            file.insert(version_id, transaction, http).await?;
+            file.insert(
+                version_id,
+                self.project_id,
+                transaction,
+                redis,
+                file_host,
+                http,
+            )
+            .await?;
         }
 
         DependencyBuilder::insert_many(
@@ -426,6 +493,8 @@ impl DBVersion {
         .execute(&mut *transaction)
         .await?;
 
+        cleanup_empty_attribution_groups(transaction).await?;
+
         // Sync dependencies
 
         let project_id = sqlx::query!(
@@ -716,7 +785,7 @@ impl DBVersion {
 
                 let dependencies : DashMap<DBVersionId, Vec<DependencyQueryResult>> = sqlx::query!(
                     "
-                    SELECT DISTINCT dependent_id as version_id, d.mod_dependency_id as dependency_project_id, d.dependency_id as dependency_version_id, d.dependency_file_name as file_name, d.dependency_type as dependency_type
+                    SELECT DISTINCT d.id as dependency_id, dependent_id as version_id, d.mod_dependency_id as dependency_project_id, d.dependency_id as dependency_version_id, d.dependency_file_name as file_name, d.dependency_type as dependency_type
                     FROM dependencies d
                     WHERE dependent_id = ANY($1)
                     ",
@@ -724,10 +793,12 @@ impl DBVersion {
                 ).fetch(&mut exec)
                     .try_fold(DashMap::new(), |acc : DashMap<_,Vec<DependencyQueryResult>>, m| {
                         let dependency = DependencyQueryResult {
+                            id: m.dependency_id,
                             project_id: m.dependency_project_id.map(DBProjectId),
                             version_id: m.dependency_version_id.map(DBVersionId),
                             file_name: m.file_name,
                             dependency_type: m.dependency_type,
+                            attribution: None,
                         };
 
                         acc.entry(DBVersionId(m.version_id))
@@ -862,14 +933,14 @@ impl DBVersion {
             })
     }
 
-    pub async fn get_files_from_hash<'a, 'b, E>(
+    pub async fn get_files_from_hash<'a, E>(
         algorithm: String,
         hashes: &[String],
         executor: E,
         redis: &RedisPool,
     ) -> Result<Vec<DBFile>, DatabaseError>
     where
-        E: crate::database::Executor<'a, Database = sqlx::Postgres> + Copy,
+        E: crate::database::Executor<'a, Database = sqlx::Postgres>,
     {
         let val = redis.get_cached_keys(
             VERSION_FILES_NAMESPACE,
@@ -977,10 +1048,12 @@ pub struct VersionQueryResult {
 
 #[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
 pub struct DependencyQueryResult {
+    pub id: i32,
     pub project_id: Option<DBProjectId>,
     pub version_id: Option<DBVersionId>,
     pub file_name: Option<String>,
     pub dependency_type: String,
+    pub attribution: Option<crate::models::projects::DependencyAttribution>,
 }
 
 #[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]