diff --git a/apps/frontend/src/components/ui/moderation/GlobalDetailLocalTraceCard.vue b/apps/frontend/src/components/ui/moderation/GlobalDetailLocalTraceCard.vue index a6756ce443..8329b64b33 100644 --- a/apps/frontend/src/components/ui/moderation/GlobalDetailLocalTraceCard.vue +++ b/apps/frontend/src/components/ui/moderation/GlobalDetailLocalTraceCard.vue @@ -5,14 +5,8 @@

{{ trace.project_name }}

-

- {{ trace.version_number }} -

+

@@ -33,9 +27,11 @@ diff --git a/apps/frontend/src/components/ui/moderation/GlobalDetailTracesList.vue b/apps/frontend/src/components/ui/moderation/GlobalDetailTracesList.vue index 8d42abbc1e..1d4fd8cb8a 100644 --- a/apps/frontend/src/components/ui/moderation/GlobalDetailTracesList.vue +++ b/apps/frontend/src/components/ui/moderation/GlobalDetailTracesList.vue @@ -65,7 +65,7 @@

Path - {{ decodeTracePath(getLatestLocalTrace(trace)?.file_path ?? '') }} +

@@ -133,6 +133,7 @@ import { } from '@modrinth/ui' import GlobalDetailLocalTraceCard from '~/components/ui/moderation/GlobalDetailLocalTraceCard.vue' +import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue' const client = injectModrinthClient() const { addNotification } = injectNotificationManager() @@ -165,14 +166,6 @@ function getLatestLocalTrace(trace: Labrinth.TechReview.Internal.GlobalIssueDeta return trace.local_traces.at(-1) } -function decodeTracePath(path: string): string { - try { - return decodeURIComponent(path) - } catch { - return path - } -} - function getSeverityBadgeColor( severity: Labrinth.TechReview.Internal.DelphiSeverity | undefined, ): string { diff --git a/apps/frontend/src/components/ui/moderation/IssueDetailPath.vue b/apps/frontend/src/components/ui/moderation/IssueDetailPath.vue new file mode 100644 index 0000000000..6289a9cd56 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/IssueDetailPath.vue @@ -0,0 +1,74 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue b/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue index f91bd67552..3e595510ca 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue @@ -6,7 +6,6 @@ import { CheckCheckIcon, CheckIcon, ChevronDownIcon, - ChevronRightIcon, ClipboardCopyIcon, CodeIcon, CopyIcon, @@ -49,6 +48,7 @@ import { import dayjs from 'dayjs' import { computed, nextTick, reactive, ref, watch } from 'vue' +import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue' import type { UnsafeFile } from '~/components/ui/moderation/MaliciousSummaryModal.vue' import ThreadView from '~/components/ui/thread/ThreadView.vue' @@ -601,6 +601,29 @@ async function copyToClipboard(code: string, detailId: string) { } } +async function copyDetailCelInput(detailId: string) { + if (copyingCelDetails.has(detailId)) return + + copyingCelDetails.add(detailId) + try { + const input = await client.labrinth.tech_review_internal.getDetailRuleInput(detailId) + await navigator.clipboard.writeText(JSON.stringify(input, null, 2)) + copiedCelDetails.add(detailId) + setTimeout(() => { + copiedCelDetails.delete(detailId) + }, 2000) + } catch (error) { + console.error('Failed to copy CEL input:', error) + addNotification({ + type: 'error', + title: 'Failed to copy CEL input', + text: 'An error occurred while loading the trace rule input.', + }) + } finally { + copyingCelDetails.delete(detailId) + } +} + function getDetailDecision( detailId: string, backendStatus: Labrinth.TechReview.Internal.DelphiReportIssueStatus, @@ -999,6 +1022,8 @@ async function updateGlobalDetailStatus( const expandedClasses = reactive>(new Set()) const autoExpandedFileIds = reactive>(new Set()) const showCopyFeedback = reactive>(new Map()) +const copyingCelDetails = reactive>(new Set()) +const copiedCelDetails = reactive>(new Set()) const highlightedSourceCache = reactive>(new Map()) const LAZY_LOAD_CLASS_SOURCE_MINIMUM = 2 @@ -1025,7 +1050,7 @@ interface JarGroup { function splitJarSegments(jar: string | null, currentFileName: string | null): string[] { if (!jar) return [] const segments = jar - .split(/[/#]/) + .split('#') .map((s) => decodeURIComponent(s.trim())) .filter((s) => s.length > 0) // Skip the first segment if it matches the current file tab (it's already shown in the file list) @@ -1770,27 +1795,12 @@ function copyId() { class="border-b border-solid border-surface-1 px-4 py-3" >
-
- -
+
- {{ - truncateMiddle(classItem.filePath) - }} +
+ + +

Path - {{ decodeTracePath(latestLocalTrace.file_path) }} +

@@ -103,6 +103,7 @@ import { } from '@modrinth/ui' import GlobalDetailLocalTraceCard from '~/components/ui/moderation/GlobalDetailLocalTraceCard.vue' +import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue' const client = injectModrinthClient() const { addNotification } = injectNotificationManager() @@ -137,14 +138,6 @@ const pageEnd = computed(() => ) const latestLocalTrace = computed(() => trace.value?.local_traces.at(-1)) -function decodeTracePath(path: string): string { - try { - return decodeURIComponent(path) - } catch { - return path - } -} - function getSeverityBadgeColor(severity: Labrinth.TechReview.Internal.DelphiSeverity): string { switch (severity) { case 'severe': diff --git a/apps/frontend/src/pages/moderation/technical-review/rules.vue b/apps/frontend/src/pages/moderation/technical-review/rules.vue index 254415f84d..66035cd75e 100644 --- a/apps/frontend/src/pages/moderation/technical-review/rules.vue +++ b/apps/frontend/src/pages/moderation/technical-review/rules.vue @@ -327,11 +327,11 @@

- - {{ detail.file_path }} +

@@ -385,7 +385,6 @@ import { type Labrinth, SseParser } from '@modrinth/api-client' import { ArrowLeftIcon, - ChevronRightIcon, EditIcon, EyeOffIcon, ExternalIcon, @@ -409,6 +408,8 @@ import { useDebounceFn } from '@vueuse/core' import type { Ace } from 'ace-builds' import type { Component } from 'vue' +import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue' + const DEFAULT_RULE = `input.trace.issue_type == "OBFUSCATED_NAMES" ? {"severity": "low", "hidden": false} : null` diff --git a/apps/labrinth/migrations/20260723120000_delphi_effective_rule_details.sql b/apps/labrinth/migrations/20260723120000_delphi_effective_rule_details.sql new file mode 100644 index 0000000000..65dc4e2536 --- /dev/null +++ b/apps/labrinth/migrations/20260723120000_delphi_effective_rule_details.sql @@ -0,0 +1,37 @@ +DROP VIEW delphi_issue_details_with_statuses; + +CREATE VIEW delphi_issue_details_with_statuses AS +SELECT + drid.id, + drid.issue_id, + drid.key, + drid.jar, + drid.file_path, + drid.decompiled_source, + drid.data, + drid.severity AS original_severity, + COALESCE(dre.severity, drid.severity) AS severity, + COALESCE(dre.hidden, FALSE) AS hidden, + m.id AS project_id, + didv.verdict AS local_status, + dgdv.verdict AS global_status, + COALESCE(dgdv.verdict, didv.verdict, 'pending') AS status +FROM delphi_report_issue_details drid +INNER JOIN delphi_report_issues dri ON dri.id = drid.issue_id +INNER JOIN delphi_reports dr ON dr.id = dri.report_id +INNER JOIN files f ON f.id = dr.file_id +INNER JOIN versions v ON v.id = f.version_id +INNER JOIN mods m ON m.id = v.mod_id +LEFT JOIN delphi_global_detail_verdicts dgdv + ON drid.key = dgdv.detail_key +LEFT JOIN delphi_issue_detail_verdicts didv + ON m.id = didv.project_id + AND drid.key = didv.detail_key +LEFT JOIN ( + SELECT revision + FROM delphi_rule_revisions + LIMIT 1 +) drr ON TRUE +LEFT JOIN delphi_rule_effects dre + ON dre.revision = drr.revision + AND dre.detail_id = drid.id; diff --git a/apps/labrinth/src/routes/internal/delphi/mod.rs b/apps/labrinth/src/routes/internal/delphi/mod.rs index cf92fc312b..ed275dfb6c 100644 --- a/apps/labrinth/src/routes/internal/delphi/mod.rs +++ b/apps/labrinth/src/routes/internal/delphi/mod.rs @@ -210,6 +210,7 @@ async fn ingest_report_deserialized( "Delphi found issues in file", ); + let mut inserted_detail_ids = Vec::new(); for (issue_type, issue_details) in report.issues { let issue_id = DBDelphiReportIssue { id: DelphiReportIssueId(0), // This will be set by the database @@ -229,7 +230,7 @@ async fn ingest_report_deserialized( let decompiled_source = report.decompiled_sources.get(&issue_detail.file); - ReportIssueDetail { + let detail_id = ReportIssueDetail { id: DelphiReportIssueDetailsId(0), // This will be set by the database issue_id, key: issue_detail.key.0, @@ -245,9 +246,17 @@ async fn ingest_report_deserialized( .insert(&mut transaction) .await .wrap_internal_err("failed to insert Delphi issue detail")?; + inserted_detail_ids.push(detail_id); } } + crate::routes::internal::moderation::tech_review::rules_scan::materialize_current_rule_effects( + &inserted_detail_ids, + &mut transaction, + ) + .await + .wrap_internal_err("failed to apply delphi rules to new issue details")?; + tech_review_sync::sync_project_tech_review_state( &[DBProjectId::from(report.project_id)], tech_review_sync::TechReviewExitReason::Resolved, diff --git a/apps/labrinth/src/routes/internal/delphi/rescan.rs b/apps/labrinth/src/routes/internal/delphi/rescan.rs index c7d7fa297b..cbaefd58f7 100644 --- a/apps/labrinth/src/routes/internal/delphi/rescan.rs +++ b/apps/labrinth/src/routes/internal/delphi/rescan.rs @@ -105,6 +105,7 @@ async fn fetch_unreviewed_tech_review_project_ids( WHERE didws.project_id = m.id AND didws.status = 'pending' + AND NOT didws.hidden -- see delphi.rs todo comment AND dri.issue_type != '__dummy' ) @@ -115,6 +116,7 @@ async fn fetch_unreviewed_tech_review_project_ids( WHERE didws.project_id = m.id AND didws.status IN ('safe', 'unsafe') + AND NOT didws.hidden -- see delphi.rs todo comment AND dri.issue_type != '__dummy' ) diff --git a/apps/labrinth/src/routes/internal/delphi/tech_review_sync.rs b/apps/labrinth/src/routes/internal/delphi/tech_review_sync.rs index 23a45d218a..974db515d0 100644 --- a/apps/labrinth/src/routes/internal/delphi/tech_review_sync.rs +++ b/apps/labrinth/src/routes/internal/delphi/tech_review_sync.rs @@ -60,9 +60,11 @@ const DUMMY_ISSUE_TYPE: &str = "__dummy"; pub enum TechReviewExitReason { Resolved, FileDeleted, + RulesChanged, } struct ProjectTechReviewState { + project_id: DBProjectId, has_pending_detail: bool, has_unsafe_detail: bool, has_dummy: bool, @@ -88,36 +90,35 @@ pub async fn sync_project_tech_review_state( r#" WITH project_ids AS ( SELECT unnest($1::bigint[]) AS project_id + ), + project_detail_state AS ( + SELECT + p.project_id, + COALESCE(BOOL_OR( + didws.status = 'pending' + AND NOT didws.hidden + AND dri.issue_type != $3 + ), FALSE) AS has_pending_detail, + COALESCE(BOOL_OR( + didws.status = 'unsafe' + AND NOT didws.hidden + AND dri.issue_type != $3 + ), FALSE) AS has_unsafe_detail, + COALESCE(BOOL_OR( + didws.status = 'pending' + AND dri.issue_type = $3 + ), FALSE) AS has_dummy + FROM project_ids p + LEFT JOIN delphi_issue_details_with_statuses didws + ON didws.project_id = p.project_id + LEFT JOIN delphi_report_issues dri ON dri.id = didws.issue_id + GROUP BY p.project_id ) SELECT p.project_id AS "project_id!: DBProjectId", - EXISTS( - SELECT 1 - FROM delphi_issue_details_with_statuses didws - INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id - WHERE - didws.project_id = p.project_id - AND didws.status = 'pending' - AND dri.issue_type != $3 - ) AS "has_pending_detail!", - EXISTS( - SELECT 1 - FROM delphi_issue_details_with_statuses didws - INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id - WHERE - didws.project_id = p.project_id - AND didws.status = 'unsafe' - AND dri.issue_type != $3 - ) AS "has_unsafe_detail!", - EXISTS( - SELECT 1 - FROM delphi_issue_details_with_statuses didws - INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id - WHERE - didws.project_id = p.project_id - AND didws.status = 'pending' - AND dri.issue_type = $3 - ) AS "has_dummy!", + p.has_pending_detail AS "has_pending_detail!", + p.has_unsafe_detail AS "has_unsafe_detail!", + p.has_dummy AS "has_dummy!", ( SELECT t.id FROM threads t @@ -144,7 +145,7 @@ pub async fn sync_project_tech_review_state( ORDER BY dr.created DESC, dr.id DESC LIMIT 1 ) AS "report_id: DelphiReportId" - FROM project_ids p + FROM project_detail_state p "#, &project_ids_raw, &tech_review_message_types, @@ -156,6 +157,7 @@ pub async fn sync_project_tech_review_state( for row in rows { let state = ProjectTechReviewState { + project_id: row.project_id, has_pending_detail: row.has_pending_detail, has_unsafe_detail: row.has_unsafe_detail, has_dummy: row.has_dummy, @@ -250,8 +252,23 @@ async fn sync_one_project_tech_review_state( exit_reason: TechReviewExitReason, txn: &mut PgTransaction<'_>, ) -> Result<(), ApiError> { - let needs_tech_review = - state.has_pending_detail || state.has_unsafe_detail || state.has_dummy; + let has_review_detail = state.has_pending_detail || state.has_unsafe_detail; + + if matches!(exit_reason, TechReviewExitReason::RulesChanged) + && !has_review_detail + { + remove_dummy_issue_details(state.project_id, txn).await?; + + if let Some(thread_id) = state.thread_id + && should_send_exit(state.last_tech_review_message_type.as_deref()) + { + insert_exit_message(thread_id, exit_reason, txn).await?; + } + + return Ok(()); + } + + let needs_tech_review = has_review_detail || state.has_dummy; if needs_tech_review { if (state.has_pending_detail || state.has_unsafe_detail) @@ -325,7 +342,9 @@ async fn insert_exit_message( txn: &mut PgTransaction<'_>, ) -> Result<(), ApiError> { let body = match exit_reason { - TechReviewExitReason::Resolved => MessageBody::TechReviewExited, + TechReviewExitReason::Resolved | TechReviewExitReason::RulesChanged => { + MessageBody::TechReviewExited + } TechReviewExitReason::FileDeleted => { MessageBody::TechReviewExitFileDeleted } @@ -344,6 +363,36 @@ async fn insert_exit_message( Ok(()) } +async fn remove_dummy_issue_details( + project_id: DBProjectId, + txn: &mut PgTransaction<'_>, +) -> Result<(), ApiError> { + sqlx::query!( + r#" + DELETE FROM delphi_report_issue_details detail + USING + delphi_report_issues issue, + delphi_reports report, + files file, + versions version + WHERE + detail.issue_id = issue.id + AND issue.issue_type = $2 + AND issue.report_id = report.id + AND report.file_id = file.id + AND file.version_id = version.id + AND version.mod_id = $1 + "#, + project_id as DBProjectId, + DUMMY_ISSUE_TYPE, + ) + .execute(&mut *txn) + .await + .wrap_internal_err("failed to remove dummy Delphi issue details")?; + + Ok(()) +} + async fn ensure_dummy_issue_detail( report_id: DelphiReportId, txn: &mut PgTransaction<'_>, diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 6dc3734377..c01423a488 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -122,6 +122,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { moderation::tech_review::rules::update_rule, moderation::tech_review::rules::delete_rule, moderation::tech_review::rules_scan::get_rule_schema, + moderation::tech_review::rules_scan::get_detail_rule_input, moderation::tech_review::rules_scan::scan_rules, moderation::tech_review::get_project_report, moderation::tech_review::submit_report, diff --git a/apps/labrinth/src/routes/internal/moderation/mod.rs b/apps/labrinth/src/routes/internal/moderation/mod.rs index 7b3dac3ca1..e745939f89 100644 --- a/apps/labrinth/src/routes/internal/moderation/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/mod.rs @@ -295,6 +295,7 @@ pub async fn get_projects_internal( FROM delphi_issue_details_with_statuses didws WHERE didws.project_id = m.id AND didws.status = 'pending' + AND NOT didws.hidden ) ) ), @@ -542,6 +543,7 @@ pub async fn get_projects_internal( FROM delphi_issue_details_with_statuses didws WHERE didws.project_id = m.id AND didws.status = 'pending' + AND NOT didws.hidden ) ) ), @@ -770,6 +772,7 @@ pub async fn get_project_ids( FROM delphi_issue_details_with_statuses didws WHERE didws.project_id = m.id AND didws.status = 'pending' + AND NOT didws.hidden ) ) ), @@ -898,6 +901,7 @@ pub async fn get_project_ids( FROM delphi_issue_details_with_statuses didws WHERE didws.project_id = mods.id AND didws.status = 'pending' + AND NOT didws.hidden ) ) ORDER BY diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review.rs b/apps/labrinth/src/routes/internal/moderation/tech_review.rs index 92b230cd79..853d0b05fb 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review.rs @@ -254,7 +254,9 @@ pub async fn get_issue( ) ), '[]'::jsonb) FROM delphi_issue_details_with_statuses didws - WHERE didws.issue_id = dri.id + WHERE + didws.issue_id = dri.id + AND NOT didws.hidden ) ) AS "data!: sqlx::types::Json" FROM delphi_report_issues dri @@ -309,6 +311,16 @@ pub async fn get_report( 'file_size', f.size, 'flag_reason', 'delphi', 'download_url', f.url, + 'severity', COALESCE(( + SELECT MAX(didws.severity) + FROM delphi_report_issues severity_issue + INNER JOIN delphi_issue_details_with_statuses didws + ON didws.issue_id = severity_issue.id + WHERE + severity_issue.report_id = dr.id + AND severity_issue.issue_type != '__dummy' + AND NOT didws.hidden + ), 'low'::delphi_severity), -- TODO: replace with `json_array` in Postgres 16 'issues', ( SELECT coalesce(json_agg( @@ -331,15 +343,24 @@ pub async fn get_report( ) ), '[]'::jsonb) FROM delphi_issue_details_with_statuses didws - WHERE didws.issue_id = dri.id + WHERE + didws.issue_id = dri.id + AND NOT didws.hidden ) ) ), '[]'::json) FROM delphi_report_issues dri WHERE - dri.report_id = dr.id + dri.report_id = dr.id -- see delphi.rs todo comment AND dri.issue_type != '__dummy' + AND EXISTS ( + SELECT 1 + FROM delphi_issue_details_with_statuses visible_detail + WHERE + visible_detail.issue_id = dri.id + AND NOT visible_detail.hidden + ) ) ) AS "data!: sqlx::types::Json" FROM delphi_reports dr @@ -534,7 +555,9 @@ async fn fetch_project_reports( didws.global_status AS "global_status?: DelphiStatus", didws.status AS "status!: DelphiStatus" FROM delphi_issue_details_with_statuses didws - WHERE didws.issue_id = ANY($1::bigint[]) + WHERE + didws.issue_id = ANY($1::bigint[]) + AND NOT didws.hidden ORDER BY didws.issue_id, didws.id "#, &issue_ids.iter().map(|i| i.0).collect::>() @@ -640,6 +663,10 @@ async fn fetch_project_reports( .get(&issue_row.id) .unwrap_or(&empty_details); + if issue_details.is_empty() { + continue; + } + file_issues.push(FileIssue { id: issue_row.id, report_id: issue_row.report_id, @@ -648,12 +675,23 @@ async fn fetch_project_reports( }); } + if file_issues.is_empty() { + continue; + } + + let severity = file_issues + .iter() + .flat_map(|issue| issue.details.iter()) + .map(|detail| detail.severity) + .max() + .unwrap_or(report_row.severity); + file_reports.push(FileReport { report_id: report_row.report_id, file_id: FileId::from(file_row.file_id), created: report_row.created, flag_reason: FlagReason::Delphi, - severity: report_row.severity, + severity, file_name: file_row.filename.clone(), file_size: file_row.size, download_url: file_row.url.clone(), @@ -784,6 +822,7 @@ pub async fn search_projects( AND (cardinality($6::text[]) = 0 OR m.status = ANY($6::text[])) AND (cardinality($7::text[]) = 0 OR dri.issue_type = ANY($7::text[])) AND didws.status = 'pending' + AND NOT didws.hidden AND ( $5::text IS NULL OR ($5::text = 'unreplied' AND (tm_last.id IS NULL OR u_last.role IS NULL OR u_last.role NOT IN ('moderator', 'admin'))) @@ -793,8 +832,8 @@ pub async fn search_projects( ORDER BY CASE WHEN $3 = 'created_asc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END ASC, CASE WHEN $3 = 'created_desc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END DESC, - CASE WHEN $3 = 'severity_asc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END ASC, - CASE WHEN $3 = 'severity_desc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END DESC, + CASE WHEN $3 = 'severity_asc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END ASC, + CASE WHEN $3 = 'severity_desc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END DESC, -- tie-breaker: oldest reports MIN(dr.created) ASC LIMIT $1 OFFSET $2 @@ -1039,6 +1078,7 @@ pub async fn submit_report( WHERE m.id = $1 AND didws.status = 'pending' + AND NOT didws.hidden -- see delphi.rs todo comment AND dri.issue_type != '__dummy' "#, diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review/global.rs b/apps/labrinth/src/routes/internal/moderation/tech_review/global.rs index 82aa8055b1..09f47d3ea3 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review/global.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review/global.rs @@ -188,6 +188,7 @@ pub async fn search_global_issue_details( FROM delphi_global_detail_verdicts dgdv LEFT JOIN delphi_issue_details_with_statuses didws ON didws.key = dgdv.detail_key + AND NOT didws.hidden LEFT JOIN delphi_report_issues dri ON dri.id = didws.issue_id AND dri.issue_type != '__dummy' @@ -248,6 +249,7 @@ pub async fn search_global_issue_details( AND didv.detail_key = didws.key WHERE didws.key = ANY($1::text[]) + AND NOT didws.hidden AND dri.issue_type != '__dummy' ) SELECT @@ -367,6 +369,7 @@ pub async fn get_global_issue_detail( FROM delphi_global_detail_verdicts dgdv LEFT JOIN delphi_issue_details_with_statuses didws ON didws.key = dgdv.detail_key + AND NOT didws.hidden LEFT JOIN delphi_report_issues dri ON dri.id = didws.issue_id AND dri.issue_type != '__dummy' @@ -412,6 +415,7 @@ pub async fn get_global_issue_detail( WHERE didws.key = $1 AND ($2::bigint IS NULL OR didws.id > $2) + AND NOT didws.hidden AND dri.issue_type != '__dummy' ORDER BY didws.id LIMIT $3 diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs b/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs index e6895558ac..c3f9d79d21 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs @@ -12,10 +12,17 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use utoipa::{PartialSchema, ToSchema}; use super::rules::DelphiRuleEffect; +use crate::routes::internal::delphi::tech_review_sync::{ + self, TechReviewExitReason, +}; use crate::{ auth::check_is_moderator_from_headers, database::{ - PgPool, PgTransaction, models::delphi_report_item::DelphiSeverity, + PgPool, PgTransaction, ReadOnlyPgPool, + models::{ + DBProjectId, DelphiReportIssueDetailsId, + delphi_report_item::DelphiSeverity, + }, redis::RedisPool, }, models::pats::Scopes, @@ -25,9 +32,12 @@ use crate::{ const RULE_SCAN_LOCK_ID: i64 = 0x6465_6c70_6869_7275; const PROGRESS_INTERVAL: usize = 50; +const DUMMY_ISSUE_TYPE: &str = "__dummy"; pub fn config(cfg: &mut actix_web::web::ServiceConfig) { - cfg.service(get_rule_schema).service(scan_rules); + cfg.service(get_rule_schema) + .service(get_detail_rule_input) + .service(scan_rules); } #[derive(Serialize)] @@ -45,40 +55,40 @@ struct RuleScanErrorEvent<'a> { } #[derive(Serialize, utoipa::ToSchema)] -pub(super) struct RuleInput { - pub(super) schema_version: u32, - pub(super) trace: RuleTrace, - pub(super) scan: RuleScan, - pub(super) artifact: RuleArtifact, - pub(super) scope: RuleScope, +pub struct RuleInput { + pub schema_version: u32, + pub trace: RuleTrace, + pub scan: RuleScan, + pub artifact: RuleArtifact, + pub scope: RuleScope, } #[derive(Serialize, utoipa::ToSchema)] -pub(super) struct RuleTrace { - pub(super) key: String, - pub(super) issue_type: String, - pub(super) severity: DelphiSeverity, - pub(super) jar: Option, - pub(super) file_path: String, - pub(super) data: HashMap, +pub struct RuleTrace { + pub key: String, + pub issue_type: String, + pub severity: DelphiSeverity, + pub jar: Option, + pub file_path: String, + pub data: HashMap, } #[derive(Serialize, utoipa::ToSchema)] -pub(super) struct RuleScan { - pub(super) delphi_version: i32, +pub struct RuleScan { + pub delphi_version: i32, } #[derive(Serialize, utoipa::ToSchema)] -pub(super) struct RuleArtifact { - pub(super) size: Option, - pub(super) hashes: BTreeMap, +pub struct RuleArtifact { + pub size: Option, + pub hashes: BTreeMap, } #[derive(Serialize, utoipa::ToSchema)] -pub(super) struct RuleScope { - pub(super) project_id: Option, - pub(super) version_id: Option, - pub(super) file_id: Option, +pub struct RuleScope { + pub project_id: Option, + pub version_id: Option, + pub file_id: Option, } struct CompiledRule { @@ -145,6 +155,101 @@ pub async fn get_rule_schema( })) } +/// Get the exact CEL input for a Delphi issue detail. +#[utoipa::path( + context_path = "/moderation/tech-review", + tag = "moderation", + security(("bearer_auth" = [])), + responses( + (status = OK, body = RuleInput), + (status = NOT_FOUND, description = "Detail not found") + ) +)] +#[get("/rules/details/{detail_id}/input")] +pub async fn get_detail_rule_input( + req: HttpRequest, + pool: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, + path: web::Path<(DelphiReportIssueDetailsId,)>, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PROJECT_READ, + ) + .await?; + + let (detail_id,) = path.into_inner(); + let detail = crate::util::error::Context::wrap_internal_err( + sqlx::query!( + r#" + SELECT + detail.key, + issue.issue_type, + detail.severity AS "severity: DelphiSeverity", + detail.jar, + detail.file_path, + detail.data AS "data: Json>", + report.delphi_version, + file.size AS "size?", + file.id AS "file_id?", + version.id AS "version_id?", + version.mod_id AS "project_id?", + COALESCE(file_hashes.hashes, '{}'::jsonb) + AS "hashes!: Json>" + FROM delphi_report_issue_details detail + INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id + INNER JOIN delphi_reports report ON report.id = issue.report_id + LEFT JOIN files file ON file.id = report.file_id + LEFT JOIN versions version ON version.id = file.version_id + LEFT JOIN LATERAL ( + SELECT + jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes + FROM hashes + WHERE hashes.file_id = file.id + ) file_hashes ON TRUE + WHERE + detail.id = $1 + AND issue.issue_type != $2 + "#, + detail_id as DelphiReportIssueDetailsId, + DUMMY_ISSUE_TYPE, + ) + .fetch_optional(&***ro_pool) + .await, + "failed to fetch delphi rule input", + )? + .ok_or(ApiError::NotFound)?; + + Ok(web::Json(RuleInput { + schema_version: 1, + trace: RuleTrace { + key: detail.key, + issue_type: detail.issue_type, + severity: detail.severity, + jar: detail.jar, + file_path: detail.file_path, + data: detail.data.0, + }, + scan: RuleScan { + delphi_version: detail.delphi_version, + }, + artifact: RuleArtifact { + size: detail.size, + hashes: detail.hashes.0, + }, + scope: RuleScope { + project_id: detail.project_id.map(to_public_id), + version_id: detail.version_id.map(to_public_id), + file_id: detail.file_id.map(to_public_id), + }, + })) +} + fn schema_to_value( schema: T, ) -> Result { @@ -274,31 +379,16 @@ async fn run_scan( .checked_add(1) .ok_or_else(|| eyre!("delphi rule revision overflowed"))?; - let rules = sqlx::query!( - r#" - SELECT id, rule - FROM delphi_rules - WHERE NOT delete_on_next_revision - ORDER BY id - "#, - ) - .fetch_all(&mut transaction) - .await - .wrap_err("failed to fetch delphi rules")? - .into_iter() - .map(|rule| { - let program = cel::Program::compile(&rule.rule).map_err(|error| { - eyre!("failed to compile delphi rule {}: {error}", rule.id) - })?; - Ok(CompiledRule { - id: rule.id, - program, - }) - }) - .collect::>>()?; + let rules = fetch_compiled_rules(&mut transaction).await?; let total = sqlx::query_scalar!( - "SELECT COUNT(*) AS \"count!\" FROM delphi_report_issue_details", + r#" + SELECT COUNT(*) AS "count!" + FROM delphi_report_issue_details detail + INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id + WHERE issue.issue_type != $1 + "#, + DUMMY_ISSUE_TYPE, ) .fetch_one(&mut transaction) .await @@ -333,8 +423,10 @@ async fn run_scan( FROM hashes GROUP BY file_id ) file_hashes ON file_hashes.file_id = file.id + WHERE issue.issue_type != $1 ORDER BY detail.id "#, + DUMMY_ISSUE_TYPE, ) .fetch(&mut transaction); @@ -407,6 +499,237 @@ async fn run_scan( send_progress(sender, "publishing", revision, total, total, effects.len()); + insert_materialized_effects(revision, &effects, &mut transaction).await?; + + let affected_projects = sqlx::query!( + r#" + WITH project_membership AS ( + SELECT + detail.project_id, + BOOL_OR( + detail.status IN ('pending', 'unsafe') + AND issue.issue_type != $2 + AND NOT detail.hidden + ) AS old_needs_review, + BOOL_OR( + detail.status IN ('pending', 'unsafe') + AND issue.issue_type != $2 + AND NOT COALESCE(new_effect.hidden, FALSE) + ) AS new_needs_review + FROM delphi_issue_details_with_statuses detail + INNER JOIN delphi_report_issues issue + ON issue.id = detail.issue_id + LEFT JOIN delphi_rule_effects new_effect + ON new_effect.revision = $1 + AND new_effect.detail_id = detail.id + GROUP BY detail.project_id + ) + SELECT project_id AS "project_id!: DBProjectId" + FROM project_membership + WHERE old_needs_review IS DISTINCT FROM new_needs_review + "#, + revision, + DUMMY_ISSUE_TYPE, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("failed to fetch projects affected by delphi rule changes")?; + + sqlx::query!( + "UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision", + revision, + ) + .execute(&mut transaction) + .await + .wrap_err("failed to update delphi rule revisions")?; + sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision) + .execute(&mut transaction) + .await + .wrap_err("failed to publish the delphi rule revision")?; + + tech_review_sync::sync_project_tech_review_state( + &affected_projects + .iter() + .map(|project| project.project_id) + .collect::>(), + TechReviewExitReason::RulesChanged, + &mut transaction, + ) + .await + .map_err(|error| { + eyre!(error) + .wrap_err("failed to sync projects affected by delphi rule changes") + })?; + + sqlx::query!( + "DELETE FROM delphi_rule_effects WHERE revision <> $1", + revision, + ) + .execute(&mut transaction) + .await + .wrap_err("failed to delete old delphi rule effects")?; + sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision") + .execute(&mut transaction) + .await + .wrap_err("failed to delete retired delphi rules")?; + + transaction + .commit() + .await + .wrap_err("failed to commit the delphi rule scan")?; + + Ok(ScanSummary { + revision, + scanned: total, + total, + effects: effects.len(), + }) +} + +pub(crate) async fn materialize_current_rule_effects( + detail_ids: &[DelphiReportIssueDetailsId], + transaction: &mut PgTransaction<'_>, +) -> Result<()> { + if detail_ids.is_empty() { + return Ok(()); + } + + let revision = sqlx::query_scalar!( + "SELECT revision FROM delphi_rule_revisions LIMIT 1", + ) + .fetch_one(&mut *transaction) + .await + .wrap_err("failed to fetch the current delphi rule revision")?; + let rules = fetch_compiled_rules(transaction).await?; + + if rules.is_empty() { + return Ok(()); + } + + let details = sqlx::query!( + r#" + SELECT + detail.id, + detail.key, + issue.issue_type, + detail.severity AS "severity: DelphiSeverity", + detail.jar, + detail.file_path, + detail.data AS "data: Json>", + report.delphi_version, + file.size AS "size?", + file.id AS "file_id?", + version.id AS "version_id?", + version.mod_id AS "project_id?", + COALESCE(file_hashes.hashes, '{}'::jsonb) + AS "hashes!: Json>" + FROM delphi_report_issue_details detail + INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id + INNER JOIN delphi_reports report ON report.id = issue.report_id + LEFT JOIN files file ON file.id = report.file_id + LEFT JOIN versions version ON version.id = file.version_id + LEFT JOIN LATERAL ( + SELECT + jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes + FROM hashes + WHERE hashes.file_id = file.id + ) file_hashes ON TRUE + WHERE + detail.id = ANY($1::bigint[]) + AND issue.issue_type != $2 + ORDER BY detail.id + "#, + &detail_ids.iter().map(|id| id.0).collect::>(), + DUMMY_ISSUE_TYPE, + ) + .fetch_all(&mut *transaction) + .await + .wrap_err("failed to fetch new delphi issue details")?; + + let mut effects = Vec::new(); + for detail in details { + let input = RuleInput { + schema_version: 1, + trace: RuleTrace { + key: detail.key, + issue_type: detail.issue_type, + severity: detail.severity, + jar: detail.jar, + file_path: detail.file_path, + data: detail.data.0, + }, + scan: RuleScan { + delphi_version: detail.delphi_version, + }, + artifact: RuleArtifact { + size: detail.size, + hashes: detail.hashes.0, + }, + scope: RuleScope { + project_id: detail.project_id.map(to_public_id), + version_id: detail.version_id.map(to_public_id), + file_id: detail.file_id.map(to_public_id), + }, + }; + + for rule in &rules { + let effect = + evaluate_rule(&rule.program, &input).wrap_err_with(|| { + format!( + "failed to evaluate delphi rule {} for detail {}", + rule.id, detail.id + ) + })?; + if let Some(effect) = effect { + effects.push(MaterializedEffect { + detail_id: detail.id, + rule_id: rule.id, + effect, + }); + break; + } + } + } + + insert_materialized_effects(revision, &effects, transaction).await +} + +async fn fetch_compiled_rules( + transaction: &mut PgTransaction<'_>, +) -> Result> { + sqlx::query!( + r#" + SELECT id, rule + FROM delphi_rules + WHERE NOT delete_on_next_revision + ORDER BY id + "#, + ) + .fetch_all(&mut *transaction) + .await + .wrap_err("failed to fetch delphi rules")? + .into_iter() + .map(|rule| { + let program = cel::Program::compile(&rule.rule).map_err(|error| { + eyre!("failed to compile delphi rule {}: {error}", rule.id) + })?; + Ok(CompiledRule { + id: rule.id, + program, + }) + }) + .collect() +} + +async fn insert_materialized_effects( + revision: i64, + effects: &[MaterializedEffect], + transaction: &mut PgTransaction<'_>, +) -> Result<()> { + if effects.is_empty() { + return Ok(()); + } + let detail_ids = effects .iter() .map(|effect| effect.detail_id) @@ -424,69 +747,34 @@ async fn run_scan( .map(|effect| effect.effect.hidden) .collect::>(); - if !effects.is_empty() { - sqlx::query!( - r#" - INSERT INTO delphi_rule_effects ( - revision, - detail_id, - rule_id, - severity, - hidden - ) - SELECT $1, effect.* - FROM UNNEST( - $2::BIGINT[], - $3::BIGINT[], - $4::delphi_severity[], - $5::BOOLEAN[] - ) AS effect(detail_id, rule_id, severity, hidden) - "#, + sqlx::query!( + r#" + INSERT INTO delphi_rule_effects ( revision, - &detail_ids, - &rule_ids, - &severities as &[Option], - &hidden, + detail_id, + rule_id, + severity, + hidden ) - .execute(&mut transaction) - .await - .wrap_err("failed to insert delphi rule effects")?; - } - - sqlx::query!( - "DELETE FROM delphi_rule_effects WHERE revision <> $1", + SELECT $1, effect.* + FROM UNNEST( + $2::BIGINT[], + $3::BIGINT[], + $4::delphi_severity[], + $5::BOOLEAN[] + ) AS effect(detail_id, rule_id, severity, hidden) + "#, revision, + &detail_ids, + &rule_ids, + &severities as &[Option], + &hidden, ) - .execute(&mut transaction) + .execute(&mut *transaction) .await - .wrap_err("failed to delete old delphi rule effects")?; - sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision") - .execute(&mut transaction) - .await - .wrap_err("failed to delete retired delphi rules")?; - sqlx::query!( - "UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision", - revision, - ) - .execute(&mut transaction) - .await - .wrap_err("failed to update delphi rule revisions")?; - sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision) - .execute(&mut transaction) - .await - .wrap_err("failed to publish the delphi rule revision")?; + .wrap_err("failed to insert delphi rule effects")?; - transaction - .commit() - .await - .wrap_err("failed to commit the delphi rule scan")?; - - Ok(ScanSummary { - revision, - scanned: total, - total, - effects: effects.len(), - }) + Ok(()) } pub(super) fn evaluate_rule( diff --git a/packages/api-client/src/modules/labrinth/tech-review/internal.ts b/packages/api-client/src/modules/labrinth/tech-review/internal.ts index 5cf7e278fb..2ad32b78ff 100644 --- a/packages/api-client/src/modules/labrinth/tech-review/internal.ts +++ b/packages/api-client/src/modules/labrinth/tech-review/internal.ts @@ -28,6 +28,19 @@ export class LabrinthTechReviewInternalModule extends AbstractModule { ) } + public async getDetailRuleInput( + detailId: string, + ): Promise { + return this.client.request( + `/moderation/tech-review/rules/details/${detailId}/input`, + { + api: 'labrinth', + version: 'internal', + method: 'GET', + }, + ) + } + public async getRuleAffectedDetails( id: number, ): Promise { diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index 8f9e55939d..3089099050 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -2289,6 +2289,30 @@ export namespace Labrinth { components: Record } + export type RuleInput = { + schema_version: number + trace: { + key: string + issue_type: string + severity: DelphiSeverity + jar: string | null + file_path: string + data: Record + } + scan: { + delphi_version: number + } + artifact: { + size: number | null + hashes: Record + } + scope: { + project_id: string | null + version_id: string | null + file_id: string | null + } + } + export type TestDelphiRuleResponse = { effects: Array }