sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-08-31 03:55:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

rules affect tech review detail statuses

Browse Source
This commit is contained in:
aecsocket
2026-07-24 13:49:20 +01:00
parent 4b07ed720e
commit 8d82c228cf
17 changed files with 760 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/frontend/src/components/ui/moderation/GlobalDetailLocalTraceCard.vue
+5 -17
View File
@@ -5,14 +5,8 @@
<p class="m-0 break-words font-semibold text-contrast">
{{ trace.project_name }}
</p>
<p class="m-0 mt-1 flex flex-wrap items-center gap-1 text-sm text-secondary">
<span class="break-all">{{ trace.version_number }}</span>
<ChevronRightIcon class="size-4 shrink-0" aria-hidden="true" />
<span class="break-all">{{ decodeTracePath(trace.file_name) }}</span>
<template v-if="trace.jar">
<ChevronRightIcon class="size-4 shrink-0" aria-hidden="true" />
<span class="break-all">{{ decodeTracePath(trace.jar) }}</span>
</template>
<p class="m-0 mt-1 text-sm text-secondary">
<IssueDetailPath :segments="[trace.version_number, trace.file_name, trace.jar]" />
</p>
</div>
<div class="flex flex-wrap items-center gap-2">
@@ -33,9 +27,11 @@
<script setup lang="ts">
import type { Labrinth } from '@modrinth/api-client'
import { ChevronRightIcon, ExternalIcon } from '@modrinth/assets'
import { ExternalIcon } from '@modrinth/assets'
import { Badge, ButtonStyled } from '@modrinth/ui'
import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue'
const props = defineProps<{
trace: Labrinth.TechReview.Internal.GlobalIssueDetailTrace
}>()
@@ -46,12 +42,4 @@ const localTraceLink = computed(
props.trace.detail_id,
)}`,
)
function decodeTracePath(path: string): string {
try {
return decodeURIComponent(path)
} catch {
return path
}
}
</script>
apps/frontend/src/components/ui/moderation/GlobalDetailTracesList.vue
+2 -9
View File
@@ -65,7 +65,7 @@
</p>
<p class="m-0 break-all text-secondary">
<span class="font-semibold text-contrast">Path</span>
{{ decodeTracePath(getLatestLocalTrace(trace)?.file_path ?? '') }}
<IssueDetailPath :segments="[getLatestLocalTrace(trace)?.file_path]" />
</p>
</div>
</div>
@@ -133,6 +133,7 @@ import {
} from '@modrinth/ui'
import GlobalDetailLocalTraceCard from '~/components/ui/moderation/GlobalDetailLocalTraceCard.vue'
import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue'
const client = injectModrinthClient()
const { addNotification } = injectNotificationManager()
@@ -165,14 +166,6 @@ function getLatestLocalTrace(trace: Labrinth.TechReview.Internal.GlobalIssueDeta
return trace.local_traces.at(-1)
}
function decodeTracePath(path: string): string {
try {
return decodeURIComponent(path)
} catch {
return path
}
}
function getSeverityBadgeColor(
severity: Labrinth.TechReview.Internal.DelphiSeverity | undefined,
): string {
apps/frontend/src/components/ui/moderation/IssueDetailPath.vue
+74
View File
@@ -0,0 +1,74 @@
<template>
<span class="inline-flex min-w-0 flex-wrap items-center gap-1">
<template v-for="(segment, index) in decodedSegments" :key="`${segment}-${index}`">
<ChevronRightIcon v-if="index > 0" class="size-4 shrink-0" aria-hidden="true" />
<span
v-tooltip="isTruncated(segment) ? segment : undefined"
class="break-all"
:class="{
'font-semibold text-contrast': emphasizeLast && index === decodedSegments.length - 1,
'text-secondary': emphasizeLast && index < decodedSegments.length - 1,
}"
>
{{ formatSegment(segment) }}
</span>
</template>
</span>
</template>
<script setup lang="ts">
import { ChevronRightIcon } from '@modrinth/assets'
const props = withDefaults(
defineProps<{
segments: readonly (string | null | undefined)[]
truncate?: boolean
maxLength?: number
emphasizeLast?: boolean
decode?: boolean
hideBaseMrpack?: boolean
}>(),
{
truncate: false,
maxLength: 120,
emphasizeLast: false,
decode: true,
hideBaseMrpack: false,
},
)
const decodedSegments = computed(() => {
const segments = props.segments
.flatMap((segment) => segment?.split('#') ?? [])
.filter((segment) => segment.length > 0)
.map((segment) => (props.decode ? decodePath(segment) : segment))
if (props.hideBaseMrpack && segments[0]?.toLowerCase().endsWith('.mrpack')) {
return segments.slice(1)
}
return segments
})
function decodePath(path: string): string {
try {
return decodeURIComponent(path)
} catch {
return path
}
}
function isTruncated(segment: string): boolean {
return props.truncate && segment.length > props.maxLength
}
function formatSegment(segment: string): string {
if (!isTruncated(segment)) return segment
const separator = '...'
const charsToShow = props.maxLength - separator.length
const frontChars = Math.ceil(charsToShow / 3)
const backChars = Math.floor((charsToShow * 2) / 3)
return segment.slice(0, frontChars) + separator + segment.slice(-backChars)
}
</script>
apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue
+56 -26
View File
@@ -6,7 +6,6 @@ import {
CheckCheckIcon,
CheckIcon,
ChevronDownIcon,
ChevronRightIcon,
ClipboardCopyIcon,
CodeIcon,
CopyIcon,
@@ -49,6 +48,7 @@ import {
import dayjs from 'dayjs'
import { computed, nextTick, reactive, ref, watch } from 'vue'
import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue'
import type { UnsafeFile } from '~/components/ui/moderation/MaliciousSummaryModal.vue'
import ThreadView from '~/components/ui/thread/ThreadView.vue'
@@ -601,6 +601,29 @@ async function copyToClipboard(code: string, detailId: string) {
}
}
async function copyDetailCelInput(detailId: string) {
if (copyingCelDetails.has(detailId)) return
copyingCelDetails.add(detailId)
try {
const input = await client.labrinth.tech_review_internal.getDetailRuleInput(detailId)
await navigator.clipboard.writeText(JSON.stringify(input, null, 2))
copiedCelDetails.add(detailId)
setTimeout(() => {
copiedCelDetails.delete(detailId)
}, 2000)
} catch (error) {
console.error('Failed to copy CEL input:', error)
addNotification({
type: 'error',
title: 'Failed to copy CEL input',
text: 'An error occurred while loading the trace rule input.',
})
} finally {
copyingCelDetails.delete(detailId)
}
}
function getDetailDecision(
detailId: string,
backendStatus: Labrinth.TechReview.Internal.DelphiReportIssueStatus,
@@ -999,6 +1022,8 @@ async function updateGlobalDetailStatus(
const expandedClasses = reactive<Set<string>>(new Set())
const autoExpandedFileIds = reactive<Set<string>>(new Set())
const showCopyFeedback = reactive<Map<string, boolean>>(new Map())
const copyingCelDetails = reactive<Set<string>>(new Set())
const copiedCelDetails = reactive<Set<string>>(new Set())
const highlightedSourceCache = reactive<Map<string, { source: string; lines: string[] }>>(new Map())
const LAZY_LOAD_CLASS_SOURCE_MINIMUM = 2
@@ -1025,7 +1050,7 @@ interface JarGroup {
function splitJarSegments(jar: string | null, currentFileName: string | null): string[] {
if (!jar) return []
const segments = jar
.split(/[/#]/)
.split('#')
.map((s) => decodeURIComponent(s.trim()))
.filter((s) => s.length > 0)
// Skip the first segment if it matches the current file tab (it's already shown in the file list)
@@ -1770,27 +1795,12 @@ function copyId() {
class="border-b border-solid border-surface-1 px-4 py-3"
>
<div class="flex flex-wrap items-center justify-between gap-3">
<div class="flex flex-wrap items-center gap-1">
<template
v-for="(segment, index) in jarGroup.segments"
:key="`${jarGroup.key}-${index}`"
>
<span
class="font-mono text-sm"
:class="
index === jarGroup.segments.length - 1
? 'font-semibold text-contrast'
: 'text-secondary'
"
>
{{ segment }}
</span>
<ChevronRightIcon
v-if="index < jarGroup.segments.length - 1"
class="size-4 text-secondary"
/>
</template>
</div>
<IssueDetailPath
:segments="jarGroup.segments"
:decode="false"
class="font-mono text-sm"
emphasize-last
/>
<div
v-if="getJarRemainingUnmarkedCount(jarGroup) > 0"
@@ -1864,9 +1874,12 @@ function copyId() {
</button>
</ButtonStyled>
<span v-tooltip="classItem.filePath" class="font-mono font-semibold">{{
truncateMiddle(classItem.filePath)
}}</span>
<IssueDetailPath
:segments="[classItem.jar, classItem.filePath]"
class="font-mono font-semibold"
hide-base-mrpack
truncate
/>
<div
class="rounded-full border-solid px-2.5 py-1"
@@ -1941,6 +1954,23 @@ function copyId() {
</div>
<div class="detail-verdict-action-groups">
<ButtonStyled>
<button
type="button"
:disabled="copyingCelDetails.has(flag.detail.id)"
@click="copyDetailCelInput(flag.detail.id)"
>
<LoaderCircleIcon
v-if="copyingCelDetails.has(flag.detail.id)"
class="animate-spin"
aria-hidden="true"
/>
<ClipboardCopyIcon v-else aria-hidden="true" />
<span aria-live="polite">
{{ copiedCelDetails.has(flag.detail.id) ? 'Copied!' : 'Copy CEL' }}
</span>
</button>
</ButtonStyled>
<div
class="detail-verdict-buttons"
role="group"
apps/frontend/src/pages/moderation/global-traces/[key].vue
+2 -9
View File
@@ -43,7 +43,7 @@
</p>
<p class="m-0 break-all text-secondary">
<span class="font-semibold text-contrast">Path</span>
{{ decodeTracePath(latestLocalTrace.file_path) }}
<IssueDetailPath :segments="[latestLocalTrace.file_path]" />
</p>
</div>
</div>
@@ -103,6 +103,7 @@ import {
} from '@modrinth/ui'
import GlobalDetailLocalTraceCard from '~/components/ui/moderation/GlobalDetailLocalTraceCard.vue'
import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue'
const client = injectModrinthClient()
const { addNotification } = injectNotificationManager()
@@ -137,14 +138,6 @@ const pageEnd = computed(() =>
)
const latestLocalTrace = computed(() => trace.value?.local_traces.at(-1))
function decodeTracePath(path: string): string {
try {
return decodeURIComponent(path)
} catch {
return path
}
}
function getSeverityBadgeColor(severity: Labrinth.TechReview.Internal.DelphiSeverity): string {
switch (severity) {
case 'severe':
apps/frontend/src/pages/moderation/technical-review/rules.vue
+7 -6
View File
@@ -327,11 +327,11 @@
<p
class="m-0 mt-0.5 flex min-w-0 items-center gap-1 font-mono text-xs text-secondary"
>
<template v-if="detail.jar">
<span class="truncate">{{ detail.jar }}</span>
<ChevronRightIcon class="size-3.5 shrink-0" aria-hidden="true" />
</template>
<span class="truncate">{{ detail.file_path }}</span>
<IssueDetailPath
:segments="[detail.jar, detail.file_path]"
hide-base-mrpack
truncate
/>
</p>
</div>
<ButtonStyled>
@@ -385,7 +385,6 @@
import { type Labrinth, SseParser } from '@modrinth/api-client'
import {
ArrowLeftIcon,
ChevronRightIcon,
EditIcon,
EyeOffIcon,
ExternalIcon,
@@ -409,6 +408,8 @@ import { useDebounceFn } from '@vueuse/core'
import type { Ace } from 'ace-builds'
import type { Component } from 'vue'
import IssueDetailPath from '~/components/ui/moderation/IssueDetailPath.vue'
const DEFAULT_RULE = `input.trace.issue_type == "OBFUSCATED_NAMES"
? {"severity": "low", "hidden": false}
: null`
apps/labrinth/migrations/20260723120000_delphi_effective_rule_details.sql
+37
View File
@@ -0,0 +1,37 @@
DROP VIEW delphi_issue_details_with_statuses;
CREATE VIEW delphi_issue_details_with_statuses AS
SELECT
drid.id,
drid.issue_id,
drid.key,
drid.jar,
drid.file_path,
drid.decompiled_source,
drid.data,
drid.severity AS original_severity,
COALESCE(dre.severity, drid.severity) AS severity,
COALESCE(dre.hidden, FALSE) AS hidden,
m.id AS project_id,
didv.verdict AS local_status,
dgdv.verdict AS global_status,
COALESCE(dgdv.verdict, didv.verdict, 'pending') AS status
FROM delphi_report_issue_details drid
INNER JOIN delphi_report_issues dri ON dri.id = drid.issue_id
INNER JOIN delphi_reports dr ON dr.id = dri.report_id
INNER JOIN files f ON f.id = dr.file_id
INNER JOIN versions v ON v.id = f.version_id
INNER JOIN mods m ON m.id = v.mod_id
LEFT JOIN delphi_global_detail_verdicts dgdv
ON drid.key = dgdv.detail_key
LEFT JOIN delphi_issue_detail_verdicts didv
ON m.id = didv.project_id
AND drid.key = didv.detail_key
LEFT JOIN (
SELECT revision
FROM delphi_rule_revisions
LIMIT 1
) drr ON TRUE
LEFT JOIN delphi_rule_effects dre
ON dre.revision = drr.revision
AND dre.detail_id = drid.id;
apps/labrinth/src/routes/internal/delphi/mod.rs
+10 -1
View File
@@ -210,6 +210,7 @@ async fn ingest_report_deserialized(
"Delphi found issues in file",
);
let mut inserted_detail_ids = Vec::new();
for (issue_type, issue_details) in report.issues {
let issue_id = DBDelphiReportIssue {
id: DelphiReportIssueId(0), // This will be set by the database
@@ -229,7 +230,7 @@ async fn ingest_report_deserialized(
let decompiled_source =
report.decompiled_sources.get(&issue_detail.file);
ReportIssueDetail {
let detail_id = ReportIssueDetail {
id: DelphiReportIssueDetailsId(0), // This will be set by the database
issue_id,
key: issue_detail.key.0,
@@ -245,9 +246,17 @@ async fn ingest_report_deserialized(
.insert(&mut transaction)
.await
.wrap_internal_err("failed to insert Delphi issue detail")?;
inserted_detail_ids.push(detail_id);
}
}
crate::routes::internal::moderation::tech_review::rules_scan::materialize_current_rule_effects(
&inserted_detail_ids,
&mut transaction,
)
.await
.wrap_internal_err("failed to apply delphi rules to new issue details")?;
tech_review_sync::sync_project_tech_review_state(
&[DBProjectId::from(report.project_id)],
tech_review_sync::TechReviewExitReason::Resolved,
apps/labrinth/src/routes/internal/delphi/rescan.rs
+2
View File
@@ -105,6 +105,7 @@ async fn fetch_unreviewed_tech_review_project_ids(
WHERE
didws.project_id = m.id
AND didws.status = 'pending'
AND NOT didws.hidden
-- see delphi.rs todo comment
AND dri.issue_type != '__dummy'
)
@@ -115,6 +116,7 @@ async fn fetch_unreviewed_tech_review_project_ids(
WHERE
didws.project_id = m.id
AND didws.status IN ('safe', 'unsafe')
AND NOT didws.hidden
-- see delphi.rs todo comment
AND dri.issue_type != '__dummy'
)
apps/labrinth/src/routes/internal/delphi/tech_review_sync.rs
+80 -31
View File
@@ -60,9 +60,11 @@ const DUMMY_ISSUE_TYPE: &str = "__dummy";
pub enum TechReviewExitReason {
Resolved,
FileDeleted,
RulesChanged,
}
struct ProjectTechReviewState {
project_id: DBProjectId,
has_pending_detail: bool,
has_unsafe_detail: bool,
has_dummy: bool,
@@ -88,36 +90,35 @@ pub async fn sync_project_tech_review_state(
r#"
WITH project_ids AS (
SELECT unnest($1::bigint[]) AS project_id
),
project_detail_state AS (
SELECT
p.project_id,
COALESCE(BOOL_OR(
didws.status = 'pending'
AND NOT didws.hidden
AND dri.issue_type != $3
), FALSE) AS has_pending_detail,
COALESCE(BOOL_OR(
didws.status = 'unsafe'
AND NOT didws.hidden
AND dri.issue_type != $3
), FALSE) AS has_unsafe_detail,
COALESCE(BOOL_OR(
didws.status = 'pending'
AND dri.issue_type = $3
), FALSE) AS has_dummy
FROM project_ids p
LEFT JOIN delphi_issue_details_with_statuses didws
ON didws.project_id = p.project_id
LEFT JOIN delphi_report_issues dri ON dri.id = didws.issue_id
GROUP BY p.project_id
)
SELECT
p.project_id AS "project_id!: DBProjectId",
EXISTS(
SELECT 1
FROM delphi_issue_details_with_statuses didws
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
WHERE
didws.project_id = p.project_id
AND didws.status = 'pending'
AND dri.issue_type != $3
) AS "has_pending_detail!",
EXISTS(
SELECT 1
FROM delphi_issue_details_with_statuses didws
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
WHERE
didws.project_id = p.project_id
AND didws.status = 'unsafe'
AND dri.issue_type != $3
) AS "has_unsafe_detail!",
EXISTS(
SELECT 1
FROM delphi_issue_details_with_statuses didws
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
WHERE
didws.project_id = p.project_id
AND didws.status = 'pending'
AND dri.issue_type = $3
) AS "has_dummy!",
p.has_pending_detail AS "has_pending_detail!",
p.has_unsafe_detail AS "has_unsafe_detail!",
p.has_dummy AS "has_dummy!",
(
SELECT t.id
FROM threads t
@@ -144,7 +145,7 @@ pub async fn sync_project_tech_review_state(
ORDER BY dr.created DESC, dr.id DESC
LIMIT 1
) AS "report_id: DelphiReportId"
FROM project_ids p
FROM project_detail_state p
"#,
&project_ids_raw,
&tech_review_message_types,
@@ -156,6 +157,7 @@ pub async fn sync_project_tech_review_state(
for row in rows {
let state = ProjectTechReviewState {
project_id: row.project_id,
has_pending_detail: row.has_pending_detail,
has_unsafe_detail: row.has_unsafe_detail,
has_dummy: row.has_dummy,
@@ -250,8 +252,23 @@ async fn sync_one_project_tech_review_state(
exit_reason: TechReviewExitReason,
txn: &mut PgTransaction<'_>,
) -> Result<(), ApiError> {
let needs_tech_review =
state.has_pending_detail || state.has_unsafe_detail || state.has_dummy;
let has_review_detail = state.has_pending_detail || state.has_unsafe_detail;
if matches!(exit_reason, TechReviewExitReason::RulesChanged)
&& !has_review_detail
{
remove_dummy_issue_details(state.project_id, txn).await?;
if let Some(thread_id) = state.thread_id
&& should_send_exit(state.last_tech_review_message_type.as_deref())
{
insert_exit_message(thread_id, exit_reason, txn).await?;
}
return Ok(());
}
let needs_tech_review = has_review_detail || state.has_dummy;
if needs_tech_review {
if (state.has_pending_detail || state.has_unsafe_detail)
@@ -325,7 +342,9 @@ async fn insert_exit_message(
txn: &mut PgTransaction<'_>,
) -> Result<(), ApiError> {
let body = match exit_reason {
TechReviewExitReason::Resolved => MessageBody::TechReviewExited,
TechReviewExitReason::Resolved | TechReviewExitReason::RulesChanged => {
MessageBody::TechReviewExited
}
TechReviewExitReason::FileDeleted => {
MessageBody::TechReviewExitFileDeleted
}
@@ -344,6 +363,36 @@ async fn insert_exit_message(
Ok(())
}
async fn remove_dummy_issue_details(
project_id: DBProjectId,
txn: &mut PgTransaction<'_>,
) -> Result<(), ApiError> {
sqlx::query!(
r#"
DELETE FROM delphi_report_issue_details detail
USING
delphi_report_issues issue,
delphi_reports report,
files file,
versions version
WHERE
detail.issue_id = issue.id
AND issue.issue_type = $2
AND issue.report_id = report.id
AND report.file_id = file.id
AND file.version_id = version.id
AND version.mod_id = $1
"#,
project_id as DBProjectId,
DUMMY_ISSUE_TYPE,
)
.execute(&mut *txn)
.await
.wrap_internal_err("failed to remove dummy Delphi issue details")?;
Ok(())
}
async fn ensure_dummy_issue_detail(
report_id: DelphiReportId,
txn: &mut PgTransaction<'_>,
apps/labrinth/src/routes/internal/mod.rs
+1
View File
@@ -122,6 +122,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
moderation::tech_review::rules::update_rule,
moderation::tech_review::rules::delete_rule,
moderation::tech_review::rules_scan::get_rule_schema,
moderation::tech_review::rules_scan::get_detail_rule_input,
moderation::tech_review::rules_scan::scan_rules,
moderation::tech_review::get_project_report,
moderation::tech_review::submit_report,
apps/labrinth/src/routes/internal/moderation/mod.rs
+4
View File
@@ -295,6 +295,7 @@ pub async fn get_projects_internal(
FROM delphi_issue_details_with_statuses didws
WHERE didws.project_id = m.id
AND didws.status = 'pending'
AND NOT didws.hidden
)
)
),
@@ -542,6 +543,7 @@ pub async fn get_projects_internal(
FROM delphi_issue_details_with_statuses didws
WHERE didws.project_id = m.id
AND didws.status = 'pending'
AND NOT didws.hidden
)
)
),
@@ -770,6 +772,7 @@ pub async fn get_project_ids(
FROM delphi_issue_details_with_statuses didws
WHERE didws.project_id = m.id
AND didws.status = 'pending'
AND NOT didws.hidden
)
)
),
@@ -898,6 +901,7 @@ pub async fn get_project_ids(
FROM delphi_issue_details_with_statuses didws
WHERE didws.project_id = mods.id
AND didws.status = 'pending'
AND NOT didws.hidden
)
)
ORDER BY
apps/labrinth/src/routes/internal/moderation/tech_review.rs
+47 -7
View File
@@ -254,7 +254,9 @@ pub async fn get_issue(
)
), '[]'::jsonb)
FROM delphi_issue_details_with_statuses didws
WHERE didws.issue_id = dri.id
WHERE
didws.issue_id = dri.id
AND NOT didws.hidden
)
) AS "data!: sqlx::types::Json<FileIssue>"
FROM delphi_report_issues dri
@@ -309,6 +311,16 @@ pub async fn get_report(
'file_size', f.size,
'flag_reason', 'delphi',
'download_url', f.url,
'severity', COALESCE((
SELECT MAX(didws.severity)
FROM delphi_report_issues severity_issue
INNER JOIN delphi_issue_details_with_statuses didws
ON didws.issue_id = severity_issue.id
WHERE
severity_issue.report_id = dr.id
AND severity_issue.issue_type != '__dummy'
AND NOT didws.hidden
), 'low'::delphi_severity),
-- TODO: replace with `json_array` in Postgres 16
'issues', (
SELECT coalesce(json_agg(
@@ -331,15 +343,24 @@ pub async fn get_report(
)
), '[]'::jsonb)
FROM delphi_issue_details_with_statuses didws
WHERE didws.issue_id = dri.id
WHERE
didws.issue_id = dri.id
AND NOT didws.hidden
)
)
), '[]'::json)
FROM delphi_report_issues dri
WHERE
dri.report_id = dr.id
dri.report_id = dr.id
-- see delphi.rs todo comment
AND dri.issue_type != '__dummy'
AND EXISTS (
SELECT 1
FROM delphi_issue_details_with_statuses visible_detail
WHERE
visible_detail.issue_id = dri.id
AND NOT visible_detail.hidden
)
)
) AS "data!: sqlx::types::Json<FileReport>"
FROM delphi_reports dr
@@ -534,7 +555,9 @@ async fn fetch_project_reports(
didws.global_status AS "global_status?: DelphiStatus",
didws.status AS "status!: DelphiStatus"
FROM delphi_issue_details_with_statuses didws
WHERE didws.issue_id = ANY($1::bigint[])
WHERE
didws.issue_id = ANY($1::bigint[])
AND NOT didws.hidden
ORDER BY didws.issue_id, didws.id
"#,
&issue_ids.iter().map(|i| i.0).collect::<Vec<_>>()
@@ -640,6 +663,10 @@ async fn fetch_project_reports(
.get(&issue_row.id)
.unwrap_or(&empty_details);
if issue_details.is_empty() {
continue;
}
file_issues.push(FileIssue {
id: issue_row.id,
report_id: issue_row.report_id,
@@ -648,12 +675,23 @@ async fn fetch_project_reports(
});
}
if file_issues.is_empty() {
continue;
}
let severity = file_issues
.iter()
.flat_map(|issue| issue.details.iter())
.map(|detail| detail.severity)
.max()
.unwrap_or(report_row.severity);
file_reports.push(FileReport {
report_id: report_row.report_id,
file_id: FileId::from(file_row.file_id),
created: report_row.created,
flag_reason: FlagReason::Delphi,
severity: report_row.severity,
severity,
file_name: file_row.filename.clone(),
file_size: file_row.size,
download_url: file_row.url.clone(),
@@ -784,6 +822,7 @@ pub async fn search_projects(
AND (cardinality($6::text[]) = 0 OR m.status = ANY($6::text[]))
AND (cardinality($7::text[]) = 0 OR dri.issue_type = ANY($7::text[]))
AND didws.status = 'pending'
AND NOT didws.hidden
AND (
$5::text IS NULL
OR ($5::text = 'unreplied' AND (tm_last.id IS NULL OR u_last.role IS NULL OR u_last.role NOT IN ('moderator', 'admin')))
@@ -793,8 +832,8 @@ pub async fn search_projects(
ORDER BY
CASE WHEN $3 = 'created_asc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END ASC,
CASE WHEN $3 = 'created_desc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END DESC,
CASE WHEN $3 = 'severity_asc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END ASC,
CASE WHEN $3 = 'severity_desc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END DESC,
CASE WHEN $3 = 'severity_asc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END ASC,
CASE WHEN $3 = 'severity_desc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END DESC,
-- tie-breaker: oldest reports
MIN(dr.created) ASC
LIMIT $1 OFFSET $2
@@ -1039,6 +1078,7 @@ pub async fn submit_report(
WHERE
m.id = $1
AND didws.status = 'pending'
AND NOT didws.hidden
-- see delphi.rs todo comment
AND dri.issue_type != '__dummy'
"#,
apps/labrinth/src/routes/internal/moderation/tech_review/global.rs
+4
View File
@@ -188,6 +188,7 @@ pub async fn search_global_issue_details(
FROM delphi_global_detail_verdicts dgdv
LEFT JOIN delphi_issue_details_with_statuses didws
ON didws.key = dgdv.detail_key
AND NOT didws.hidden
LEFT JOIN delphi_report_issues dri
ON dri.id = didws.issue_id
AND dri.issue_type != '__dummy'
@@ -248,6 +249,7 @@ pub async fn search_global_issue_details(
AND didv.detail_key = didws.key
WHERE
didws.key = ANY($1::text[])
AND NOT didws.hidden
AND dri.issue_type != '__dummy'
)
SELECT
@@ -367,6 +369,7 @@ pub async fn get_global_issue_detail(
FROM delphi_global_detail_verdicts dgdv
LEFT JOIN delphi_issue_details_with_statuses didws
ON didws.key = dgdv.detail_key
AND NOT didws.hidden
LEFT JOIN delphi_report_issues dri
ON dri.id = didws.issue_id
AND dri.issue_type != '__dummy'
@@ -412,6 +415,7 @@ pub async fn get_global_issue_detail(
WHERE
didws.key = $1
AND ($2::bigint IS NULL OR didws.id > $2)
AND NOT didws.hidden
AND dri.issue_type != '__dummy'
ORDER BY didws.id
LIMIT $3
apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs
+392 -104
View File
@@ -12,10 +12,17 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
use utoipa::{PartialSchema, ToSchema};
use super::rules::DelphiRuleEffect;
use crate::routes::internal::delphi::tech_review_sync::{
self, TechReviewExitReason,
};
use crate::{
auth::check_is_moderator_from_headers,
database::{
PgPool, PgTransaction, models::delphi_report_item::DelphiSeverity,
PgPool, PgTransaction, ReadOnlyPgPool,
models::{
DBProjectId, DelphiReportIssueDetailsId,
delphi_report_item::DelphiSeverity,
},
redis::RedisPool,
},
models::pats::Scopes,
@@ -25,9 +32,12 @@ use crate::{
const RULE_SCAN_LOCK_ID: i64 = 0x6465_6c70_6869_7275;
const PROGRESS_INTERVAL: usize = 50;
const DUMMY_ISSUE_TYPE: &str = "__dummy";
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.service(get_rule_schema).service(scan_rules);
cfg.service(get_rule_schema)
.service(get_detail_rule_input)
.service(scan_rules);
}
#[derive(Serialize)]
@@ -45,40 +55,40 @@ struct RuleScanErrorEvent<'a> {
}
#[derive(Serialize, utoipa::ToSchema)]
pub(super) struct RuleInput {
pub(super) schema_version: u32,
pub(super) trace: RuleTrace,
pub(super) scan: RuleScan,
pub(super) artifact: RuleArtifact,
pub(super) scope: RuleScope,
pub struct RuleInput {
pub schema_version: u32,
pub trace: RuleTrace,
pub scan: RuleScan,
pub artifact: RuleArtifact,
pub scope: RuleScope,
}
#[derive(Serialize, utoipa::ToSchema)]
pub(super) struct RuleTrace {
pub(super) key: String,
pub(super) issue_type: String,
pub(super) severity: DelphiSeverity,
pub(super) jar: Option<String>,
pub(super) file_path: String,
pub(super) data: HashMap<String, serde_json::Value>,
pub struct RuleTrace {
pub key: String,
pub issue_type: String,
pub severity: DelphiSeverity,
pub jar: Option<String>,
pub file_path: String,
pub data: HashMap<String, serde_json::Value>,
}
#[derive(Serialize, utoipa::ToSchema)]
pub(super) struct RuleScan {
pub(super) delphi_version: i32,
pub struct RuleScan {
pub delphi_version: i32,
}
#[derive(Serialize, utoipa::ToSchema)]
pub(super) struct RuleArtifact {
pub(super) size: Option<i32>,
pub(super) hashes: BTreeMap<String, String>,
pub struct RuleArtifact {
pub size: Option<i32>,
pub hashes: BTreeMap<String, String>,
}
#[derive(Serialize, utoipa::ToSchema)]
pub(super) struct RuleScope {
pub(super) project_id: Option<String>,
pub(super) version_id: Option<String>,
pub(super) file_id: Option<String>,
pub struct RuleScope {
pub project_id: Option<String>,
pub version_id: Option<String>,
pub file_id: Option<String>,
}
struct CompiledRule {
@@ -145,6 +155,101 @@ pub async fn get_rule_schema(
}))
}
/// Get the exact CEL input for a Delphi issue detail.
#[utoipa::path(
context_path = "/moderation/tech-review",
tag = "moderation",
security(("bearer_auth" = [])),
responses(
(status = OK, body = RuleInput),
(status = NOT_FOUND, description = "Detail not found")
)
)]
#[get("/rules/details/{detail_id}/input")]
pub async fn get_detail_rule_input(
req: HttpRequest,
pool: web::Data<PgPool>,
ro_pool: web::Data<ReadOnlyPgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
path: web::Path<(DelphiReportIssueDetailsId,)>,
) -> Result<web::Json<RuleInput>, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Scopes::PROJECT_READ,
)
.await?;
let (detail_id,) = path.into_inner();
let detail = crate::util::error::Context::wrap_internal_err(
sqlx::query!(
r#"
SELECT
detail.key,
issue.issue_type,
detail.severity AS "severity: DelphiSeverity",
detail.jar,
detail.file_path,
detail.data AS "data: Json<HashMap<String, serde_json::Value>>",
report.delphi_version,
file.size AS "size?",
file.id AS "file_id?",
version.id AS "version_id?",
version.mod_id AS "project_id?",
COALESCE(file_hashes.hashes, '{}'::jsonb)
AS "hashes!: Json<BTreeMap<String, String>>"
FROM delphi_report_issue_details detail
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
INNER JOIN delphi_reports report ON report.id = issue.report_id
LEFT JOIN files file ON file.id = report.file_id
LEFT JOIN versions version ON version.id = file.version_id
LEFT JOIN LATERAL (
SELECT
jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes
FROM hashes
WHERE hashes.file_id = file.id
) file_hashes ON TRUE
WHERE
detail.id = $1
AND issue.issue_type != $2
"#,
detail_id as DelphiReportIssueDetailsId,
DUMMY_ISSUE_TYPE,
)
.fetch_optional(&***ro_pool)
.await,
"failed to fetch delphi rule input",
)?
.ok_or(ApiError::NotFound)?;
Ok(web::Json(RuleInput {
schema_version: 1,
trace: RuleTrace {
key: detail.key,
issue_type: detail.issue_type,
severity: detail.severity,
jar: detail.jar,
file_path: detail.file_path,
data: detail.data.0,
},
scan: RuleScan {
delphi_version: detail.delphi_version,
},
artifact: RuleArtifact {
size: detail.size,
hashes: detail.hashes.0,
},
scope: RuleScope {
project_id: detail.project_id.map(to_public_id),
version_id: detail.version_id.map(to_public_id),
file_id: detail.file_id.map(to_public_id),
},
}))
}
fn schema_to_value<T: Serialize>(
schema: T,
) -> Result<serde_json::Value, ApiError> {
@@ -274,31 +379,16 @@ async fn run_scan(
.checked_add(1)
.ok_or_else(|| eyre!("delphi rule revision overflowed"))?;
let rules = sqlx::query!(
r#"
SELECT id, rule
FROM delphi_rules
WHERE NOT delete_on_next_revision
ORDER BY id
"#,
)
.fetch_all(&mut transaction)
.await
.wrap_err("failed to fetch delphi rules")?
.into_iter()
.map(|rule| {
let program = cel::Program::compile(&rule.rule).map_err(|error| {
eyre!("failed to compile delphi rule {}: {error}", rule.id)
})?;
Ok(CompiledRule {
id: rule.id,
program,
})
})
.collect::<Result<Vec<_>>>()?;
let rules = fetch_compiled_rules(&mut transaction).await?;
let total = sqlx::query_scalar!(
"SELECT COUNT(*) AS \"count!\" FROM delphi_report_issue_details",
r#"
SELECT COUNT(*) AS "count!"
FROM delphi_report_issue_details detail
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
WHERE issue.issue_type != $1
"#,
DUMMY_ISSUE_TYPE,
)
.fetch_one(&mut transaction)
.await
@@ -333,8 +423,10 @@ async fn run_scan(
FROM hashes
GROUP BY file_id
) file_hashes ON file_hashes.file_id = file.id
WHERE issue.issue_type != $1
ORDER BY detail.id
"#,
DUMMY_ISSUE_TYPE,
)
.fetch(&mut transaction);
@@ -407,6 +499,237 @@ async fn run_scan(
send_progress(sender, "publishing", revision, total, total, effects.len());
insert_materialized_effects(revision, &effects, &mut transaction).await?;
let affected_projects = sqlx::query!(
r#"
WITH project_membership AS (
SELECT
detail.project_id,
BOOL_OR(
detail.status IN ('pending', 'unsafe')
AND issue.issue_type != $2
AND NOT detail.hidden
) AS old_needs_review,
BOOL_OR(
detail.status IN ('pending', 'unsafe')
AND issue.issue_type != $2
AND NOT COALESCE(new_effect.hidden, FALSE)
) AS new_needs_review
FROM delphi_issue_details_with_statuses detail
INNER JOIN delphi_report_issues issue
ON issue.id = detail.issue_id
LEFT JOIN delphi_rule_effects new_effect
ON new_effect.revision = $1
AND new_effect.detail_id = detail.id
GROUP BY detail.project_id
)
SELECT project_id AS "project_id!: DBProjectId"
FROM project_membership
WHERE old_needs_review IS DISTINCT FROM new_needs_review
"#,
revision,
DUMMY_ISSUE_TYPE,
)
.fetch_all(&mut transaction)
.await
.wrap_err("failed to fetch projects affected by delphi rule changes")?;
sqlx::query!(
"UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision",
revision,
)
.execute(&mut transaction)
.await
.wrap_err("failed to update delphi rule revisions")?;
sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision)
.execute(&mut transaction)
.await
.wrap_err("failed to publish the delphi rule revision")?;
tech_review_sync::sync_project_tech_review_state(
&affected_projects
.iter()
.map(|project| project.project_id)
.collect::<Vec<_>>(),
TechReviewExitReason::RulesChanged,
&mut transaction,
)
.await
.map_err(|error| {
eyre!(error)
.wrap_err("failed to sync projects affected by delphi rule changes")
})?;
sqlx::query!(
"DELETE FROM delphi_rule_effects WHERE revision <> $1",
revision,
)
.execute(&mut transaction)
.await
.wrap_err("failed to delete old delphi rule effects")?;
sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision")
.execute(&mut transaction)
.await
.wrap_err("failed to delete retired delphi rules")?;
transaction
.commit()
.await
.wrap_err("failed to commit the delphi rule scan")?;
Ok(ScanSummary {
revision,
scanned: total,
total,
effects: effects.len(),
})
}
pub(crate) async fn materialize_current_rule_effects(
detail_ids: &[DelphiReportIssueDetailsId],
transaction: &mut PgTransaction<'_>,
) -> Result<()> {
if detail_ids.is_empty() {
return Ok(());
}
let revision = sqlx::query_scalar!(
"SELECT revision FROM delphi_rule_revisions LIMIT 1",
)
.fetch_one(&mut *transaction)
.await
.wrap_err("failed to fetch the current delphi rule revision")?;
let rules = fetch_compiled_rules(transaction).await?;
if rules.is_empty() {
return Ok(());
}
let details = sqlx::query!(
r#"
SELECT
detail.id,
detail.key,
issue.issue_type,
detail.severity AS "severity: DelphiSeverity",
detail.jar,
detail.file_path,
detail.data AS "data: Json<HashMap<String, serde_json::Value>>",
report.delphi_version,
file.size AS "size?",
file.id AS "file_id?",
version.id AS "version_id?",
version.mod_id AS "project_id?",
COALESCE(file_hashes.hashes, '{}'::jsonb)
AS "hashes!: Json<BTreeMap<String, String>>"
FROM delphi_report_issue_details detail
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
INNER JOIN delphi_reports report ON report.id = issue.report_id
LEFT JOIN files file ON file.id = report.file_id
LEFT JOIN versions version ON version.id = file.version_id
LEFT JOIN LATERAL (
SELECT
jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes
FROM hashes
WHERE hashes.file_id = file.id
) file_hashes ON TRUE
WHERE
detail.id = ANY($1::bigint[])
AND issue.issue_type != $2
ORDER BY detail.id
"#,
&detail_ids.iter().map(|id| id.0).collect::<Vec<_>>(),
DUMMY_ISSUE_TYPE,
)
.fetch_all(&mut *transaction)
.await
.wrap_err("failed to fetch new delphi issue details")?;
let mut effects = Vec::new();
for detail in details {
let input = RuleInput {
schema_version: 1,
trace: RuleTrace {
key: detail.key,
issue_type: detail.issue_type,
severity: detail.severity,
jar: detail.jar,
file_path: detail.file_path,
data: detail.data.0,
},
scan: RuleScan {
delphi_version: detail.delphi_version,
},
artifact: RuleArtifact {
size: detail.size,
hashes: detail.hashes.0,
},
scope: RuleScope {
project_id: detail.project_id.map(to_public_id),
version_id: detail.version_id.map(to_public_id),
file_id: detail.file_id.map(to_public_id),
},
};
for rule in &rules {
let effect =
evaluate_rule(&rule.program, &input).wrap_err_with(|| {
format!(
"failed to evaluate delphi rule {} for detail {}",
rule.id, detail.id
)
})?;
if let Some(effect) = effect {
effects.push(MaterializedEffect {
detail_id: detail.id,
rule_id: rule.id,
effect,
});
break;
}
}
}
insert_materialized_effects(revision, &effects, transaction).await
}
async fn fetch_compiled_rules(
transaction: &mut PgTransaction<'_>,
) -> Result<Vec<CompiledRule>> {
sqlx::query!(
r#"
SELECT id, rule
FROM delphi_rules
WHERE NOT delete_on_next_revision
ORDER BY id
"#,
)
.fetch_all(&mut *transaction)
.await
.wrap_err("failed to fetch delphi rules")?
.into_iter()
.map(|rule| {
let program = cel::Program::compile(&rule.rule).map_err(|error| {
eyre!("failed to compile delphi rule {}: {error}", rule.id)
})?;
Ok(CompiledRule {
id: rule.id,
program,
})
})
.collect()
}
async fn insert_materialized_effects(
revision: i64,
effects: &[MaterializedEffect],
transaction: &mut PgTransaction<'_>,
) -> Result<()> {
if effects.is_empty() {
return Ok(());
}
let detail_ids = effects
.iter()
.map(|effect| effect.detail_id)
@@ -424,69 +747,34 @@ async fn run_scan(
.map(|effect| effect.effect.hidden)
.collect::<Vec<_>>();
if !effects.is_empty() {
sqlx::query!(
r#"
INSERT INTO delphi_rule_effects (
revision,
detail_id,
rule_id,
severity,
hidden
)
SELECT $1, effect.*
FROM UNNEST(
$2::BIGINT[],
$3::BIGINT[],
$4::delphi_severity[],
$5::BOOLEAN[]
) AS effect(detail_id, rule_id, severity, hidden)
"#,
sqlx::query!(
r#"
INSERT INTO delphi_rule_effects (
revision,
&detail_ids,
&rule_ids,
&severities as &[Option<DelphiSeverity>],
&hidden,
detail_id,
rule_id,
severity,
hidden
)
.execute(&mut transaction)
.await
.wrap_err("failed to insert delphi rule effects")?;
}
sqlx::query!(
"DELETE FROM delphi_rule_effects WHERE revision <> $1",
SELECT $1, effect.*
FROM UNNEST(
$2::BIGINT[],
$3::BIGINT[],
$4::delphi_severity[],
$5::BOOLEAN[]
) AS effect(detail_id, rule_id, severity, hidden)
"#,
revision,
&detail_ids,
&rule_ids,
&severities as &[Option<DelphiSeverity>],
&hidden,
)
.execute(&mut transaction)
.execute(&mut *transaction)
.await
.wrap_err("failed to delete old delphi rule effects")?;
sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision")
.execute(&mut transaction)
.await
.wrap_err("failed to delete retired delphi rules")?;
sqlx::query!(
"UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision",
revision,
)
.execute(&mut transaction)
.await
.wrap_err("failed to update delphi rule revisions")?;
sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision)
.execute(&mut transaction)
.await
.wrap_err("failed to publish the delphi rule revision")?;
.wrap_err("failed to insert delphi rule effects")?;
transaction
.commit()
.await
.wrap_err("failed to commit the delphi rule scan")?;
Ok(ScanSummary {
revision,
scanned: total,
total,
effects: effects.len(),
})
Ok(())
}
pub(super) fn evaluate_rule(