mirror of
https://github.com/modrinth/code.git
synced 2026-09-05 06:19:11 +00:00
rules affect tech review detail statuses
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
DROP VIEW delphi_issue_details_with_statuses;
|
||||
|
||||
CREATE VIEW delphi_issue_details_with_statuses AS
|
||||
SELECT
|
||||
drid.id,
|
||||
drid.issue_id,
|
||||
drid.key,
|
||||
drid.jar,
|
||||
drid.file_path,
|
||||
drid.decompiled_source,
|
||||
drid.data,
|
||||
drid.severity AS original_severity,
|
||||
COALESCE(dre.severity, drid.severity) AS severity,
|
||||
COALESCE(dre.hidden, FALSE) AS hidden,
|
||||
m.id AS project_id,
|
||||
didv.verdict AS local_status,
|
||||
dgdv.verdict AS global_status,
|
||||
COALESCE(dgdv.verdict, didv.verdict, 'pending') AS status
|
||||
FROM delphi_report_issue_details drid
|
||||
INNER JOIN delphi_report_issues dri ON dri.id = drid.issue_id
|
||||
INNER JOIN delphi_reports dr ON dr.id = dri.report_id
|
||||
INNER JOIN files f ON f.id = dr.file_id
|
||||
INNER JOIN versions v ON v.id = f.version_id
|
||||
INNER JOIN mods m ON m.id = v.mod_id
|
||||
LEFT JOIN delphi_global_detail_verdicts dgdv
|
||||
ON drid.key = dgdv.detail_key
|
||||
LEFT JOIN delphi_issue_detail_verdicts didv
|
||||
ON m.id = didv.project_id
|
||||
AND drid.key = didv.detail_key
|
||||
LEFT JOIN (
|
||||
SELECT revision
|
||||
FROM delphi_rule_revisions
|
||||
LIMIT 1
|
||||
) drr ON TRUE
|
||||
LEFT JOIN delphi_rule_effects dre
|
||||
ON dre.revision = drr.revision
|
||||
AND dre.detail_id = drid.id;
|
||||
@@ -210,6 +210,7 @@ async fn ingest_report_deserialized(
|
||||
"Delphi found issues in file",
|
||||
);
|
||||
|
||||
let mut inserted_detail_ids = Vec::new();
|
||||
for (issue_type, issue_details) in report.issues {
|
||||
let issue_id = DBDelphiReportIssue {
|
||||
id: DelphiReportIssueId(0), // This will be set by the database
|
||||
@@ -229,7 +230,7 @@ async fn ingest_report_deserialized(
|
||||
let decompiled_source =
|
||||
report.decompiled_sources.get(&issue_detail.file);
|
||||
|
||||
ReportIssueDetail {
|
||||
let detail_id = ReportIssueDetail {
|
||||
id: DelphiReportIssueDetailsId(0), // This will be set by the database
|
||||
issue_id,
|
||||
key: issue_detail.key.0,
|
||||
@@ -245,9 +246,17 @@ async fn ingest_report_deserialized(
|
||||
.insert(&mut transaction)
|
||||
.await
|
||||
.wrap_internal_err("failed to insert Delphi issue detail")?;
|
||||
inserted_detail_ids.push(detail_id);
|
||||
}
|
||||
}
|
||||
|
||||
crate::routes::internal::moderation::tech_review::rules_scan::materialize_current_rule_effects(
|
||||
&inserted_detail_ids,
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.wrap_internal_err("failed to apply delphi rules to new issue details")?;
|
||||
|
||||
tech_review_sync::sync_project_tech_review_state(
|
||||
&[DBProjectId::from(report.project_id)],
|
||||
tech_review_sync::TechReviewExitReason::Resolved,
|
||||
|
||||
@@ -105,6 +105,7 @@ async fn fetch_unreviewed_tech_review_project_ids(
|
||||
WHERE
|
||||
didws.project_id = m.id
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
-- see delphi.rs todo comment
|
||||
AND dri.issue_type != '__dummy'
|
||||
)
|
||||
@@ -115,6 +116,7 @@ async fn fetch_unreviewed_tech_review_project_ids(
|
||||
WHERE
|
||||
didws.project_id = m.id
|
||||
AND didws.status IN ('safe', 'unsafe')
|
||||
AND NOT didws.hidden
|
||||
-- see delphi.rs todo comment
|
||||
AND dri.issue_type != '__dummy'
|
||||
)
|
||||
|
||||
@@ -60,9 +60,11 @@ const DUMMY_ISSUE_TYPE: &str = "__dummy";
|
||||
pub enum TechReviewExitReason {
|
||||
Resolved,
|
||||
FileDeleted,
|
||||
RulesChanged,
|
||||
}
|
||||
|
||||
struct ProjectTechReviewState {
|
||||
project_id: DBProjectId,
|
||||
has_pending_detail: bool,
|
||||
has_unsafe_detail: bool,
|
||||
has_dummy: bool,
|
||||
@@ -88,36 +90,35 @@ pub async fn sync_project_tech_review_state(
|
||||
r#"
|
||||
WITH project_ids AS (
|
||||
SELECT unnest($1::bigint[]) AS project_id
|
||||
),
|
||||
project_detail_state AS (
|
||||
SELECT
|
||||
p.project_id,
|
||||
COALESCE(BOOL_OR(
|
||||
didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
AND dri.issue_type != $3
|
||||
), FALSE) AS has_pending_detail,
|
||||
COALESCE(BOOL_OR(
|
||||
didws.status = 'unsafe'
|
||||
AND NOT didws.hidden
|
||||
AND dri.issue_type != $3
|
||||
), FALSE) AS has_unsafe_detail,
|
||||
COALESCE(BOOL_OR(
|
||||
didws.status = 'pending'
|
||||
AND dri.issue_type = $3
|
||||
), FALSE) AS has_dummy
|
||||
FROM project_ids p
|
||||
LEFT JOIN delphi_issue_details_with_statuses didws
|
||||
ON didws.project_id = p.project_id
|
||||
LEFT JOIN delphi_report_issues dri ON dri.id = didws.issue_id
|
||||
GROUP BY p.project_id
|
||||
)
|
||||
SELECT
|
||||
p.project_id AS "project_id!: DBProjectId",
|
||||
EXISTS(
|
||||
SELECT 1
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
|
||||
WHERE
|
||||
didws.project_id = p.project_id
|
||||
AND didws.status = 'pending'
|
||||
AND dri.issue_type != $3
|
||||
) AS "has_pending_detail!",
|
||||
EXISTS(
|
||||
SELECT 1
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
|
||||
WHERE
|
||||
didws.project_id = p.project_id
|
||||
AND didws.status = 'unsafe'
|
||||
AND dri.issue_type != $3
|
||||
) AS "has_unsafe_detail!",
|
||||
EXISTS(
|
||||
SELECT 1
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
|
||||
WHERE
|
||||
didws.project_id = p.project_id
|
||||
AND didws.status = 'pending'
|
||||
AND dri.issue_type = $3
|
||||
) AS "has_dummy!",
|
||||
p.has_pending_detail AS "has_pending_detail!",
|
||||
p.has_unsafe_detail AS "has_unsafe_detail!",
|
||||
p.has_dummy AS "has_dummy!",
|
||||
(
|
||||
SELECT t.id
|
||||
FROM threads t
|
||||
@@ -144,7 +145,7 @@ pub async fn sync_project_tech_review_state(
|
||||
ORDER BY dr.created DESC, dr.id DESC
|
||||
LIMIT 1
|
||||
) AS "report_id: DelphiReportId"
|
||||
FROM project_ids p
|
||||
FROM project_detail_state p
|
||||
"#,
|
||||
&project_ids_raw,
|
||||
&tech_review_message_types,
|
||||
@@ -156,6 +157,7 @@ pub async fn sync_project_tech_review_state(
|
||||
|
||||
for row in rows {
|
||||
let state = ProjectTechReviewState {
|
||||
project_id: row.project_id,
|
||||
has_pending_detail: row.has_pending_detail,
|
||||
has_unsafe_detail: row.has_unsafe_detail,
|
||||
has_dummy: row.has_dummy,
|
||||
@@ -250,8 +252,23 @@ async fn sync_one_project_tech_review_state(
|
||||
exit_reason: TechReviewExitReason,
|
||||
txn: &mut PgTransaction<'_>,
|
||||
) -> Result<(), ApiError> {
|
||||
let needs_tech_review =
|
||||
state.has_pending_detail || state.has_unsafe_detail || state.has_dummy;
|
||||
let has_review_detail = state.has_pending_detail || state.has_unsafe_detail;
|
||||
|
||||
if matches!(exit_reason, TechReviewExitReason::RulesChanged)
|
||||
&& !has_review_detail
|
||||
{
|
||||
remove_dummy_issue_details(state.project_id, txn).await?;
|
||||
|
||||
if let Some(thread_id) = state.thread_id
|
||||
&& should_send_exit(state.last_tech_review_message_type.as_deref())
|
||||
{
|
||||
insert_exit_message(thread_id, exit_reason, txn).await?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let needs_tech_review = has_review_detail || state.has_dummy;
|
||||
|
||||
if needs_tech_review {
|
||||
if (state.has_pending_detail || state.has_unsafe_detail)
|
||||
@@ -325,7 +342,9 @@ async fn insert_exit_message(
|
||||
txn: &mut PgTransaction<'_>,
|
||||
) -> Result<(), ApiError> {
|
||||
let body = match exit_reason {
|
||||
TechReviewExitReason::Resolved => MessageBody::TechReviewExited,
|
||||
TechReviewExitReason::Resolved | TechReviewExitReason::RulesChanged => {
|
||||
MessageBody::TechReviewExited
|
||||
}
|
||||
TechReviewExitReason::FileDeleted => {
|
||||
MessageBody::TechReviewExitFileDeleted
|
||||
}
|
||||
@@ -344,6 +363,36 @@ async fn insert_exit_message(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn remove_dummy_issue_details(
|
||||
project_id: DBProjectId,
|
||||
txn: &mut PgTransaction<'_>,
|
||||
) -> Result<(), ApiError> {
|
||||
sqlx::query!(
|
||||
r#"
|
||||
DELETE FROM delphi_report_issue_details detail
|
||||
USING
|
||||
delphi_report_issues issue,
|
||||
delphi_reports report,
|
||||
files file,
|
||||
versions version
|
||||
WHERE
|
||||
detail.issue_id = issue.id
|
||||
AND issue.issue_type = $2
|
||||
AND issue.report_id = report.id
|
||||
AND report.file_id = file.id
|
||||
AND file.version_id = version.id
|
||||
AND version.mod_id = $1
|
||||
"#,
|
||||
project_id as DBProjectId,
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.execute(&mut *txn)
|
||||
.await
|
||||
.wrap_internal_err("failed to remove dummy Delphi issue details")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_dummy_issue_detail(
|
||||
report_id: DelphiReportId,
|
||||
txn: &mut PgTransaction<'_>,
|
||||
|
||||
@@ -122,6 +122,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
moderation::tech_review::rules::update_rule,
|
||||
moderation::tech_review::rules::delete_rule,
|
||||
moderation::tech_review::rules_scan::get_rule_schema,
|
||||
moderation::tech_review::rules_scan::get_detail_rule_input,
|
||||
moderation::tech_review::rules_scan::scan_rules,
|
||||
moderation::tech_review::get_project_report,
|
||||
moderation::tech_review::submit_report,
|
||||
|
||||
@@ -295,6 +295,7 @@ pub async fn get_projects_internal(
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.project_id = m.id
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
)
|
||||
),
|
||||
@@ -542,6 +543,7 @@ pub async fn get_projects_internal(
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.project_id = m.id
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
)
|
||||
),
|
||||
@@ -770,6 +772,7 @@ pub async fn get_project_ids(
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.project_id = m.id
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
)
|
||||
),
|
||||
@@ -898,6 +901,7 @@ pub async fn get_project_ids(
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.project_id = mods.id
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
)
|
||||
ORDER BY
|
||||
|
||||
@@ -254,7 +254,9 @@ pub async fn get_issue(
|
||||
)
|
||||
), '[]'::jsonb)
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.issue_id = dri.id
|
||||
WHERE
|
||||
didws.issue_id = dri.id
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
) AS "data!: sqlx::types::Json<FileIssue>"
|
||||
FROM delphi_report_issues dri
|
||||
@@ -309,6 +311,16 @@ pub async fn get_report(
|
||||
'file_size', f.size,
|
||||
'flag_reason', 'delphi',
|
||||
'download_url', f.url,
|
||||
'severity', COALESCE((
|
||||
SELECT MAX(didws.severity)
|
||||
FROM delphi_report_issues severity_issue
|
||||
INNER JOIN delphi_issue_details_with_statuses didws
|
||||
ON didws.issue_id = severity_issue.id
|
||||
WHERE
|
||||
severity_issue.report_id = dr.id
|
||||
AND severity_issue.issue_type != '__dummy'
|
||||
AND NOT didws.hidden
|
||||
), 'low'::delphi_severity),
|
||||
-- TODO: replace with `json_array` in Postgres 16
|
||||
'issues', (
|
||||
SELECT coalesce(json_agg(
|
||||
@@ -331,15 +343,24 @@ pub async fn get_report(
|
||||
)
|
||||
), '[]'::jsonb)
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.issue_id = dri.id
|
||||
WHERE
|
||||
didws.issue_id = dri.id
|
||||
AND NOT didws.hidden
|
||||
)
|
||||
)
|
||||
), '[]'::json)
|
||||
FROM delphi_report_issues dri
|
||||
WHERE
|
||||
dri.report_id = dr.id
|
||||
dri.report_id = dr.id
|
||||
-- see delphi.rs todo comment
|
||||
AND dri.issue_type != '__dummy'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM delphi_issue_details_with_statuses visible_detail
|
||||
WHERE
|
||||
visible_detail.issue_id = dri.id
|
||||
AND NOT visible_detail.hidden
|
||||
)
|
||||
)
|
||||
) AS "data!: sqlx::types::Json<FileReport>"
|
||||
FROM delphi_reports dr
|
||||
@@ -534,7 +555,9 @@ async fn fetch_project_reports(
|
||||
didws.global_status AS "global_status?: DelphiStatus",
|
||||
didws.status AS "status!: DelphiStatus"
|
||||
FROM delphi_issue_details_with_statuses didws
|
||||
WHERE didws.issue_id = ANY($1::bigint[])
|
||||
WHERE
|
||||
didws.issue_id = ANY($1::bigint[])
|
||||
AND NOT didws.hidden
|
||||
ORDER BY didws.issue_id, didws.id
|
||||
"#,
|
||||
&issue_ids.iter().map(|i| i.0).collect::<Vec<_>>()
|
||||
@@ -640,6 +663,10 @@ async fn fetch_project_reports(
|
||||
.get(&issue_row.id)
|
||||
.unwrap_or(&empty_details);
|
||||
|
||||
if issue_details.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
file_issues.push(FileIssue {
|
||||
id: issue_row.id,
|
||||
report_id: issue_row.report_id,
|
||||
@@ -648,12 +675,23 @@ async fn fetch_project_reports(
|
||||
});
|
||||
}
|
||||
|
||||
if file_issues.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let severity = file_issues
|
||||
.iter()
|
||||
.flat_map(|issue| issue.details.iter())
|
||||
.map(|detail| detail.severity)
|
||||
.max()
|
||||
.unwrap_or(report_row.severity);
|
||||
|
||||
file_reports.push(FileReport {
|
||||
report_id: report_row.report_id,
|
||||
file_id: FileId::from(file_row.file_id),
|
||||
created: report_row.created,
|
||||
flag_reason: FlagReason::Delphi,
|
||||
severity: report_row.severity,
|
||||
severity,
|
||||
file_name: file_row.filename.clone(),
|
||||
file_size: file_row.size,
|
||||
download_url: file_row.url.clone(),
|
||||
@@ -784,6 +822,7 @@ pub async fn search_projects(
|
||||
AND (cardinality($6::text[]) = 0 OR m.status = ANY($6::text[]))
|
||||
AND (cardinality($7::text[]) = 0 OR dri.issue_type = ANY($7::text[]))
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
AND (
|
||||
$5::text IS NULL
|
||||
OR ($5::text = 'unreplied' AND (tm_last.id IS NULL OR u_last.role IS NULL OR u_last.role NOT IN ('moderator', 'admin')))
|
||||
@@ -793,8 +832,8 @@ pub async fn search_projects(
|
||||
ORDER BY
|
||||
CASE WHEN $3 = 'created_asc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END ASC,
|
||||
CASE WHEN $3 = 'created_desc' THEN MIN(dr.created) ELSE TO_TIMESTAMP(0) END DESC,
|
||||
CASE WHEN $3 = 'severity_asc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END ASC,
|
||||
CASE WHEN $3 = 'severity_desc' THEN MAX(dr.severity) ELSE 'low'::delphi_severity END DESC,
|
||||
CASE WHEN $3 = 'severity_asc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END ASC,
|
||||
CASE WHEN $3 = 'severity_desc' THEN MAX(didws.severity) ELSE 'low'::delphi_severity END DESC,
|
||||
-- tie-breaker: oldest reports
|
||||
MIN(dr.created) ASC
|
||||
LIMIT $1 OFFSET $2
|
||||
@@ -1039,6 +1078,7 @@ pub async fn submit_report(
|
||||
WHERE
|
||||
m.id = $1
|
||||
AND didws.status = 'pending'
|
||||
AND NOT didws.hidden
|
||||
-- see delphi.rs todo comment
|
||||
AND dri.issue_type != '__dummy'
|
||||
"#,
|
||||
|
||||
@@ -188,6 +188,7 @@ pub async fn search_global_issue_details(
|
||||
FROM delphi_global_detail_verdicts dgdv
|
||||
LEFT JOIN delphi_issue_details_with_statuses didws
|
||||
ON didws.key = dgdv.detail_key
|
||||
AND NOT didws.hidden
|
||||
LEFT JOIN delphi_report_issues dri
|
||||
ON dri.id = didws.issue_id
|
||||
AND dri.issue_type != '__dummy'
|
||||
@@ -248,6 +249,7 @@ pub async fn search_global_issue_details(
|
||||
AND didv.detail_key = didws.key
|
||||
WHERE
|
||||
didws.key = ANY($1::text[])
|
||||
AND NOT didws.hidden
|
||||
AND dri.issue_type != '__dummy'
|
||||
)
|
||||
SELECT
|
||||
@@ -367,6 +369,7 @@ pub async fn get_global_issue_detail(
|
||||
FROM delphi_global_detail_verdicts dgdv
|
||||
LEFT JOIN delphi_issue_details_with_statuses didws
|
||||
ON didws.key = dgdv.detail_key
|
||||
AND NOT didws.hidden
|
||||
LEFT JOIN delphi_report_issues dri
|
||||
ON dri.id = didws.issue_id
|
||||
AND dri.issue_type != '__dummy'
|
||||
@@ -412,6 +415,7 @@ pub async fn get_global_issue_detail(
|
||||
WHERE
|
||||
didws.key = $1
|
||||
AND ($2::bigint IS NULL OR didws.id > $2)
|
||||
AND NOT didws.hidden
|
||||
AND dri.issue_type != '__dummy'
|
||||
ORDER BY didws.id
|
||||
LIMIT $3
|
||||
|
||||
@@ -12,10 +12,17 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use utoipa::{PartialSchema, ToSchema};
|
||||
|
||||
use super::rules::DelphiRuleEffect;
|
||||
use crate::routes::internal::delphi::tech_review_sync::{
|
||||
self, TechReviewExitReason,
|
||||
};
|
||||
use crate::{
|
||||
auth::check_is_moderator_from_headers,
|
||||
database::{
|
||||
PgPool, PgTransaction, models::delphi_report_item::DelphiSeverity,
|
||||
PgPool, PgTransaction, ReadOnlyPgPool,
|
||||
models::{
|
||||
DBProjectId, DelphiReportIssueDetailsId,
|
||||
delphi_report_item::DelphiSeverity,
|
||||
},
|
||||
redis::RedisPool,
|
||||
},
|
||||
models::pats::Scopes,
|
||||
@@ -25,9 +32,12 @@ use crate::{
|
||||
|
||||
const RULE_SCAN_LOCK_ID: i64 = 0x6465_6c70_6869_7275;
|
||||
const PROGRESS_INTERVAL: usize = 50;
|
||||
const DUMMY_ISSUE_TYPE: &str = "__dummy";
|
||||
|
||||
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
|
||||
cfg.service(get_rule_schema).service(scan_rules);
|
||||
cfg.service(get_rule_schema)
|
||||
.service(get_detail_rule_input)
|
||||
.service(scan_rules);
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -45,40 +55,40 @@ struct RuleScanErrorEvent<'a> {
|
||||
}
|
||||
|
||||
#[derive(Serialize, utoipa::ToSchema)]
|
||||
pub(super) struct RuleInput {
|
||||
pub(super) schema_version: u32,
|
||||
pub(super) trace: RuleTrace,
|
||||
pub(super) scan: RuleScan,
|
||||
pub(super) artifact: RuleArtifact,
|
||||
pub(super) scope: RuleScope,
|
||||
pub struct RuleInput {
|
||||
pub schema_version: u32,
|
||||
pub trace: RuleTrace,
|
||||
pub scan: RuleScan,
|
||||
pub artifact: RuleArtifact,
|
||||
pub scope: RuleScope,
|
||||
}
|
||||
|
||||
#[derive(Serialize, utoipa::ToSchema)]
|
||||
pub(super) struct RuleTrace {
|
||||
pub(super) key: String,
|
||||
pub(super) issue_type: String,
|
||||
pub(super) severity: DelphiSeverity,
|
||||
pub(super) jar: Option<String>,
|
||||
pub(super) file_path: String,
|
||||
pub(super) data: HashMap<String, serde_json::Value>,
|
||||
pub struct RuleTrace {
|
||||
pub key: String,
|
||||
pub issue_type: String,
|
||||
pub severity: DelphiSeverity,
|
||||
pub jar: Option<String>,
|
||||
pub file_path: String,
|
||||
pub data: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, utoipa::ToSchema)]
|
||||
pub(super) struct RuleScan {
|
||||
pub(super) delphi_version: i32,
|
||||
pub struct RuleScan {
|
||||
pub delphi_version: i32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, utoipa::ToSchema)]
|
||||
pub(super) struct RuleArtifact {
|
||||
pub(super) size: Option<i32>,
|
||||
pub(super) hashes: BTreeMap<String, String>,
|
||||
pub struct RuleArtifact {
|
||||
pub size: Option<i32>,
|
||||
pub hashes: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, utoipa::ToSchema)]
|
||||
pub(super) struct RuleScope {
|
||||
pub(super) project_id: Option<String>,
|
||||
pub(super) version_id: Option<String>,
|
||||
pub(super) file_id: Option<String>,
|
||||
pub struct RuleScope {
|
||||
pub project_id: Option<String>,
|
||||
pub version_id: Option<String>,
|
||||
pub file_id: Option<String>,
|
||||
}
|
||||
|
||||
struct CompiledRule {
|
||||
@@ -145,6 +155,101 @@ pub async fn get_rule_schema(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Get the exact CEL input for a Delphi issue detail.
|
||||
#[utoipa::path(
|
||||
context_path = "/moderation/tech-review",
|
||||
tag = "moderation",
|
||||
security(("bearer_auth" = [])),
|
||||
responses(
|
||||
(status = OK, body = RuleInput),
|
||||
(status = NOT_FOUND, description = "Detail not found")
|
||||
)
|
||||
)]
|
||||
#[get("/rules/details/{detail_id}/input")]
|
||||
pub async fn get_detail_rule_input(
|
||||
req: HttpRequest,
|
||||
pool: web::Data<PgPool>,
|
||||
ro_pool: web::Data<ReadOnlyPgPool>,
|
||||
redis: web::Data<RedisPool>,
|
||||
session_queue: web::Data<AuthQueue>,
|
||||
path: web::Path<(DelphiReportIssueDetailsId,)>,
|
||||
) -> Result<web::Json<RuleInput>, ApiError> {
|
||||
check_is_moderator_from_headers(
|
||||
&req,
|
||||
&**pool,
|
||||
&redis,
|
||||
&session_queue,
|
||||
Scopes::PROJECT_READ,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (detail_id,) = path.into_inner();
|
||||
let detail = crate::util::error::Context::wrap_internal_err(
|
||||
sqlx::query!(
|
||||
r#"
|
||||
SELECT
|
||||
detail.key,
|
||||
issue.issue_type,
|
||||
detail.severity AS "severity: DelphiSeverity",
|
||||
detail.jar,
|
||||
detail.file_path,
|
||||
detail.data AS "data: Json<HashMap<String, serde_json::Value>>",
|
||||
report.delphi_version,
|
||||
file.size AS "size?",
|
||||
file.id AS "file_id?",
|
||||
version.id AS "version_id?",
|
||||
version.mod_id AS "project_id?",
|
||||
COALESCE(file_hashes.hashes, '{}'::jsonb)
|
||||
AS "hashes!: Json<BTreeMap<String, String>>"
|
||||
FROM delphi_report_issue_details detail
|
||||
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
|
||||
INNER JOIN delphi_reports report ON report.id = issue.report_id
|
||||
LEFT JOIN files file ON file.id = report.file_id
|
||||
LEFT JOIN versions version ON version.id = file.version_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT
|
||||
jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes
|
||||
FROM hashes
|
||||
WHERE hashes.file_id = file.id
|
||||
) file_hashes ON TRUE
|
||||
WHERE
|
||||
detail.id = $1
|
||||
AND issue.issue_type != $2
|
||||
"#,
|
||||
detail_id as DelphiReportIssueDetailsId,
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.fetch_optional(&***ro_pool)
|
||||
.await,
|
||||
"failed to fetch delphi rule input",
|
||||
)?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
|
||||
Ok(web::Json(RuleInput {
|
||||
schema_version: 1,
|
||||
trace: RuleTrace {
|
||||
key: detail.key,
|
||||
issue_type: detail.issue_type,
|
||||
severity: detail.severity,
|
||||
jar: detail.jar,
|
||||
file_path: detail.file_path,
|
||||
data: detail.data.0,
|
||||
},
|
||||
scan: RuleScan {
|
||||
delphi_version: detail.delphi_version,
|
||||
},
|
||||
artifact: RuleArtifact {
|
||||
size: detail.size,
|
||||
hashes: detail.hashes.0,
|
||||
},
|
||||
scope: RuleScope {
|
||||
project_id: detail.project_id.map(to_public_id),
|
||||
version_id: detail.version_id.map(to_public_id),
|
||||
file_id: detail.file_id.map(to_public_id),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
fn schema_to_value<T: Serialize>(
|
||||
schema: T,
|
||||
) -> Result<serde_json::Value, ApiError> {
|
||||
@@ -274,31 +379,16 @@ async fn run_scan(
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| eyre!("delphi rule revision overflowed"))?;
|
||||
|
||||
let rules = sqlx::query!(
|
||||
r#"
|
||||
SELECT id, rule
|
||||
FROM delphi_rules
|
||||
WHERE NOT delete_on_next_revision
|
||||
ORDER BY id
|
||||
"#,
|
||||
)
|
||||
.fetch_all(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to fetch delphi rules")?
|
||||
.into_iter()
|
||||
.map(|rule| {
|
||||
let program = cel::Program::compile(&rule.rule).map_err(|error| {
|
||||
eyre!("failed to compile delphi rule {}: {error}", rule.id)
|
||||
})?;
|
||||
Ok(CompiledRule {
|
||||
id: rule.id,
|
||||
program,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let rules = fetch_compiled_rules(&mut transaction).await?;
|
||||
|
||||
let total = sqlx::query_scalar!(
|
||||
"SELECT COUNT(*) AS \"count!\" FROM delphi_report_issue_details",
|
||||
r#"
|
||||
SELECT COUNT(*) AS "count!"
|
||||
FROM delphi_report_issue_details detail
|
||||
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
|
||||
WHERE issue.issue_type != $1
|
||||
"#,
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.fetch_one(&mut transaction)
|
||||
.await
|
||||
@@ -333,8 +423,10 @@ async fn run_scan(
|
||||
FROM hashes
|
||||
GROUP BY file_id
|
||||
) file_hashes ON file_hashes.file_id = file.id
|
||||
WHERE issue.issue_type != $1
|
||||
ORDER BY detail.id
|
||||
"#,
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.fetch(&mut transaction);
|
||||
|
||||
@@ -407,6 +499,237 @@ async fn run_scan(
|
||||
|
||||
send_progress(sender, "publishing", revision, total, total, effects.len());
|
||||
|
||||
insert_materialized_effects(revision, &effects, &mut transaction).await?;
|
||||
|
||||
let affected_projects = sqlx::query!(
|
||||
r#"
|
||||
WITH project_membership AS (
|
||||
SELECT
|
||||
detail.project_id,
|
||||
BOOL_OR(
|
||||
detail.status IN ('pending', 'unsafe')
|
||||
AND issue.issue_type != $2
|
||||
AND NOT detail.hidden
|
||||
) AS old_needs_review,
|
||||
BOOL_OR(
|
||||
detail.status IN ('pending', 'unsafe')
|
||||
AND issue.issue_type != $2
|
||||
AND NOT COALESCE(new_effect.hidden, FALSE)
|
||||
) AS new_needs_review
|
||||
FROM delphi_issue_details_with_statuses detail
|
||||
INNER JOIN delphi_report_issues issue
|
||||
ON issue.id = detail.issue_id
|
||||
LEFT JOIN delphi_rule_effects new_effect
|
||||
ON new_effect.revision = $1
|
||||
AND new_effect.detail_id = detail.id
|
||||
GROUP BY detail.project_id
|
||||
)
|
||||
SELECT project_id AS "project_id!: DBProjectId"
|
||||
FROM project_membership
|
||||
WHERE old_needs_review IS DISTINCT FROM new_needs_review
|
||||
"#,
|
||||
revision,
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.fetch_all(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to fetch projects affected by delphi rule changes")?;
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision",
|
||||
revision,
|
||||
)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to update delphi rule revisions")?;
|
||||
sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to publish the delphi rule revision")?;
|
||||
|
||||
tech_review_sync::sync_project_tech_review_state(
|
||||
&affected_projects
|
||||
.iter()
|
||||
.map(|project| project.project_id)
|
||||
.collect::<Vec<_>>(),
|
||||
TechReviewExitReason::RulesChanged,
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
eyre!(error)
|
||||
.wrap_err("failed to sync projects affected by delphi rule changes")
|
||||
})?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM delphi_rule_effects WHERE revision <> $1",
|
||||
revision,
|
||||
)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to delete old delphi rule effects")?;
|
||||
sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision")
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to delete retired delphi rules")?;
|
||||
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.wrap_err("failed to commit the delphi rule scan")?;
|
||||
|
||||
Ok(ScanSummary {
|
||||
revision,
|
||||
scanned: total,
|
||||
total,
|
||||
effects: effects.len(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn materialize_current_rule_effects(
|
||||
detail_ids: &[DelphiReportIssueDetailsId],
|
||||
transaction: &mut PgTransaction<'_>,
|
||||
) -> Result<()> {
|
||||
if detail_ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let revision = sqlx::query_scalar!(
|
||||
"SELECT revision FROM delphi_rule_revisions LIMIT 1",
|
||||
)
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.wrap_err("failed to fetch the current delphi rule revision")?;
|
||||
let rules = fetch_compiled_rules(transaction).await?;
|
||||
|
||||
if rules.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let details = sqlx::query!(
|
||||
r#"
|
||||
SELECT
|
||||
detail.id,
|
||||
detail.key,
|
||||
issue.issue_type,
|
||||
detail.severity AS "severity: DelphiSeverity",
|
||||
detail.jar,
|
||||
detail.file_path,
|
||||
detail.data AS "data: Json<HashMap<String, serde_json::Value>>",
|
||||
report.delphi_version,
|
||||
file.size AS "size?",
|
||||
file.id AS "file_id?",
|
||||
version.id AS "version_id?",
|
||||
version.mod_id AS "project_id?",
|
||||
COALESCE(file_hashes.hashes, '{}'::jsonb)
|
||||
AS "hashes!: Json<BTreeMap<String, String>>"
|
||||
FROM delphi_report_issue_details detail
|
||||
INNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id
|
||||
INNER JOIN delphi_reports report ON report.id = issue.report_id
|
||||
LEFT JOIN files file ON file.id = report.file_id
|
||||
LEFT JOIN versions version ON version.id = file.version_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT
|
||||
jsonb_object_agg(algorithm, encode(hash, 'hex')) AS hashes
|
||||
FROM hashes
|
||||
WHERE hashes.file_id = file.id
|
||||
) file_hashes ON TRUE
|
||||
WHERE
|
||||
detail.id = ANY($1::bigint[])
|
||||
AND issue.issue_type != $2
|
||||
ORDER BY detail.id
|
||||
"#,
|
||||
&detail_ids.iter().map(|id| id.0).collect::<Vec<_>>(),
|
||||
DUMMY_ISSUE_TYPE,
|
||||
)
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.wrap_err("failed to fetch new delphi issue details")?;
|
||||
|
||||
let mut effects = Vec::new();
|
||||
for detail in details {
|
||||
let input = RuleInput {
|
||||
schema_version: 1,
|
||||
trace: RuleTrace {
|
||||
key: detail.key,
|
||||
issue_type: detail.issue_type,
|
||||
severity: detail.severity,
|
||||
jar: detail.jar,
|
||||
file_path: detail.file_path,
|
||||
data: detail.data.0,
|
||||
},
|
||||
scan: RuleScan {
|
||||
delphi_version: detail.delphi_version,
|
||||
},
|
||||
artifact: RuleArtifact {
|
||||
size: detail.size,
|
||||
hashes: detail.hashes.0,
|
||||
},
|
||||
scope: RuleScope {
|
||||
project_id: detail.project_id.map(to_public_id),
|
||||
version_id: detail.version_id.map(to_public_id),
|
||||
file_id: detail.file_id.map(to_public_id),
|
||||
},
|
||||
};
|
||||
|
||||
for rule in &rules {
|
||||
let effect =
|
||||
evaluate_rule(&rule.program, &input).wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to evaluate delphi rule {} for detail {}",
|
||||
rule.id, detail.id
|
||||
)
|
||||
})?;
|
||||
if let Some(effect) = effect {
|
||||
effects.push(MaterializedEffect {
|
||||
detail_id: detail.id,
|
||||
rule_id: rule.id,
|
||||
effect,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
insert_materialized_effects(revision, &effects, transaction).await
|
||||
}
|
||||
|
||||
async fn fetch_compiled_rules(
|
||||
transaction: &mut PgTransaction<'_>,
|
||||
) -> Result<Vec<CompiledRule>> {
|
||||
sqlx::query!(
|
||||
r#"
|
||||
SELECT id, rule
|
||||
FROM delphi_rules
|
||||
WHERE NOT delete_on_next_revision
|
||||
ORDER BY id
|
||||
"#,
|
||||
)
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.wrap_err("failed to fetch delphi rules")?
|
||||
.into_iter()
|
||||
.map(|rule| {
|
||||
let program = cel::Program::compile(&rule.rule).map_err(|error| {
|
||||
eyre!("failed to compile delphi rule {}: {error}", rule.id)
|
||||
})?;
|
||||
Ok(CompiledRule {
|
||||
id: rule.id,
|
||||
program,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn insert_materialized_effects(
|
||||
revision: i64,
|
||||
effects: &[MaterializedEffect],
|
||||
transaction: &mut PgTransaction<'_>,
|
||||
) -> Result<()> {
|
||||
if effects.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let detail_ids = effects
|
||||
.iter()
|
||||
.map(|effect| effect.detail_id)
|
||||
@@ -424,69 +747,34 @@ async fn run_scan(
|
||||
.map(|effect| effect.effect.hidden)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !effects.is_empty() {
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO delphi_rule_effects (
|
||||
revision,
|
||||
detail_id,
|
||||
rule_id,
|
||||
severity,
|
||||
hidden
|
||||
)
|
||||
SELECT $1, effect.*
|
||||
FROM UNNEST(
|
||||
$2::BIGINT[],
|
||||
$3::BIGINT[],
|
||||
$4::delphi_severity[],
|
||||
$5::BOOLEAN[]
|
||||
) AS effect(detail_id, rule_id, severity, hidden)
|
||||
"#,
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO delphi_rule_effects (
|
||||
revision,
|
||||
&detail_ids,
|
||||
&rule_ids,
|
||||
&severities as &[Option<DelphiSeverity>],
|
||||
&hidden,
|
||||
detail_id,
|
||||
rule_id,
|
||||
severity,
|
||||
hidden
|
||||
)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to insert delphi rule effects")?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM delphi_rule_effects WHERE revision <> $1",
|
||||
SELECT $1, effect.*
|
||||
FROM UNNEST(
|
||||
$2::BIGINT[],
|
||||
$3::BIGINT[],
|
||||
$4::delphi_severity[],
|
||||
$5::BOOLEAN[]
|
||||
) AS effect(detail_id, rule_id, severity, hidden)
|
||||
"#,
|
||||
revision,
|
||||
&detail_ids,
|
||||
&rule_ids,
|
||||
&severities as &[Option<DelphiSeverity>],
|
||||
&hidden,
|
||||
)
|
||||
.execute(&mut transaction)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.wrap_err("failed to delete old delphi rule effects")?;
|
||||
sqlx::query!("DELETE FROM delphi_rules WHERE delete_on_next_revision")
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to delete retired delphi rules")?;
|
||||
sqlx::query!(
|
||||
"UPDATE delphi_rules SET revision = $1 WHERE NOT delete_on_next_revision",
|
||||
revision,
|
||||
)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to update delphi rule revisions")?;
|
||||
sqlx::query!("UPDATE delphi_rule_revisions SET revision = $1", revision)
|
||||
.execute(&mut transaction)
|
||||
.await
|
||||
.wrap_err("failed to publish the delphi rule revision")?;
|
||||
.wrap_err("failed to insert delphi rule effects")?;
|
||||
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.wrap_err("failed to commit the delphi rule scan")?;
|
||||
|
||||
Ok(ScanSummary {
|
||||
revision,
|
||||
scanned: total,
|
||||
total,
|
||||
effects: effects.len(),
|
||||
})
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn evaluate_rule(
|
||||
|
||||
Reference in New Issue
Block a user