clean up db stuff

This commit is contained in:
aecsocket
2026-07-27 10:41:26 +01:00
parent 18c95e2f61
commit 849943ed4e
11 changed files with 101 additions and 98 deletions
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tSELECT\n\t\t\tdelphi_rule.id,\n\t\t\tdelphi_rule.name,\n\t\t\tdelphi_rule.rule,\n\t\t\tdelphi_rule.priority,\n\t\t\tdelphi_rule.revision,\n\t\t\tdelphi_rule.created_at,\n\t\t\tdelphi_rule.updated_at,\n\t\t\tdelphi_rule.created_by,\n\t\t\tdelphi_rule.updated_by,\n\t\t\tCOALESCE(preview.affected_details_count, 0)\n\t\t\t\tAS \"affected_details_count!\",\n\t\t\tpreview.detail_id AS \"detail_id?: DelphiReportIssueDetailsId\",\n\t\t\tpreview.issue_id AS \"issue_id?: DelphiReportIssueId\",\n\t\t\tpreview.project_id AS \"project_id?: DBProjectId\",\n\t\t\tpreview.project_name AS \"project_name?\",\n\t\t\tpreview.project_icon_url AS \"project_icon_url?\",\n\t\t\tpreview.version_id AS \"version_id?: DBVersionId\",\n\t\t\tpreview.version_name AS \"version_name?\",\n\t\t\tpreview.version_number AS \"version_number?\",\n\t\t\tpreview.issue_type AS \"issue_type?\",\n\t\t\tpreview.key AS \"key?\",\n\t\t\tpreview.jar AS \"jar?\",\n\t\t\tpreview.file_path AS \"file_path?\",\n\t\t\tpreview.original_severity AS \"original_severity?: DelphiSeverity\",\n\t\t\tpreview.severity AS \"effect_severity?: DelphiSeverity\",\n\t\t\tpreview.hidden AS \"hidden?\"\n\t\tFROM delphi_rules delphi_rule\n\t\tLEFT JOIN LATERAL (\n\t\t\tSELECT\n\t\t\t\teffect.detail_id,\n\t\t\t\tdetail.issue_id,\n\t\t\t\tversion.mod_id AS project_id,\n\t\t\t\tproject.name AS project_name,\n\t\t\t\tproject.icon_url AS project_icon_url,\n\t\t\t\tversion.id AS version_id,\n\t\t\t\tversion.name AS version_name,\n\t\t\t\tversion.version_number,\n\t\t\t\tissue.issue_type,\n\t\t\t\tdetail.key,\n\t\t\t\tdetail.jar,\n\t\t\t\tdetail.file_path,\n\t\t\t\tdetail.severity AS original_severity,\n\t\t\t\teffect.severity,\n\t\t\t\teffect.hidden,\n\t\t\t\tCOUNT(*) OVER () AS affected_details_count\n\t\t\tFROM delphi_rule_effects effect\n\t\t\tINNER JOIN delphi_rule_revisions published\n\t\t\t\tON published.revision = effect.revision\n\t\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\t\tON detail.id = effect.detail_id\n\t\t\tINNER JOIN delphi_report_issues issue\n\t\t\t\tON issue.id = detail.issue_id\n\t\t\tINNER JOIN delphi_reports report\n\t\t\t\tON report.id = issue.report_id\n\t\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\t\tWHERE effect.rule_id = delphi_rule.id\n\t\t\tORDER BY effect.detail_id DESC\n\t\t\tLIMIT 3\n\t\t) preview ON TRUE\n\t\tWHERE NOT delphi_rule.delete_on_next_revision\n\t\tORDER BY\n\t\t\tdelphi_rule.priority DESC,\n\t\t\tdelphi_rule.id,\n\t\t\tpreview.detail_id DESC\n\t\t",
"query": "\n\t\tSELECT\n\t\t\tdelphi_rule.id AS \"id!: DelphiRuleId\",\n\t\t\tdelphi_rule.name,\n\t\t\tdelphi_rule.rule,\n\t\t\tdelphi_rule.priority,\n\t\t\tdelphi_rule.revision,\n\t\t\tdelphi_rule.created_at,\n\t\t\tdelphi_rule.updated_at,\n\t\t\tdelphi_rule.created_by,\n\t\t\tdelphi_rule.updated_by,\n\t\t\tCOALESCE(preview.affected_details_count, 0)\n\t\t\t\tAS \"affected_details_count!\",\n\t\t\tpreview.detail_id AS \"detail_id?: DelphiReportIssueDetailsId\",\n\t\t\tpreview.issue_id AS \"issue_id?: DelphiReportIssueId\",\n\t\t\tpreview.project_id AS \"project_id?: DBProjectId\",\n\t\t\tpreview.project_name AS \"project_name?\",\n\t\t\tpreview.project_icon_url AS \"project_icon_url?\",\n\t\t\tpreview.version_id AS \"version_id?: DBVersionId\",\n\t\t\tpreview.version_name AS \"version_name?\",\n\t\t\tpreview.version_number AS \"version_number?\",\n\t\t\tpreview.issue_type AS \"issue_type?\",\n\t\t\tpreview.key AS \"key?\",\n\t\t\tpreview.jar AS \"jar?\",\n\t\t\tpreview.file_path AS \"file_path?\",\n\t\t\tpreview.original_severity AS \"original_severity?: DelphiSeverity\",\n\t\t\tpreview.severity AS \"effect_severity?: DelphiSeverity\",\n\t\t\tpreview.hidden AS \"hidden?\"\n\t\tFROM delphi_rules delphi_rule\n\t\tLEFT JOIN LATERAL (\n\t\t\tSELECT\n\t\t\t\teffect.detail_id,\n\t\t\t\tdetail.issue_id,\n\t\t\t\tversion.mod_id AS project_id,\n\t\t\t\tproject.name AS project_name,\n\t\t\t\tproject.icon_url AS project_icon_url,\n\t\t\t\tversion.id AS version_id,\n\t\t\t\tversion.name AS version_name,\n\t\t\t\tversion.version_number,\n\t\t\t\tissue.issue_type,\n\t\t\t\tdetail.key,\n\t\t\t\tdetail.jar,\n\t\t\t\tdetail.file_path,\n\t\t\t\tdetail.severity AS original_severity,\n\t\t\t\teffect.severity,\n\t\t\t\teffect.hidden,\n\t\t\t\tCOUNT(*) OVER () AS affected_details_count\n\t\t\tFROM delphi_rule_effects effect\n\t\t\tINNER JOIN delphi_rule_revisions published\n\t\t\t\tON published.revision = effect.revision\n\t\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\t\tON detail.id = effect.detail_id\n\t\t\tINNER JOIN delphi_report_issues issue\n\t\t\t\tON issue.id = detail.issue_id\n\t\t\tINNER JOIN delphi_reports report\n\t\t\t\tON report.id = issue.report_id\n\t\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\t\tWHERE effect.rule_id = delphi_rule.id\n\t\t\tORDER BY effect.detail_id DESC\n\t\t\tLIMIT 3\n\t\t) preview ON TRUE\n\t\tWHERE NOT delphi_rule.delete_on_next_revision\n\t\tORDER BY\n\t\t\tdelphi_rule.priority DESC,\n\t\t\tdelphi_rule.id,\n\t\t\tpreview.detail_id DESC\n\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"name": "id!: DelphiRuleId",
"type_info": "Int8"
},
{
@@ -184,5 +184,5 @@
false
]
},
"hash": "8c69ee780e418c4ffd5c7b8d23dea9eb742e7365dd3c93479a9d7a5b6283c9b1"
"hash": "159d482c1859b0d0d42a15fb2b49e3a9a17755694448e00bf5e5faccb3ace76d"
}
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, rule\n FROM delphi_rules\n WHERE NOT delete_on_next_revision\n ORDER BY priority DESC, id\n ",
"query": "\n SELECT id AS \"id!: DelphiRuleId\", rule\n FROM delphi_rules\n WHERE NOT delete_on_next_revision\n ORDER BY priority DESC, id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"name": "id!: DelphiRuleId",
"type_info": "Int8"
},
{
@@ -22,5 +22,5 @@
false
]
},
"hash": "9a93028d19a05ebffb654fbb84ce2326fde91ed9762ad05ee0acda7fdf0245b5"
"hash": "3f1aaeed6e394294fa32da7e7f83765fb64bd9499a796470d544466886e1a808"
}
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tINSERT INTO delphi_rules (\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t)\n\t\tVALUES (\n\t\t\t$1,\n\t\t\t$2,\n\t\t\t$3,\n\t\t\t(SELECT revision + 1 FROM delphi_rule_revisions LIMIT 1),\n\t\t\t$4,\n\t\t\t$4\n\t\t)\n\t\tRETURNING\n\t\t\tid,\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_at,\n\t\t\tupdated_at,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t",
"query": "\n\t\tINSERT INTO delphi_rules (\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t)\n\t\tVALUES (\n\t\t\t$1,\n\t\t\t$2,\n\t\t\t$3,\n\t\t\t(SELECT revision + 1 FROM delphi_rule_revisions LIMIT 1),\n\t\t\t$4,\n\t\t\t$4\n\t\t)\n\t\tRETURNING\n\t\t\tid AS \"id!: DelphiRuleId\",\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_at,\n\t\t\tupdated_at,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"name": "id!: DelphiRuleId",
"type_info": "Int8"
},
{
@@ -69,5 +69,5 @@
true
]
},
"hash": "2008d5b755863a2128670198c3cd6e41587c64e2e1c48afe9e68ef42ce7f9358"
"hash": "b0004df8d3b1c5f9193a21f981452ce4ae56f608ec698f6f1ebb4c470bb2f0f5"
}
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "\n\t\tUPDATE delphi_rules\n\t\tSET\n\t\t\tname = $2,\n\t\t\trule = $3,\n\t\t\tpriority = $4,\n\t\t\trevision = (\n\t\t\t\tSELECT revision + 1 FROM delphi_rule_revisions LIMIT 1\n\t\t\t),\n\t\t\tupdated_at = CURRENT_TIMESTAMP,\n\t\t\tupdated_by = $5\n\t\tWHERE id = $1 AND NOT delete_on_next_revision\n\t\tRETURNING\n\t\t\tid,\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_at,\n\t\t\tupdated_at,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t",
"query": "\n\t\tUPDATE delphi_rules\n\t\tSET\n\t\t\tname = $2,\n\t\t\trule = $3,\n\t\t\tpriority = $4,\n\t\t\trevision = (\n\t\t\t\tSELECT revision + 1 FROM delphi_rule_revisions LIMIT 1\n\t\t\t),\n\t\t\tupdated_at = CURRENT_TIMESTAMP,\n\t\t\tupdated_by = $5\n\t\tWHERE id = $1 AND NOT delete_on_next_revision\n\t\tRETURNING\n\t\t\tid AS \"id!: DelphiRuleId\",\n\t\t\tname,\n\t\t\trule,\n\t\t\tpriority,\n\t\t\trevision,\n\t\t\tcreated_at,\n\t\t\tupdated_at,\n\t\t\tcreated_by,\n\t\t\tupdated_by\n\t\t",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"name": "id!: DelphiRuleId",
"type_info": "Int8"
},
{
@@ -70,5 +70,5 @@
true
]
},
"hash": "5261240093589617a854a28fca81b7d3faa88dad9575cdf30ae8aa9e7721879f"
"hash": "d973a1edd200b4cd68ec30556a1e12240885f2372265597efa6e0361831d1318"
}
@@ -8,6 +8,7 @@ CREATE TABLE delphi_rules (
ON DELETE SET NULL,
name VARCHAR(256) NOT NULL,
rule VARCHAR(65536) NOT NULL,
priority INTEGER NOT NULL DEFAULT 0,
revision BIGINT NOT NULL DEFAULT 0,
delete_on_next_revision BOOL NOT NULL DEFAULT FALSE
);
@@ -32,3 +33,70 @@ CREATE TABLE delphi_rule_effects (
);
CREATE INDEX delphi_rule_effects_rule_id ON delphi_rule_effects(rule_id);
DROP VIEW delphi_issue_details_with_statuses;
CREATE VIEW delphi_issue_details_with_statuses AS
SELECT
drid.id,
drid.issue_id,
drid.key,
drid.jar,
drid.file_path,
drid.decompiled_source,
drid.data,
drid.severity AS original_severity,
COALESCE(dre.severity, drid.severity) AS severity,
COALESCE(dre.hidden, FALSE) AS hidden,
m.id AS project_id,
didv.verdict AS local_status,
dgdv.verdict AS global_status,
COALESCE(dgdv.verdict, didv.verdict, 'pending') AS status
FROM delphi_report_issue_details drid
INNER JOIN delphi_report_issues dri ON dri.id = drid.issue_id
INNER JOIN delphi_reports dr ON dr.id = dri.report_id
INNER JOIN files f ON f.id = dr.file_id
INNER JOIN versions v ON v.id = f.version_id
INNER JOIN mods m ON m.id = v.mod_id
LEFT JOIN delphi_global_detail_verdicts dgdv
ON drid.key = dgdv.detail_key
LEFT JOIN delphi_issue_detail_verdicts didv
ON m.id = didv.project_id
AND drid.key = didv.detail_key
LEFT JOIN (
SELECT revision
FROM delphi_rule_revisions
LIMIT 1
) drr ON TRUE
LEFT JOIN delphi_rule_effects dre
ON dre.revision = drr.revision
AND dre.detail_id = drid.id;
CREATE TABLE delphi_tech_review_queue (
project_id BIGINT PRIMARY KEY REFERENCES mods(id)
ON DELETE CASCADE
);
INSERT INTO delphi_tech_review_queue (project_id)
SELECT DISTINCT didws.project_id
FROM delphi_issue_details_with_statuses didws
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
WHERE
(
dri.issue_type = '__dummy'
AND didws.status = 'pending'
)
OR (
dri.issue_type != '__dummy'
AND didws.status IN ('pending', 'unsafe')
AND NOT didws.hidden
);
DELETE FROM delphi_report_issue_details detail
USING delphi_report_issues issue
WHERE
detail.issue_id = issue.id
AND issue.issue_type = '__dummy';
DELETE FROM delphi_report_issues
WHERE issue_type = '__dummy';
@@ -1,37 +0,0 @@
DROP VIEW delphi_issue_details_with_statuses;
CREATE VIEW delphi_issue_details_with_statuses AS
SELECT
drid.id,
drid.issue_id,
drid.key,
drid.jar,
drid.file_path,
drid.decompiled_source,
drid.data,
drid.severity AS original_severity,
COALESCE(dre.severity, drid.severity) AS severity,
COALESCE(dre.hidden, FALSE) AS hidden,
m.id AS project_id,
didv.verdict AS local_status,
dgdv.verdict AS global_status,
COALESCE(dgdv.verdict, didv.verdict, 'pending') AS status
FROM delphi_report_issue_details drid
INNER JOIN delphi_report_issues dri ON dri.id = drid.issue_id
INNER JOIN delphi_reports dr ON dr.id = dri.report_id
INNER JOIN files f ON f.id = dr.file_id
INNER JOIN versions v ON v.id = f.version_id
INNER JOIN mods m ON m.id = v.mod_id
LEFT JOIN delphi_global_detail_verdicts dgdv
ON drid.key = dgdv.detail_key
LEFT JOIN delphi_issue_detail_verdicts didv
ON m.id = didv.project_id
AND drid.key = didv.detail_key
LEFT JOIN (
SELECT revision
FROM delphi_rule_revisions
LIMIT 1
) drr ON TRUE
LEFT JOIN delphi_rule_effects dre
ON dre.revision = drr.revision
AND dre.detail_id = drid.id;
@@ -1,2 +0,0 @@
ALTER TABLE delphi_rules
ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;
@@ -1,28 +0,0 @@
CREATE TABLE delphi_tech_review_queue (
project_id BIGINT PRIMARY KEY REFERENCES mods(id)
ON DELETE CASCADE
);
INSERT INTO delphi_tech_review_queue (project_id)
SELECT DISTINCT didws.project_id
FROM delphi_issue_details_with_statuses didws
INNER JOIN delphi_report_issues dri ON dri.id = didws.issue_id
WHERE
(
dri.issue_type = '__dummy'
AND didws.status = 'pending'
)
OR (
dri.issue_type != '__dummy'
AND didws.status IN ('pending', 'unsafe')
AND NOT didws.hidden
);
DELETE FROM delphi_report_issue_details detail
USING delphi_report_issues issue
WHERE
detail.issue_id = issue.id
AND issue.issue_type = '__dummy';
DELETE FROM delphi_report_issues
WHERE issue_type = '__dummy';
+1
View File
@@ -292,4 +292,5 @@ id_type!(StatusId as i32);
id_type!(DelphiReportId as i64);
id_type!(DelphiReportIssueId as i64);
id_type!(DelphiReportIssueDetailsId as i64);
id_type!(DelphiRuleId as i64);
id_type!(SharedInstanceId as i64);
@@ -15,7 +15,8 @@ use crate::{
PgPool, ReadOnlyPgPool,
models::{
DBProjectId, DBVersionId, DelphiReportIssueDetailsId,
DelphiReportIssueId, delphi_report_item::DelphiSeverity,
DelphiReportIssueId, DelphiRuleId,
delphi_report_item::DelphiSeverity,
},
},
models::{
@@ -42,7 +43,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
#[derive(Debug, Serialize, utoipa::ToSchema)]
pub struct DelphiRule {
pub id: i64,
pub id: DelphiRuleId,
pub name: String,
pub rule: String,
pub priority: i32,
@@ -270,7 +271,7 @@ pub async fn get_rules(
let rules = sqlx::query!(
r#"
SELECT
delphi_rule.id,
delphi_rule.id AS "id!: DelphiRuleId",
delphi_rule.name,
delphi_rule.rule,
delphi_rule.priority,
@@ -421,7 +422,7 @@ pub async fn get_rule_affected_details(
ro_pool: web::Data<ReadOnlyPgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
path: web::Path<(i64,)>,
path: web::Path<(DelphiRuleId,)>,
) -> Result<web::Json<Vec<DelphiRuleAffectedDetail>>, ApiError> {
check_is_moderator_from_headers(
&req,
@@ -464,7 +465,7 @@ pub async fn get_rule_affected_details(
WHERE effect.rule_id = $1
ORDER BY effect.detail_id DESC
"#,
rule_id,
rule_id as DelphiRuleId,
)
.fetch_all(&***ro_pool)
.await
@@ -540,7 +541,7 @@ pub async fn create_rule(
$4
)
RETURNING
id,
id AS "id!: DelphiRuleId",
name,
rule,
priority,
@@ -588,7 +589,7 @@ pub async fn update_rule(
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
path: web::Path<(i64,)>,
path: web::Path<(DelphiRuleId,)>,
body: web::Json<WriteDelphiRule>,
) -> Result<web::Json<DelphiRule>, ApiError> {
let user = check_is_moderator_from_headers(
@@ -617,7 +618,7 @@ pub async fn update_rule(
updated_by = $5
WHERE id = $1 AND NOT delete_on_next_revision
RETURNING
id,
id AS "id!: DelphiRuleId",
name,
rule,
priority,
@@ -627,7 +628,7 @@ pub async fn update_rule(
created_by,
updated_by
"#,
id,
id as DelphiRuleId,
rule.name,
rule.rule,
rule.priority,
@@ -666,7 +667,7 @@ pub async fn delete_rule(
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
path: web::Path<(i64,)>,
path: web::Path<(DelphiRuleId,)>,
) -> Result<(), ApiError> {
let user = check_is_moderator_from_headers(
&req,
@@ -691,7 +692,7 @@ pub async fn delete_rule(
WHERE id = $1 AND NOT delete_on_next_revision
RETURNING id
"#,
id,
id as DelphiRuleId,
user.id.0 as i64,
)
.fetch_optional(&**pool)
@@ -21,7 +21,7 @@ use crate::{
database::{
PgPool, PgTransaction, ReadOnlyPgPool,
models::{
DBProjectId, DelphiReportIssueDetailsId,
DBProjectId, DelphiReportIssueDetailsId, DelphiRuleId,
delphi_report_item::DelphiSeverity,
},
},
@@ -90,13 +90,13 @@ pub struct RuleScope {
}
struct CompiledRule {
id: i64,
id: DelphiRuleId,
program: cel::Program,
}
struct MaterializedEffect {
detail_id: i64,
rule_id: i64,
rule_id: DelphiRuleId,
effect: DelphiRuleEffect,
}
@@ -459,7 +459,7 @@ async fn run_scan(
.wrap_err_with(|| {
format!(
"failed to evaluate delphi rule {} for detail {detail_id}",
rule.id
rule.id.0
)
})?;
if let Some(effect) = effect {
@@ -667,7 +667,7 @@ pub(crate) async fn materialize_current_rule_effects(
evaluate_rule(&rule.program, &input).wrap_err_with(|| {
format!(
"failed to evaluate delphi rule {} for detail {}",
rule.id, detail.id
rule.id.0, detail.id
)
})?;
if let Some(effect) = effect {
@@ -689,7 +689,7 @@ async fn fetch_compiled_rules(
) -> Result<Vec<CompiledRule>> {
sqlx::query!(
r#"
SELECT id, rule
SELECT id AS "id!: DelphiRuleId", rule
FROM delphi_rules
WHERE NOT delete_on_next_revision
ORDER BY priority DESC, id
@@ -701,7 +701,7 @@ async fn fetch_compiled_rules(
.into_iter()
.map(|rule| {
let program = cel::Program::compile(&rule.rule).map_err(|error| {
eyre!("failed to compile delphi rule {}: {error}", rule.id)
eyre!("failed to compile delphi rule {}: {error}", rule.id.0)
})?;
Ok(CompiledRule {
id: rule.id,
@@ -726,7 +726,7 @@ async fn insert_materialized_effects(
.collect::<Vec<_>>();
let rule_ids = effects
.iter()
.map(|effect| effect.rule_id)
.map(|effect| effect.rule_id.0)
.collect::<Vec<_>>();
let severities = effects
.iter()