fix: file attribution scanning not marking completed files (#6500)

* fix: file attribution scanning not marking completed files * Clean up unused attribution files/groups on deletion * prepare * Handle file scan failures gracefully with tracking
2026-08-25 00:55:25 +00:00 · 2026-06-25 15:57:51 +00:00
parent f4a2999b20
commit 527757d158
16 changed files with 215 additions and 182 deletions
@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        update file_scans\n        set\n            attributions_scanned_at = $2,\n            attributions_scan_result = $3\n        where file_id = $1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Timestamptz",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "11228045ad657e66693d4b384beacbd50d74251f59c1147e71b0ef293a19805e"
+}
@@ -1,34 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                                        SELECT mel.id, mel.flame_project_id, mel.status status\n                                        FROM moderation_external_licenses mel\n                                        WHERE mel.flame_project_id = ANY($1)\n                                        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "flame_project_id",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 2,
-        "name": "status",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4Array"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      false
-    ]
-  },
-  "hash": "3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399"
-}
@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                update file_scans\n                set attributions_scanned_at = $2\n                where file_id = $1\n                ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Timestamptz"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4748e425c6bbd4d154bf658bf695661485a9700bffa413c5e94d80a525b53e27"
-}
@@ -1,28 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                                        SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n                                        FROM moderation_external_files mef\n                                        INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n                                        WHERE mef.sha1 = ANY($1)\n                                        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "sha1",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "status",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "ByteaArray"
-      ]
-    },
-    "nullable": [
-      null,
-      false
-    ]
-  },
-  "hash": "4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64"
-}
@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        insert into file_scans (file_id, attributions_scanned_at)\n        values ($1, now())\n        on conflict (file_id) do update set attributions_scanned_at = now()\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "80751dbca09a9dcb469a30fe5b8225e520baf50750a78875f582349610439a55"
-}
@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        insert into file_scans (\n            file_id,\n            attributions_scanned_at,\n            attributions_scan_result\n        )\n        values ($1, now(), $2)\n        on conflict (file_id) do update set\n            attributions_scanned_at = now(),\n            attributions_scan_result = $2\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "89e0688880584ca6745961772445cede934143b33d5ca86b6c6f889e5beee543"
+}
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "\n        DELETE FROM project_attribution_groups g\n        WHERE NOT EXISTS (\n            SELECT 1\n            FROM project_attribution_files paf\n            INNER JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n            WHERE paf.group_id = g.id\n        )\n        ",
+  "query": "\n        DELETE FROM project_attribution_groups g\n        WHERE NOT EXISTS (\n            SELECT 1\n            FROM project_attribution_files paf\n            WHERE paf.group_id = g.id\n        )\n        ",
  "describe": {
    "columns": [],
    "parameters": {
@@ -8,5 +8,5 @@
    },
    "nullable": []
  },
-  "hash": "301959b1ec20a4925f86188bc5e1c0cdd11f0d15004ce9f91fbeefcc3a27f8f4"
+  "hash": "bba6df9227000db16828d99517d1cafb5b682b3a50ea5f31f580a8198023a316"
 }
@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                                        UPDATE files\n                                        SET metadata = $1\n                                        WHERE id = $2\n                                        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db"
-}
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        select count(*) as \"count!\"\n        from file_scans fa\n        inner join files f on f.id = fa.file_id\n        inner join attribution_enforced_versions aev on aev.id = f.version_id\n        where fa.attributions_scanned_at is null\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "c47f6e360d4771339345c48f1eb62b36241639858e1f8e6aa72a2db6b6def51e"
+}
@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                                            UPDATE files\n                                            SET metadata = $1\n                                            WHERE id = $2\n                                            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37"
-}
@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        DELETE FROM project_attribution_files paf\n        USING project_attribution_groups pag\n        WHERE pag.id = paf.group_id\n            AND NOT EXISTS (\n            SELECT 1\n            FROM override_file_sources ofs\n            INNER JOIN files f ON f.id = ofs.file_id\n            INNER JOIN versions v ON v.id = f.version_id\n            WHERE ofs.sha1 = paf.sha1\n                AND v.mod_id = pag.project_id\n        )\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "d46c53877e97d54f38895f070bdc0a2417ba87ecac65edd03e8dc7a677f5efa9"
+}
@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        select count(*) as \"count!\" from file_scans\n        where attributions_scanned_at is null\n        ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "count!",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "eaafc6b2e29562112ae323e68955aa99e70f8ecbdc1b2500f7d7e89af8fc6e29"
-}