Batch pending file scanning ops (#6491)

* Batch pending file scanning ops

* prepare

* move some stuff out of the txn

* scan concurrency

* prepr

* 🦀 automod attribution is dead, long live automod

* fix ci

* update scan file filtering
This commit is contained in:
aecsocket
2026-06-24 21:46:07 +00:00
committed by GitHub
parent 586cdc90f9
commit f4a2999b20
6 changed files with 176 additions and 394 deletions
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_scans fa\n inner join files f on f.id = fa.file_id\n inner join attribution_enforced_versions aev on aev.id = f.version_id\n inner join versions v on v.id = f.version_id\n where fa.attributions_scanned_at is null\n ",
"query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_scans fa\n inner join files f on f.id = fa.file_id\n inner join attribution_enforced_versions aev on aev.id = f.version_id\n inner join versions v on v.id = f.version_id\n where fa.attributions_scanned_at is null\n order by fa.file_id\n limit $1\n ",
"describe": {
"columns": [
{
@@ -20,7 +20,9 @@
}
],
"parameters": {
"Left": []
"Left": [
"Int8"
]
},
"nullable": [
false,
@@ -28,5 +30,5 @@
false
]
},
"hash": "0dda0265b39d4c8b019eb1ea6d164ee639a6deb0f47c1dcdd83a5816308faab0"
"hash": "502a0fcf14a874beb2709d5518fb7f4dd3ec2d110bd6e75caa1fbe3014d8e412"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "\n select count(*) as \"count!\" from file_scans\n where attributions_scanned_at is null\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "eaafc6b2e29562112ae323e68955aa99e70f8ecbdc1b2500f7d7e89af8fc6e29"
}