diff --git a/packages/app-lib/.sqlx/query-59a314ee065e24af6d7138fcf11eed0eafc29732a15f25d051964660941c8d33.json b/packages/app-lib/.sqlx/query-59a314ee065e24af6d7138fcf11eed0eafc29732a15f25d051964660941c8d33.json new file mode 100644 index 0000000000..6542c63d10 --- /dev/null +++ b/packages/app-lib/.sqlx/query-59a314ee065e24af6d7138fcf11eed0eafc29732a15f25d051964660941c8d33.json @@ -0,0 +1,74 @@ +{ + "db_name": "SQLite", + "query": "\n\t\tINSERT INTO instance_files (\n\t\t\tid,\n\t\t\tinstance_id,\n\t\t\trelative_path,\n\t\t\tfile_name,\n\t\t\tenabled,\n\t\t\tsha1,\n\t\t\tsize,\n\t\t\tmissing,\n\t\t\tadded_at,\n\t\t\tmodified_at\n\t\t)\n\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON CONFLICT (instance_id, relative_path) DO UPDATE SET\n\t\t\tfile_name = excluded.file_name,\n\t\t\tenabled = excluded.enabled,\n\t\t\tsha1 = excluded.sha1,\n\t\t\tsize = excluded.size,\n\t\t\tmissing = excluded.missing,\n\t\t\tmodified_at = excluded.modified_at\n\t\tRETURNING\n\t\t\tid,\n\t\t\tinstance_id,\n\t\t\trelative_path,\n\t\t\tfile_name,\n\t\t\tenabled,\n\t\t\tsha1,\n\t\t\tsize,\n\t\t\tmissing,\n\t\t\tadded_at,\n\t\t\tmodified_at\n\t\t", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "instance_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "relative_path", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "file_name", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "enabled", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "sha1", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "size", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "missing", + "ordinal": 7, + "type_info": "Integer" + }, + { + "name": "added_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "modified_at", + "ordinal": 9, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 10 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "59a314ee065e24af6d7138fcf11eed0eafc29732a15f25d051964660941c8d33" +} diff --git a/packages/app-lib/.sqlx/query-636bb0c426ac275d64f7c84b3349894d10faf920d56814c0c2336e4cd5002743.json b/packages/app-lib/.sqlx/query-636bb0c426ac275d64f7c84b3349894d10faf920d56814c0c2336e4cd5002743.json deleted file mode 100644 index 62a6c9db07..0000000000 --- a/packages/app-lib/.sqlx/query-636bb0c426ac275d64f7c84b3349894d10faf920d56814c0c2336e4cd5002743.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n\t\t\tSELECT id\n\t\t\tFROM instance_content_entries\n\t\t\tWHERE content_set_id = ?\n\t\t\t\tAND project_id = ?\n\t\t\t\tAND version_id = ?\n\t\t\tORDER BY modified_at DESC\n\t\t\tLIMIT 1\n\t\t\t", - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 3 - }, - "nullable": [ - false - ] - }, - "hash": "636bb0c426ac275d64f7c84b3349894d10faf920d56814c0c2336e4cd5002743" -} diff --git a/packages/app-lib/.sqlx/query-6a4dd5b455a6aa19bb866faa915b8bfd4b7aca017e919d6240d7e1e2fc412e5c.json b/packages/app-lib/.sqlx/query-6a4dd5b455a6aa19bb866faa915b8bfd4b7aca017e919d6240d7e1e2fc412e5c.json new file mode 100644 index 0000000000..2b7fab3068 --- /dev/null +++ b/packages/app-lib/.sqlx/query-6a4dd5b455a6aa19bb866faa915b8bfd4b7aca017e919d6240d7e1e2fc412e5c.json @@ -0,0 +1,92 @@ +{ + "db_name": "SQLite", + "query": "\n\t\t\tINSERT INTO instance_content_entries (\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t)\n\t\t\tVALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\t\tRETURNING\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "instance_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "content_set_id", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "file_id", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "project_type", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "project_id", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "version_id", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "source_kind", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "server_requirement", + "ordinal": 8, + "type_info": "Text" + }, + { + "name": "client_requirement", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "enabled", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "added_at", + "ordinal": 11, + "type_info": "Integer" + }, + { + "name": "modified_at", + "ordinal": 12, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 12 + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "6a4dd5b455a6aa19bb866faa915b8bfd4b7aca017e919d6240d7e1e2fc412e5c" +} diff --git a/packages/app-lib/.sqlx/query-6a6e05b5c64a7061046383f69e07bc3f24cbde279858f3f440a78089b696a718.json b/packages/app-lib/.sqlx/query-6a6e05b5c64a7061046383f69e07bc3f24cbde279858f3f440a78089b696a718.json deleted file mode 100644 index 6db99f6781..0000000000 --- a/packages/app-lib/.sqlx/query-6a6e05b5c64a7061046383f69e07bc3f24cbde279858f3f440a78089b696a718.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n\t\t\tSELECT id\n\t\t\tFROM instance_content_entries\n\t\t\tWHERE content_set_id = ? AND file_id = ?\n\t\t\tORDER BY modified_at DESC\n\t\t\tLIMIT 1\n\t\t\t", - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - false - ] - }, - "hash": "6a6e05b5c64a7061046383f69e07bc3f24cbde279858f3f440a78089b696a718" -} diff --git a/packages/app-lib/.sqlx/query-6d76a6f26705044500983aaae9796a39ac667278b0c805da2320384a37c3baa1.json b/packages/app-lib/.sqlx/query-6d76a6f26705044500983aaae9796a39ac667278b0c805da2320384a37c3baa1.json new file mode 100644 index 0000000000..b7ed62a7d7 --- /dev/null +++ b/packages/app-lib/.sqlx/query-6d76a6f26705044500983aaae9796a39ac667278b0c805da2320384a37c3baa1.json @@ -0,0 +1,74 @@ +{ + "db_name": "SQLite", + "query": "\n\t\tUPDATE instance_files\n\t\tSET\n\t\t\tmissing = ?,\n\t\t\tmodified_at = ?\n\t\tWHERE id = ?\n\t\tRETURNING\n\t\t\tid,\n\t\t\tinstance_id,\n\t\t\trelative_path,\n\t\t\tfile_name,\n\t\t\tenabled,\n\t\t\tsha1,\n\t\t\tsize,\n\t\t\tmissing,\n\t\t\tadded_at,\n\t\t\tmodified_at\n\t\t", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "instance_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "relative_path", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "file_name", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "enabled", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "sha1", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "size", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "missing", + "ordinal": 7, + "type_info": "Integer" + }, + { + "name": "added_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "modified_at", + "ordinal": 9, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "6d76a6f26705044500983aaae9796a39ac667278b0c805da2320384a37c3baa1" +} diff --git a/packages/app-lib/.sqlx/query-81e6135eeba87a5f2084e0747b4675ff87456469e98c07520023981e3b3d121a.json b/packages/app-lib/.sqlx/query-81e6135eeba87a5f2084e0747b4675ff87456469e98c07520023981e3b3d121a.json deleted file mode 100644 index 080c770311..0000000000 --- a/packages/app-lib/.sqlx/query-81e6135eeba87a5f2084e0747b4675ff87456469e98c07520023981e3b3d121a.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n\t\tINSERT INTO instance_content_entries (\n\t\t\tid,\n\t\t\tinstance_id,\n\t\t\tcontent_set_id,\n\t\t\tfile_id,\n\t\t\tproject_type,\n\t\t\tproject_id,\n\t\t\tversion_id,\n\t\t\tsource_kind,\n\t\t\tserver_requirement,\n\t\t\tclient_requirement,\n\t\t\tenabled,\n\t\t\tadded_at,\n\t\t\tmodified_at\n\t\t)\n\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON CONFLICT(id) DO UPDATE SET\n\t\t\tfile_id = excluded.file_id,\n\t\t\tproject_type = excluded.project_type,\n\t\t\tproject_id = excluded.project_id,\n\t\t\tversion_id = excluded.version_id,\n\t\t\tsource_kind = excluded.source_kind,\n\t\t\tserver_requirement = excluded.server_requirement,\n\t\t\tclient_requirement = excluded.client_requirement,\n\t\t\tenabled = excluded.enabled,\n\t\t\tmodified_at = excluded.modified_at\n\t\t", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - }, - "hash": "81e6135eeba87a5f2084e0747b4675ff87456469e98c07520023981e3b3d121a" -} diff --git a/packages/app-lib/.sqlx/query-8c54b73c00cfc43b47ca08cca4dbdc381afa80c2f493d04c8a54875619386caf.json b/packages/app-lib/.sqlx/query-8c54b73c00cfc43b47ca08cca4dbdc381afa80c2f493d04c8a54875619386caf.json deleted file mode 100644 index d1311adfd6..0000000000 --- a/packages/app-lib/.sqlx/query-8c54b73c00cfc43b47ca08cca4dbdc381afa80c2f493d04c8a54875619386caf.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n\t\tUPDATE instance_files\n\t\tSET\n\t\t\tmissing = 1,\n\t\t\tmodified_at = ?\n\t\tWHERE instance_id = ?\n\t\t", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "8c54b73c00cfc43b47ca08cca4dbdc381afa80c2f493d04c8a54875619386caf" -} diff --git a/packages/app-lib/.sqlx/query-8daa2109fa7affe35b94d1fe3823ffe7601c0bca00db4ea0c8dc709bcf5642eb.json b/packages/app-lib/.sqlx/query-8daa2109fa7affe35b94d1fe3823ffe7601c0bca00db4ea0c8dc709bcf5642eb.json new file mode 100644 index 0000000000..984d0eb9dc --- /dev/null +++ b/packages/app-lib/.sqlx/query-8daa2109fa7affe35b94d1fe3823ffe7601c0bca00db4ea0c8dc709bcf5642eb.json @@ -0,0 +1,92 @@ +{ + "db_name": "SQLite", + "query": "\n\t\t\tINSERT INTO instance_content_entries (\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t)\n\t\t\tVALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\t\tON CONFLICT (content_set_id, project_id, version_id)\n\t\t\t\tWHERE file_id IS NULL\n\t\t\t\t\tAND project_id IS NOT NULL\n\t\t\t\t\tAND version_id IS NOT NULL\n\t\t\tDO UPDATE SET\n\t\t\t\tinstance_id = excluded.instance_id,\n\t\t\t\tproject_type = excluded.project_type,\n\t\t\t\tsource_kind = excluded.source_kind,\n\t\t\t\tserver_requirement = excluded.server_requirement,\n\t\t\t\tclient_requirement = excluded.client_requirement,\n\t\t\t\tenabled = excluded.enabled,\n\t\t\t\tmodified_at = excluded.modified_at\n\t\t\tRETURNING\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "instance_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "content_set_id", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "file_id", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "project_type", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "project_id", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "version_id", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "source_kind", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "server_requirement", + "ordinal": 8, + "type_info": "Text" + }, + { + "name": "client_requirement", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "enabled", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "added_at", + "ordinal": 11, + "type_info": "Integer" + }, + { + "name": "modified_at", + "ordinal": 12, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 12 + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "8daa2109fa7affe35b94d1fe3823ffe7601c0bca00db4ea0c8dc709bcf5642eb" +} diff --git a/packages/app-lib/.sqlx/query-aabe81dce68b38d0e5a9748da0df4f0f982f22861d13a848b09538b3ef594847.json b/packages/app-lib/.sqlx/query-aabe81dce68b38d0e5a9748da0df4f0f982f22861d13a848b09538b3ef594847.json deleted file mode 100644 index 47a307f9f7..0000000000 --- a/packages/app-lib/.sqlx/query-aabe81dce68b38d0e5a9748da0df4f0f982f22861d13a848b09538b3ef594847.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n\t\tINSERT INTO instance_files (\n\t\t\tid,\n\t\t\tinstance_id,\n\t\t\trelative_path,\n\t\t\tfile_name,\n\t\t\tenabled,\n\t\t\tsha1,\n\t\t\tsize,\n\t\t\tmissing,\n\t\t\tadded_at,\n\t\t\tmodified_at\n\t\t)\n\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON CONFLICT (instance_id, relative_path) DO UPDATE SET\n\t\t\tfile_name = excluded.file_name,\n\t\t\tenabled = excluded.enabled,\n\t\t\tsha1 = excluded.sha1,\n\t\t\tsize = excluded.size,\n\t\t\tmissing = excluded.missing,\n\t\t\tmodified_at = excluded.modified_at\n\t\t", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - }, - "hash": "aabe81dce68b38d0e5a9748da0df4f0f982f22861d13a848b09538b3ef594847" -} diff --git a/packages/app-lib/.sqlx/query-fbdf27acfedbf52fe79ef2db7cf29b5cd8b86308ef1202b3f06570bbb6e1419e.json b/packages/app-lib/.sqlx/query-fbdf27acfedbf52fe79ef2db7cf29b5cd8b86308ef1202b3f06570bbb6e1419e.json new file mode 100644 index 0000000000..879aa47145 --- /dev/null +++ b/packages/app-lib/.sqlx/query-fbdf27acfedbf52fe79ef2db7cf29b5cd8b86308ef1202b3f06570bbb6e1419e.json @@ -0,0 +1,92 @@ +{ + "db_name": "SQLite", + "query": "\n\t\t\tINSERT INTO instance_content_entries (\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t)\n\t\t\tVALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\t\tON CONFLICT (content_set_id, file_id)\n\t\t\t\tWHERE file_id IS NOT NULL\n\t\t\tDO UPDATE SET\n\t\t\t\tinstance_id = excluded.instance_id,\n\t\t\t\tproject_type = excluded.project_type,\n\t\t\t\tproject_id = excluded.project_id,\n\t\t\t\tversion_id = excluded.version_id,\n\t\t\t\tsource_kind = excluded.source_kind,\n\t\t\t\tserver_requirement = excluded.server_requirement,\n\t\t\t\tclient_requirement = excluded.client_requirement,\n\t\t\t\tenabled = excluded.enabled,\n\t\t\t\tmodified_at = excluded.modified_at\n\t\t\tRETURNING\n\t\t\t\tid,\n\t\t\t\tinstance_id,\n\t\t\t\tcontent_set_id,\n\t\t\t\tfile_id,\n\t\t\t\tproject_type,\n\t\t\t\tproject_id,\n\t\t\t\tversion_id,\n\t\t\t\tsource_kind,\n\t\t\t\tserver_requirement,\n\t\t\t\tclient_requirement,\n\t\t\t\tenabled,\n\t\t\t\tadded_at,\n\t\t\t\tmodified_at\n\t\t\t", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "instance_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "content_set_id", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "file_id", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "project_type", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "project_id", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "version_id", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "source_kind", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "server_requirement", + "ordinal": 8, + "type_info": "Text" + }, + { + "name": "client_requirement", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "enabled", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "added_at", + "ordinal": 11, + "type_info": "Integer" + }, + { + "name": "modified_at", + "ordinal": 12, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 13 + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "fbdf27acfedbf52fe79ef2db7cf29b5cd8b86308ef1202b3f06570bbb6e1419e" +} diff --git a/packages/app-lib/migrations/20260722120000_unique-content-entry-identities.sql b/packages/app-lib/migrations/20260722120000_unique-content-entry-identities.sql new file mode 100644 index 0000000000..0c086e3d7e --- /dev/null +++ b/packages/app-lib/migrations/20260722120000_unique-content-entry-identities.sql @@ -0,0 +1,46 @@ +-- Older app versions could save the same content entry more than once. Remove the older copies before preventing future duplicates. +WITH ranked_entries AS ( + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY content_set_id, file_id + ORDER BY modified_at DESC, added_at DESC, id DESC + ) AS duplicate_rank + FROM instance_content_entries + WHERE file_id IS NOT NULL +) +DELETE FROM instance_content_entries +WHERE id IN ( + SELECT id + FROM ranked_entries + WHERE duplicate_rank > 1 +); + +WITH ranked_entries AS ( + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY content_set_id, project_id, version_id + ORDER BY modified_at DESC, added_at DESC, id DESC + ) AS duplicate_rank + FROM instance_content_entries + WHERE file_id IS NULL + AND project_id IS NOT NULL + AND version_id IS NOT NULL +) +DELETE FROM instance_content_entries +WHERE id IN ( + SELECT id + FROM ranked_entries + WHERE duplicate_rank > 1 +); + +CREATE UNIQUE INDEX instance_content_entries_content_set_file_unique + ON instance_content_entries(content_set_id, file_id) + WHERE file_id IS NOT NULL; + +CREATE UNIQUE INDEX instance_content_entries_content_set_project_version_unique + ON instance_content_entries(content_set_id, project_id, version_id) + WHERE file_id IS NULL + AND project_id IS NOT NULL + AND version_id IS NOT NULL; diff --git a/packages/app-lib/src/api/pack/install_mrpack.rs b/packages/app-lib/src/api/pack/install_mrpack.rs index 9eea91b162..52946bd120 100644 --- a/packages/app-lib/src/api/pack/install_mrpack.rs +++ b/packages/app-lib/src/api/pack/install_mrpack.rs @@ -798,6 +798,16 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( let downloaded_bytes = file.len() as u64; let path = target_path; + content_context + .reporter + .preserve_failure_context( + context.clone(), + write(&path, &file, &state.io_semaphore).await, + ) + .await?; + let modified_at_ns = crate::state::file_modified_at_ns( + &io::metadata(&path).await?, + )?; { let _permit = state.install_db_semaphore.acquire().await?; @@ -809,6 +819,7 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( file.clone(), &content_context.instance_path, project.path.as_str(), + modified_at_ns, project .hashes .get(&PackFileHash::Sha1) @@ -822,14 +833,6 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( .await?; } - content_context - .reporter - .preserve_failure_context( - context.clone(), - write(&path, &file, &state.io_semaphore).await, - ) - .await?; - if let Some(project_type) = ProjectType::get_from_parent_folder(project.path.as_str()) { @@ -998,6 +1001,8 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( let (size, hash) = reporter .preserve_failure_context(override_context, extract_result) .await?; + let modified_at_ns = + crate::state::file_modified_at_ns(&io::metadata(&path).await?)?; { let _permit = state.install_db_semaphore.acquire().await?; @@ -1016,6 +1021,7 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( &instance_path, relative_override_file_path.as_str(), size, + modified_at_ns, hash.clone(), ProjectType::get_from_parent_folder( relative_override_file_path.as_str(), diff --git a/packages/app-lib/src/install/runner.rs b/packages/app-lib/src/install/runner.rs index 4dee0cdaeb..36515a0535 100644 --- a/packages/app-lib/src/install/runner.rs +++ b/packages/app-lib/src/install/runner.rs @@ -785,6 +785,7 @@ async fn remove_existing_imported_pack_content( metadata: &crate::state::InstanceMetadata, state: &State, ) -> crate::Result<()> { + let _content_lock = state.lock_instance_content(instance_id).await; let entries = content_rows::get_content_entries( &metadata.applied_content_set.id, &state.pool, @@ -821,18 +822,20 @@ async fn remove_existing_imported_pack_content( continue; }; crate::util::io::remove_file(base.join(&file.relative_path)).await?; + let mut tx = state.pool.begin().await?; content_rows::remove_content_entries_for_file( &metadata.applied_content_set.id, &file.id, - &state.pool, + &mut tx, ) .await?; content_rows::remove_instance_file_by_relative_path( instance_id, &file.relative_path, - &state.pool, + &mut tx, ) .await?; + tx.commit().await?; } Ok(()) diff --git a/packages/app-lib/src/state/cache.rs b/packages/app-lib/src/state/cache.rs index 74a78c794d..094dd98d93 100644 --- a/packages/app-lib/src/state/cache.rs +++ b/packages/app-lib/src/state/cache.rs @@ -373,6 +373,8 @@ impl<'de> serde::Deserialize<'de> for CachedFileUpdate { pub struct CachedFileHash { pub path: String, pub size: u64, + #[serde(default)] + pub modified_at_ns: u64, pub hash: String, pub project_type: Option, #[serde(default)] @@ -387,6 +389,37 @@ pub struct KnownModrinthFile<'a> { pub version_id: &'a str, } +pub(crate) fn file_modified_at_ns( + metadata: &std::fs::Metadata, +) -> std::io::Result { + let elapsed = metadata + .modified()? + .duration_since(std::time::UNIX_EPOCH) + .map_err(std::io::Error::other)?; + + Ok(elapsed + .as_secs() + .saturating_mul(1_000_000_000) + .saturating_add(u64::from(elapsed.subsec_nanos()))) +} + +pub(crate) fn file_hash_cache_key( + size: u64, + modified_at_ns: u64, + path: &str, +) -> String { + format!("v2-{size}-{modified_at_ns}-{path}") +} + +fn file_hash_path_from_cache_key(key: &str) -> Option<&str> { + let mut parts = key.splitn(4, '-'); + (parts.next()? == "v2").then_some(())?; + parts.next()?.parse::().ok()?; + parts.next()?.parse::().ok()?; + let path = parts.next()?; + (!path.is_empty()).then_some(path) +} + #[derive(Serialize, Deserialize, Clone, Debug)] pub struct CachedLoaderManifest { pub loader: String, @@ -674,13 +707,11 @@ impl CacheValue { | CacheValue::GameVersions(_) | CacheValue::DonationPlatforms(_) => DEFAULT_ID.to_string(), - CacheValue::FileHash(hash) => { - format!( - "{}-{}", - hash.size, - hash.path.trim_end_matches(".disabled") - ) - } + CacheValue::FileHash(hash) => file_hash_cache_key( + hash.size, + hash.modified_at_ns, + hash.path.trim_end_matches(".disabled"), + ), CacheValue::FileUpdate(hash) => { format!( "{}-{}-{}-{}", @@ -1518,13 +1549,20 @@ impl CachedEntry { instances_dir: &Path, key: String, ) -> crate::Result<(CachedEntry, bool)> { - let path = - key.split_once('-').map(|x| x.1).unwrap_or_default(); + let path = file_hash_path_from_cache_key(&key).ok_or_else( + || { + crate::ErrorKind::InputError(format!( + "Invalid file hash cache key: {key}", + )) + }, + )?; let full_path = instances_dir.join(path); let mut file = tokio::fs::File::open(&full_path).await?; - let size = file.metadata().await?.len(); + let metadata = file.metadata().await?; + let size = metadata.len(); + let modified_at_ns = file_modified_at_ns(&metadata)?; let mut hasher = sha1_smol::Sha1::new(); @@ -1544,6 +1582,7 @@ impl CachedEntry { CacheValue::FileHash(CachedFileHash { path: path.to_string(), size, + modified_at_ns, hash, project_type: ProjectType::get_from_parent_folder( &full_path, @@ -2139,6 +2178,7 @@ pub async fn cache_file_hash( bytes: bytes::Bytes, instance_id: &str, path: &str, + modified_at_ns: u64, known_hash: Option<&str>, project_type: Option, known_modrinth_file: Option>, @@ -2156,6 +2196,7 @@ pub async fn cache_file_hash( instance_id, path, size as u64, + modified_at_ns, hash, project_type, known_modrinth_file, @@ -2168,6 +2209,7 @@ pub async fn cache_file_hash_metadata( instance_id: &str, path: &str, size: u64, + modified_at_ns: u64, hash: String, project_type: Option, known_modrinth_file: Option>, @@ -2186,6 +2228,7 @@ pub async fn cache_file_hash_metadata( &[CacheValue::FileHash(CachedFileHash { path: format!("{instance_id}/{path}"), size, + modified_at_ns, hash, project_type, project_id, diff --git a/packages/app-lib/src/state/instances/adapters/filesystem.rs b/packages/app-lib/src/state/instances/adapters/filesystem.rs index df81f0c3a8..73c8ce4f5b 100644 --- a/packages/app-lib/src/state/instances/adapters/filesystem.rs +++ b/packages/app-lib/src/state/instances/adapters/filesystem.rs @@ -1,4 +1,4 @@ -use crate::state::ProjectType; +use crate::state::{ProjectType, file_hash_cache_key, file_modified_at_ns}; use crate::util::io::{self, IOError}; use std::path::{Path, PathBuf}; @@ -44,17 +44,23 @@ pub(crate) fn scan_content_files( continue; } - let size = path.metadata().map_err(IOError::from)?.len(); + let metadata = path.metadata().map_err(IOError::from)?; + let size = metadata.len(); + let modified_at_ns = + file_modified_at_ns(&metadata).map_err(IOError::from)?; let relative_path = format!("{folder}/{file_name}"); + let hash_cache_key = file_hash_cache_key( + size, + modified_at_ns, + &format!("{instance_path}/{relative_path}"), + ); files.push(ScannedContentFile { relative_path, file_name: file_name.to_string(), enabled: !file_name.ends_with(".disabled"), size, - hash_cache_key: format!( - "{size}-{instance_path}/{folder}/{file_name}" - ), + hash_cache_key, }); } } diff --git a/packages/app-lib/src/state/instances/adapters/sqlite/content_rows.rs b/packages/app-lib/src/state/instances/adapters/sqlite/content_rows.rs index 0a27b105cd..74a58bf931 100644 --- a/packages/app-lib/src/state/instances/adapters/sqlite/content_rows.rs +++ b/packages/app-lib/src/state/instances/adapters/sqlite/content_rows.rs @@ -419,33 +419,47 @@ where rows.into_iter().map(TryInto::try_into).collect() } -pub(crate) async fn mark_instance_files_missing( - instance_id: &str, +pub(crate) async fn set_instance_file_missing( + file_id: &str, + missing: bool, tx: &mut Transaction<'_, Sqlite>, -) -> crate::Result<()> { +) -> crate::Result> { + let missing = i64::from(missing); let modified_at = Utc::now().timestamp(); - - sqlx::query!( + let row = sqlx::query_as!( + InstanceFileRow, " UPDATE instance_files SET - missing = 1, + missing = ?, modified_at = ? - WHERE instance_id = ? + WHERE id = ? + RETURNING + id, + instance_id, + relative_path, + file_name, + enabled, + sha1, + size, + missing, + added_at, + modified_at ", + missing, modified_at, - instance_id, + file_id, ) - .execute(&mut **tx) + .fetch_optional(&mut **tx) .await?; - Ok(()) + row.map(TryInto::try_into).transpose() } pub(crate) async fn upsert_instance_file( file: &InstanceFile, tx: &mut Transaction<'_, Sqlite>, -) -> crate::Result<()> { +) -> crate::Result { let id = file.id.as_str(); let instance_id = file.instance_id.as_str(); let relative_path = file.relative_path.as_str(); @@ -457,7 +471,8 @@ pub(crate) async fn upsert_instance_file( let added_at = file.added_at.timestamp(); let modified_at = file.modified_at.timestamp(); - sqlx::query!( + let row = sqlx::query_as!( + InstanceFileRow, " INSERT INTO instance_files ( id, @@ -479,6 +494,17 @@ pub(crate) async fn upsert_instance_file( size = excluded.size, missing = excluded.missing, modified_at = excluded.modified_at + RETURNING + id, + instance_id, + relative_path, + file_name, + enabled, + sha1, + size, + missing, + added_at, + modified_at ", id, instance_id, @@ -491,10 +517,10 @@ pub(crate) async fn upsert_instance_file( added_at, modified_at, ) - .execute(&mut **tx) + .fetch_one(&mut **tx) .await?; - Ok(()) + row.try_into() } pub(crate) async fn get_content_entries<'e, E>( @@ -575,19 +601,11 @@ pub(crate) async fn get_instance_file_by_relative_path( pub(crate) async fn upsert_instance_file_from_parts( input: UpsertInstanceFile<'_>, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result { - let existing = get_instance_file_by_relative_path( - input.instance_id, - input.relative_path, - pool, - ) - .await?; + let now = Utc::now(); let file = InstanceFile { - id: existing - .as_ref() - .map(|file| file.id.clone()) - .unwrap_or_else(|| format!("instance-file:{}", Uuid::new_v4())), + id: format!("instance-file:{}", Uuid::new_v4()), instance_id: input.instance_id.to_string(), relative_path: input.relative_path.to_string(), file_name: input.file_name.to_string(), @@ -595,18 +613,11 @@ pub(crate) async fn upsert_instance_file_from_parts( sha1: input.sha1.to_string(), size: input.size, missing: input.missing, - added_at: existing - .as_ref() - .map(|file| file.added_at) - .unwrap_or_else(Utc::now), - modified_at: Utc::now(), + added_at: now, + modified_at: now, }; - let mut tx = pool.begin().await?; - upsert_instance_file(&file, &mut tx).await?; - tx.commit().await?; - - Ok(file) + upsert_instance_file(&file, tx).await } pub(crate) async fn rename_instance_file( @@ -615,11 +626,10 @@ pub(crate) async fn rename_instance_file( new_relative_path: &str, new_file_name: &str, enabled: bool, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result> { let enabled = i64::from(enabled); let modified_at = Utc::now().timestamp(); - let mut tx = pool.begin().await?; let source_id = sqlx::query_scalar!( " @@ -630,7 +640,7 @@ pub(crate) async fn rename_instance_file( instance_id, old_relative_path, ) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await?; let target_id = sqlx::query_scalar!( " @@ -641,7 +651,7 @@ pub(crate) async fn rename_instance_file( instance_id, new_relative_path, ) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await?; if let (Some(source_id), Some(target_id)) = @@ -669,7 +679,7 @@ pub(crate) async fn rename_instance_file( target_id, source_id, ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; sqlx::query!( @@ -683,7 +693,7 @@ pub(crate) async fn rename_instance_file( instance_id, target_id, ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; sqlx::query!( @@ -693,7 +703,7 @@ pub(crate) async fn rename_instance_file( ", target_id, ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } @@ -715,19 +725,29 @@ pub(crate) async fn rename_instance_file( instance_id, old_relative_path, ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - tx.commit().await?; + let row = sqlx::query_as!( + InstanceFileRow, + " + SELECT * + FROM instance_files + WHERE instance_id = ? AND relative_path = ? + ", + instance_id, + new_relative_path, + ) + .fetch_optional(&mut **tx) + .await?; - get_instance_file_by_relative_path(instance_id, new_relative_path, pool) - .await + row.map(TryInto::try_into).transpose() } pub(crate) async fn remove_instance_file_by_relative_path( instance_id: &str, relative_path: &str, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result<()> { sqlx::query!( " @@ -737,7 +757,7 @@ pub(crate) async fn remove_instance_file_by_relative_path( instance_id, relative_path, ) - .execute(pool) + .execute(&mut **tx) .await?; Ok(()) @@ -800,142 +820,202 @@ pub(crate) async fn get_content_entry_by_file( pub(crate) async fn upsert_content_entry_from_parts( input: UpsertContentEntry<'_>, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result { - let existing_id = if let Some(file_id) = input.file_id { - sqlx::query_scalar!( + let id = format!("content-entry:{}", Uuid::new_v4()); + let project_type = input.project_type.get_name(); + let source_kind = input.source_kind.as_str(); + let server_requirement = input.server_requirement.as_str(); + let client_requirement = input.client_requirement.as_str(); + let enabled = i64::from(input.enabled); + let now = Utc::now().timestamp(); + let row = if let Some(file_id) = input.file_id { + sqlx::query_as!( + ContentEntryRow, " - SELECT id - FROM instance_content_entries - WHERE content_set_id = ? AND file_id = ? - ORDER BY modified_at DESC - LIMIT 1 + INSERT INTO instance_content_entries ( + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (content_set_id, file_id) + WHERE file_id IS NOT NULL + DO UPDATE SET + instance_id = excluded.instance_id, + project_type = excluded.project_type, + project_id = excluded.project_id, + version_id = excluded.version_id, + source_kind = excluded.source_kind, + server_requirement = excluded.server_requirement, + client_requirement = excluded.client_requirement, + enabled = excluded.enabled, + modified_at = excluded.modified_at + RETURNING + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at ", + id, + input.instance_id, input.content_set_id, file_id, + project_type, + input.project_id, + input.version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + now, + now, ) - .fetch_optional(pool) + .fetch_one(&mut **tx) .await? } else if let (Some(project_id), Some(version_id)) = (input.project_id, input.version_id) { - sqlx::query_scalar!( + sqlx::query_as!( + ContentEntryRow, " - SELECT id - FROM instance_content_entries - WHERE content_set_id = ? - AND project_id = ? - AND version_id = ? - ORDER BY modified_at DESC - LIMIT 1 + INSERT INTO instance_content_entries ( + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at + ) + VALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (content_set_id, project_id, version_id) + WHERE file_id IS NULL + AND project_id IS NOT NULL + AND version_id IS NOT NULL + DO UPDATE SET + instance_id = excluded.instance_id, + project_type = excluded.project_type, + source_kind = excluded.source_kind, + server_requirement = excluded.server_requirement, + client_requirement = excluded.client_requirement, + enabled = excluded.enabled, + modified_at = excluded.modified_at + RETURNING + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at ", + id, + input.instance_id, input.content_set_id, + project_type, project_id, version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + now, + now, ) - .fetch_optional(pool) + .fetch_one(&mut **tx) .await? } else { - None - }; - let now = Utc::now(); - let entry = ContentEntry { - id: existing_id - .unwrap_or_else(|| format!("content-entry:{}", Uuid::new_v4())), - instance_id: input.instance_id.to_string(), - content_set_id: input.content_set_id.to_string(), - file_id: input.file_id.map(ToString::to_string), - project_type: input.project_type, - project_id: input.project_id.map(ToString::to_string), - version_id: input.version_id.map(ToString::to_string), - source_kind: input.source_kind, - server_requirement: input.server_requirement, - client_requirement: input.client_requirement, - enabled: input.enabled, - added_at: now, - modified_at: now, + sqlx::query_as!( + ContentEntryRow, + " + INSERT INTO instance_content_entries ( + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at + ) + VALUES (?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?) + RETURNING + id, + instance_id, + content_set_id, + file_id, + project_type, + project_id, + version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + added_at, + modified_at + ", + id, + input.instance_id, + input.content_set_id, + project_type, + input.project_id, + input.version_id, + source_kind, + server_requirement, + client_requirement, + enabled, + now, + now, + ) + .fetch_one(&mut **tx) + .await? }; - let added_at = get_content_entry_by_id(&entry.id, pool) - .await? - .map(|entry| entry.added_at) - .unwrap_or(entry.added_at); - let id = entry.id.as_str(); - let entry_instance_id = entry.instance_id.as_str(); - let content_set_id = entry.content_set_id.as_str(); - let file_id = entry.file_id.as_deref(); - let project_type = entry.project_type.get_name(); - let project_id = entry.project_id.as_deref(); - let version_id = entry.version_id.as_deref(); - let source_kind = entry.source_kind.as_str(); - let server_requirement = entry.server_requirement.as_str(); - let client_requirement = entry.client_requirement.as_str(); - let enabled = i64::from(entry.enabled); - let added_at = added_at.timestamp(); - let modified_at = entry.modified_at.timestamp(); - - sqlx::query!( - " - INSERT INTO instance_content_entries ( - id, - instance_id, - content_set_id, - file_id, - project_type, - project_id, - version_id, - source_kind, - server_requirement, - client_requirement, - enabled, - added_at, - modified_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - file_id = excluded.file_id, - project_type = excluded.project_type, - project_id = excluded.project_id, - version_id = excluded.version_id, - source_kind = excluded.source_kind, - server_requirement = excluded.server_requirement, - client_requirement = excluded.client_requirement, - enabled = excluded.enabled, - modified_at = excluded.modified_at - ", - id, - entry_instance_id, - content_set_id, - file_id, - project_type, - project_id, - version_id, - source_kind, - server_requirement, - client_requirement, - enabled, - added_at, - modified_at, - ) - .execute(pool) - .await?; - - get_content_entry_by_id(&entry.id, pool) - .await? - .ok_or_else(|| { - crate::ErrorKind::OtherError(format!( - "Failed to read content entry {} after upsert", - entry.id - )) - .into() - }) + row.try_into() } pub(crate) async fn set_content_entry_enabled_for_file( content_set_id: &str, file_id: &str, enabled: bool, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result { let enabled = i64::from(enabled); let modified_at = Utc::now().timestamp(); @@ -951,7 +1031,7 @@ pub(crate) async fn set_content_entry_enabled_for_file( content_set_id, file_id, ) - .execute(pool) + .execute(&mut **tx) .await?; Ok(result.rows_affected() > 0) @@ -960,7 +1040,7 @@ pub(crate) async fn set_content_entry_enabled_for_file( pub(crate) async fn remove_content_entries_for_file( content_set_id: &str, file_id: &str, - pool: &SqlitePool, + tx: &mut Transaction<'_, Sqlite>, ) -> crate::Result<()> { sqlx::query!( " @@ -970,7 +1050,7 @@ pub(crate) async fn remove_content_entries_for_file( content_set_id, file_id, ) - .execute(pool) + .execute(&mut **tx) .await?; Ok(()) diff --git a/packages/app-lib/src/state/instances/commands/apply_content_install.rs b/packages/app-lib/src/state/instances/commands/apply_content_install.rs index 0491bcbb7e..49536fb69b 100644 --- a/packages/app-lib/src/state/instances/commands/apply_content_install.rs +++ b/packages/app-lib/src/state/instances/commands/apply_content_install.rs @@ -502,6 +502,7 @@ pub(crate) async fn add_project_bytes( version_id: Option<&str>, state: &State, ) -> crate::Result { + let _content_lock = state.lock_instance_content(instance_id).await; let scope = resolve_content_scope(instance_id, None, state).await?; let project_type = match project_type { Some(project_type) => project_type, @@ -515,10 +516,14 @@ pub(crate) async fn add_project_bytes( None => fetch::sha1_async(bytes.clone()).await?, }; + fetch::write(&full_path, &bytes, &state.io_semaphore).await?; + let modified_at_ns = + crate::state::file_modified_at_ns(&io::metadata(&full_path).await?)?; cache_file_hash( bytes.clone(), &scope.instance.id, &relative_path, + modified_at_ns, Some(&sha1), Some(project_type), project_id.zip(version_id).map(|(project_id, version_id)| { @@ -530,8 +535,8 @@ pub(crate) async fn add_project_bytes( &state.pool, ) .await?; - fetch::write(&full_path, &bytes, &state.io_semaphore).await?; + let mut tx = state.pool.begin().await?; let file = content_rows::upsert_instance_file_from_parts( content_rows::UpsertInstanceFile { instance_id: &scope.instance.id, @@ -542,7 +547,7 @@ pub(crate) async fn add_project_bytes( size: bytes.len() as u64, missing: false, }, - &state.pool, + &mut tx, ) .await?; upsert_entry_for_file( @@ -552,9 +557,10 @@ pub(crate) async fn add_project_bytes( project_id, version_id, source_kind, - &state.pool, + &mut tx, ) .await?; + tx.commit().await?; Ok(relative_path) } @@ -570,12 +576,14 @@ pub(crate) async fn record_project_file( version_id: Option<&str>, state: &State, ) -> crate::Result<()> { + let _content_lock = state.lock_instance_content(instance_id).await; let scope = resolve_content_scope(instance_id, None, state).await?; let file_name = Path::new(relative_path) .file_name() .unwrap_or_default() .to_string_lossy() .to_string(); + let mut tx = state.pool.begin().await?; let file = content_rows::upsert_instance_file_from_parts( content_rows::UpsertInstanceFile { instance_id: &scope.instance.id, @@ -586,7 +594,7 @@ pub(crate) async fn record_project_file( size, missing: false, }, - &state.pool, + &mut tx, ) .await?; upsert_entry_for_file( @@ -596,9 +604,12 @@ pub(crate) async fn record_project_file( project_id, version_id, source_kind, - &state.pool, + &mut tx, ) - .await + .await?; + tx.commit().await?; + + Ok(()) } pub(crate) async fn toggle_disable_project( @@ -607,6 +618,7 @@ pub(crate) async fn toggle_disable_project( desired_enabled: Option, state: &State, ) -> crate::Result { + let _content_lock = state.lock_instance_content(instance_id).await; let scope = resolve_content_scope(instance_id, None, state).await?; let base = instance_full_path(state, &scope.instance); let trimmed = project_path.trim_end_matches(".disabled"); @@ -640,13 +652,14 @@ pub(crate) async fn toggle_disable_project( .unwrap_or_default() .to_string_lossy() .to_string(); + let mut tx = state.pool.begin().await?; let file = match content_rows::rename_instance_file( &scope.instance.id, ¤t_path, &new_path, &file_name, enabled, - &state.pool, + &mut tx, ) .await? { @@ -658,21 +671,24 @@ pub(crate) async fn toggle_disable_project( &new_path, &file_name, enabled, - &state.pool, + &mut tx, ) .await? { Some(file) => file, - None => index_existing_file(&scope, &new_path, state).await?, + None => { + index_existing_file(&scope, &new_path, state, &mut tx) + .await? + } } } - None => index_existing_file(&scope, &new_path, state).await?, + None => index_existing_file(&scope, &new_path, state, &mut tx).await?, }; let updated_entry = content_rows::set_content_entry_enabled_for_file( &scope.content_set_id, &file.id, enabled, - &state.pool, + &mut tx, ) .await?; if !updated_entry { @@ -689,10 +705,11 @@ pub(crate) async fn toggle_disable_project( None, None, ContentSourceKind::Local, - &state.pool, + &mut tx, ) .await?; } + tx.commit().await?; Ok(new_path) } @@ -702,6 +719,7 @@ pub(crate) async fn remove_project( project_path: &str, state: &State, ) -> crate::Result<()> { + let _content_lock = state.lock_instance_content(instance_id).await; let scope = resolve_content_scope(instance_id, None, state).await?; let base = instance_full_path(state, &scope.instance); let file = content_rows::get_instance_file_by_relative_path( @@ -718,18 +736,20 @@ pub(crate) async fn remove_project( } if let Some(file) = file { + let mut tx = state.pool.begin().await?; content_rows::remove_content_entries_for_file( &scope.content_set_id, &file.id, - &state.pool, + &mut tx, ) .await?; content_rows::remove_instance_file_by_relative_path( &scope.instance.id, project_path, - &state.pool, + &mut tx, ) .await?; + tx.commit().await?; } Ok(()) @@ -810,6 +830,7 @@ async fn index_existing_file( scope: &ContentScope, relative_path: &str, state: &State, + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, ) -> crate::Result { let full_path = instance_full_path(state, &scope.instance).join(relative_path); @@ -836,7 +857,7 @@ async fn index_existing_file( size, missing: false, }, - &state.pool, + tx, ) .await?; upsert_entry_for_file( @@ -846,7 +867,7 @@ async fn index_existing_file( None, None, ContentSourceKind::Local, - &state.pool, + tx, ) .await?; @@ -860,7 +881,7 @@ async fn upsert_entry_for_file( project_id: Option<&str>, version_id: Option<&str>, source_kind: ContentSourceKind, - pool: &sqlx::SqlitePool, + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, ) -> crate::Result<()> { content_rows::upsert_content_entry_from_parts( content_rows::UpsertContentEntry { @@ -875,7 +896,7 @@ async fn upsert_entry_for_file( client_requirement: ContentRequirement::Required, enabled: file.enabled, }, - pool, + tx, ) .await?; diff --git a/packages/app-lib/src/state/instances/commands/remove_instance.rs b/packages/app-lib/src/state/instances/commands/remove_instance.rs index bb27128ddb..60ac2f368c 100644 --- a/packages/app-lib/src/state/instances/commands/remove_instance.rs +++ b/packages/app-lib/src/state/instances/commands/remove_instance.rs @@ -11,8 +11,9 @@ pub(crate) async fn remove_instance( .ok_or_else(|| { crate::ErrorKind::InputError("Unknown instance".to_string()) })?; + let _content_lock = state.lock_instance_content(instance_id).await; - instance_rows::delete_instance_by_id(&instance.id, &state.pool).await?; + delete_instance_row_and_content_lock(&instance.id, state).await?; let path = state.directories.instances_dir().join(&instance.path); if path.exists() { @@ -21,3 +22,14 @@ pub(crate) async fn remove_instance( Ok(()) } + +async fn delete_instance_row_and_content_lock( + instance_id: &str, + state: &State, +) -> crate::Result<()> { + // Keep these together so deleted instances cannot leave stale entries in the per-instance lock map. + instance_rows::delete_instance_by_id(instance_id, &state.pool).await?; + state.remove_instance_content_lock(instance_id); + + Ok(()) +} diff --git a/packages/app-lib/src/state/instances/commands/sync_content_files.rs b/packages/app-lib/src/state/instances/commands/sync_content_files.rs index 3686558393..79af5ab590 100644 --- a/packages/app-lib/src/state/instances/commands/sync_content_files.rs +++ b/packages/app-lib/src/state/instances/commands/sync_content_files.rs @@ -1,9 +1,9 @@ use crate::State; use crate::state::instances::adapters::{filesystem, sqlite}; use crate::state::instances::{Instance, InstanceFile}; -use crate::state::{CachedEntry, ProjectType}; +use crate::state::{CachedEntry, ProjectType, file_hash_cache_key}; use chrono::Utc; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use uuid::Uuid; pub(crate) async fn sync_content_files( @@ -24,6 +24,7 @@ pub(crate) async fn sync_instance_content_files( instance: &Instance, state: &State, ) -> crate::Result> { + let _content_lock = state.lock_instance_content(&instance.id).await; let scanned = filesystem::scan_content_files( &state.directories.instances_dir(), &instance.path, @@ -43,10 +44,10 @@ pub(crate) async fn sync_instance_content_files( .into_iter() .map(|hash| { ( - format!( - "{}-{}", + file_hash_cache_key( hash.size, - hash.path.trim_end_matches(".disabled") + hash.modified_at_ns, + hash.path.trim_end_matches(".disabled"), ), hash, ) @@ -55,6 +56,17 @@ pub(crate) async fn sync_instance_content_files( let existing_files = sqlite::content_rows::get_instance_files(&instance.id, &state.pool) .await?; + let scanned_paths = scanned + .iter() + .map(|file| file.relative_path.clone()) + .collect::>(); + let missing_file_ids = existing_files + .iter() + .filter(|file| { + !file.missing && !scanned_paths.contains(&file.relative_path) + }) + .map(|file| file.id.clone()) + .collect::>(); let existing_files_by_path = existing_files .into_iter() .map(|file| (file.relative_path.clone(), file)) @@ -62,13 +74,17 @@ pub(crate) async fn sync_instance_content_files( let now = Utc::now(); let mut files = Vec::new(); + let mut present_without_hash_ids = Vec::new(); for file in scanned { let hash_key = file.hash_cache_key.trim_end_matches(".disabled"); + let existing_file = existing_files_by_path.get(&file.relative_path); let Some(hash) = hashes_by_key.get(hash_key) else { + if let Some(existing_file) = existing_file { + present_without_hash_ids.push(existing_file.id.clone()); + } continue; }; - let existing_file = existing_files_by_path.get(&file.relative_path); files.push(InstanceFile { id: existing_file @@ -87,16 +103,33 @@ pub(crate) async fn sync_instance_content_files( } let mut tx = state.pool.begin().await?; - sqlite::content_rows::mark_instance_files_missing(&instance.id, &mut tx) + for file_id in missing_file_ids { + sqlite::content_rows::set_instance_file_missing( + &file_id, true, &mut tx, + ) .await?; + } + let mut stored_files = + Vec::with_capacity(files.len() + present_without_hash_ids.len()); + for file_id in present_without_hash_ids { + if let Some(file) = sqlite::content_rows::set_instance_file_missing( + &file_id, false, &mut tx, + ) + .await? + { + stored_files.push(file); + } + } for file in &files { - sqlite::content_rows::upsert_instance_file(file, &mut tx).await?; + stored_files.push( + sqlite::content_rows::upsert_instance_file(file, &mut tx).await?, + ); } tx.commit().await?; - Ok(files) + Ok(stored_files) } pub(crate) fn project_type_for_file( diff --git a/packages/app-lib/src/state/legacy_converter.rs b/packages/app-lib/src/state/legacy_converter.rs index b602a09b3a..a56981167a 100644 --- a/packages/app-lib/src/state/legacy_converter.rs +++ b/packages/app-lib/src/state/legacy_converter.rs @@ -213,7 +213,10 @@ where .find(|x| x.hashes.get("sha512") == Some(&sha512)) && let Some(sha1) = file.hashes.get("sha1") { - if let Ok(metadata) = full_path.metadata() { + if let Ok(metadata) = full_path.metadata() + && let Ok(modified_at_ns) = + state::file_modified_at_ns(&metadata) + { let file_name = format!( "{}/{}", profile.path, @@ -225,6 +228,7 @@ where CachedFileHash { path: file_name, size: metadata.len(), + modified_at_ns, hash: sha1.clone(), project_type: ProjectType::get_from_parent_folder( diff --git a/packages/app-lib/src/state/mod.rs b/packages/app-lib/src/state/mod.rs index 18fa7edbb8..d667f7a62b 100644 --- a/packages/app-lib/src/state/mod.rs +++ b/packages/app-lib/src/state/mod.rs @@ -1,8 +1,9 @@ //! Theseus state management system use crate::util::fetch::{FetchSemaphore, IoSemaphore}; +use dashmap::DashMap; use std::sync::Arc; use std::sync::atomic::AtomicBool; -use tokio::sync::{OnceCell, Semaphore}; +use tokio::sync::{Mutex, OnceCell, OwnedMutexGuard, Semaphore}; use crate::state::instances::watcher::FileWatcher; use sqlx::SqlitePool; @@ -71,6 +72,8 @@ pub struct State { pub api_semaphore: FetchSemaphore, pub(crate) install_job_semaphore: Semaphore, pub(crate) install_db_semaphore: Semaphore, + /// Serializes filesystem reconciliation and content mutations per instance. + instance_content_locks: DashMap>>, /// Discord RPC pub discord_rpc: DiscordGuard, @@ -95,6 +98,23 @@ pub struct State { } impl State { + pub(crate) async fn lock_instance_content( + &self, + instance_id: &str, + ) -> OwnedMutexGuard<()> { + let lock = self + .instance_content_locks + .entry(instance_id.to_string()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone(); + + lock.lock_owned().await + } + + pub(crate) fn remove_instance_content_lock(&self, instance_id: &str) { + let _ = self.instance_content_locks.remove(instance_id); + } + pub async fn init(app_identifier: String) -> crate::Result<()> { let state = LAUNCHER_STATE .get_or_try_init(move || Self::initialize_state(app_identifier)) @@ -210,6 +230,7 @@ impl State { api_semaphore, install_job_semaphore: Semaphore::new(MAX_CONCURRENT_INSTALL_JOBS), install_db_semaphore: Semaphore::new(1), + instance_content_locks: DashMap::new(), discord_rpc, process_manager, friends_socket,