load all files into memory, use change tracking map instead of object

.github/workflows/release.yaml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
-      - run: npm ci
+      - run: npm install
       - run: npm run compile
       - run: git log $(git describe --tags --abbrev=0 HEAD~1)..HEAD --oneline > changelog.txt
       - uses: softprops/action-gh-release@v2

cli/src/main.ts

@@ -42,6 +42,7 @@ try {
             const apiKey: string = curseforge.opts().apiKey || process.env.CF_API_KEY
             const pack = await getPack()
             await pack.findAndReplaceCurseforgeFiles(apiKey)
+            await pack.savePack()
         })
 
     curseforge.command('urls')
@@ -61,6 +62,7 @@ try {
         .action(async () => {
             const pack = await getPack()
             await pack.findAndReplaceModrinthFiles()
+            await pack.savePack()
         })
 
     modrith.command('merge')

lib/src/ChangeTrackingMap.ts

@@ -0,0 +1,48 @@
+/**
+ * This map tracks when entries have been changed after the map was constructed,
+ * and provides a method to retrieve just those changes via `getChanged()`
+ * and reset change tracking via `clearChanged()`
+ */
+export default
+    class ChangeTrackingMap<K, V> extends Map<K, V> {
+    private changedKeys=new Set<K>()
+    clear(): void {
+        for (const key of this.keys()) {
+            this.changedKeys.add(key)
+        }
+        super.clear()
+    }
+    delete(key: K): boolean {
+        const deleted=super.delete(key)
+        if(deleted) this.changedKeys.add(key)
+        return deleted
+    }
+    set(key: K, value: V): this {
+        super.set(key,value)
+        this.changedKeys.add(key)
+        return this
+    }
+    /**
+     * Constructs a new map of the changed entries in this map.
+     * The special value `null` is used to indicate an entry was deleted.
+     * @returns the change map
+     */
+    getChanged(): Map<K, V | null> {
+        const changeMap=new Map<K,V|null>()
+        for (const key of this.changedKeys) {
+            if (this.has(key)) changeMap.set(key, this.get(key)!)
+            else changeMap.set(key, null)
+        }
+        return changeMap
+    }
+    /**
+     * Clears the list of changed entries,
+     * to allow continuing to track changes after appropriate action has been taken on previous changes,
+     * such as after saving changes to storage, etc.
+     * This does NOT restore the map to the pre-change state,
+     * only resets the list of changed entries.
+     */
+    clearChanged() {
+        this.changedKeys=new Set()
+    }
+}

lib/src/packwiz.ts

@@ -1,10 +1,11 @@
-import type { FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, FileSystemRemoveOptions, FileSystemWriteChunkType, Response } from "@sugoidogo/importable-types-web"
+import type { Blob, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, FileSystemRemoveOptions, FileSystemWriteChunkType, Response } from "@sugoidogo/importable-types-web"
 import type { Hash, HashFormat, PackFileMetaData, PackIndex, PackMetaData, Path, Side, Url } from "./types"
 import { parseTOML, stringifyTOML } from "confbox"
 import { forAsync } from "./forAsync.ts"
 import WorkerlessPool from "workerless"
 import { CurseforgeV1Client, type Mod } from "@xmcl/curseforge"
 import { ModrinthV2Client, type Project } from "@xmcl/modrinth"
+import ChangeTrackingMap from "./ChangeTrackingMap.ts"
 
 const pathSeperatorRegex = /[\\\/]/
 const minecrafCurseforgeGameID = 432
@@ -109,14 +110,7 @@ export class PackwizPack {
     packMetaData: PackMetaData
     packIndex: PackIndex
     /** This object stores the contents of {@link PackFileMetaData} files referenced in the {@link PackIndex} */
-    packFiles: { [key: Path]: PackFileMetaData | undefined }
-    /**
-     * This list informs the save function on what files need updates to save in-memory changes,
-     * allowing you to make multiple changes before incurring writes to storage.
-     * If you make any changes to the pack outside of the functions provided by this API,
-     * you will need to update this list.
-     */
-    packFilesModified: { [key: Path]: "write" | "delete" | undefined }
+    packFiles: ChangeTrackingMap<Path, PackFileMetaData | Uint8Array<ArrayBuffer>>
     /** Creates a new, empty modpack */
     constructor(packDirectoryHandle: FileSystemDirectoryHandle, packName: string, versions: PackMetaData["versions"]) {
         this.packDirectoryHandle = packDirectoryHandle
@@ -134,8 +128,7 @@ export class PackwizPack {
             "hash-format": "sha1",
             "files": [] as any
         }
-        this.packFiles = {}
-        this.packFilesModified = {}
+        this.packFiles = new ChangeTrackingMap()
     }
     /** Fetches a modpack from an HTTP/HTTPS url */
     static async fetchPack(packDirectoryHandle: FileSystemDirectoryHandle, packUrl: Url): Promise<PackwizPack> {
@@ -176,8 +169,10 @@ export class PackwizPack {
                 .then(response => assertOK(response).bytes())
             await assertHashMatch(fileBytes, entry["hash-format"] || packIndex["hash-format"], entry.hash, entry.file)
             await writeFile(fileHandle, fileBytes)
-            if (entry.metafile) pack.packFiles[entry.file] = parseTOML(decode(fileBytes))
+            if (entry.metafile) pack.packFiles.set(entry.file, parseTOML(decode(fileBytes)))
+            else pack.packFiles.set(entry.file, fileBytes)
         })
+        pack.packFiles.clearChanged()
         console.log("pack fetched")
         return pack
     }
@@ -214,8 +209,10 @@ export class PackwizPack {
             if (!packMetaData.options || !packMetaData.options["no-internal-hashes"]) {
                 await assertHashMatch(fileBytes, entry["hash-format"] || packIndex["hash-format"], entry.hash, entry.file)
             }
-            if (entry.metafile) pack.packFiles[entry.file] = parseTOML(decode(fileBytes))
+            if (entry.metafile) pack.packFiles.set(entry.file, parseTOML(decode(fileBytes)))
+            else pack.packFiles.set(entry.file, fileBytes)
         }
+        pack.packFiles.clearChanged()
         console.log("pack loaded")
         return pack
     }
@@ -260,11 +257,10 @@ export class PackwizPack {
                     "file": filePath,
                     "metafile": isMetafile
                 })
-                if (isMetafile) {
-                    const file = await handle.getFile()
-                    const text = await file.text()
-                    pack.packFiles[filePath] = parseTOML(text)
-                }
+                const file = await handle.getFile()
+                const fileBytes = await file.bytes()
+                if (isMetafile) pack.packFiles.set(filePath, parseTOML(decode(fileBytes)))
+                else pack.packFiles.set(filePath, fileBytes)
             })
         } while (directoryHandles.length !== 0)
         console.log("pack refreshed")
@@ -284,50 +280,53 @@ export class PackwizPack {
         }
         const packIndexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
         packIndexPathSegments.pop()
-        const packIndexDirname = packIndexPathSegments.join("/")
+        const packIndexDirname = packIndexPathSegments.join("/") || "."
+        const changeMap = this.packFiles.getChanged()
         let deletedFileCount = 0
         await forAsync(Array(this.packIndex.files.length).keys(), async index => {
             index -= deletedFileCount
             const entry = this.packIndex.files[index]
-            const fileOperation = this.packFilesModified[entry.file]
-            delete this.packFilesModified[entry.file]
-            if (!fileOperation) return
+            let data = changeMap.get(entry.file)
+            if (data === undefined) return
             const filePath = packIndexDirname + "/" + entry.file
-            if (fileOperation === "delete") {
+            if (data === null) {
                 this.packIndex.files.splice(index, 1)
                 deletedFileCount++
-                return deleteFiles(this.packDirectoryHandle, filePath)
+                console.debug("deleting " + filePath)
+                await deleteFiles(this.packDirectoryHandle, filePath)
+                if (!changeMap.delete(entry.file)) throw new Error("change map mismatch")
+                return
             }
-            const packFileMetaData = this.packFiles[entry.file]
             const fileHandle = await getFileHandle(this.packDirectoryHandle, filePath, { "create": true })
-            let packFileBytes: Uint8Array<ArrayBuffer>
-            if (packFileMetaData) {
-                packFileBytes = encode(stringifyTOML(packFileMetaData))
-                await writeFile(fileHandle, packFileBytes)
-            }
-            else {
-                const file = await fileHandle.getFile()
-                packFileBytes = await file.bytes()
-            }
+            if (!(data instanceof Uint8Array)) data = encode(stringifyTOML(data))
+            console.debug("writing " + filePath)
+            await writeFile(fileHandle, data)
             if (hashes) {
-                entry.hash = await getHash(packFileBytes, entry["hash-format"] || this.packIndex["hash-format"])
+                entry.hash = await getHash(data, entry["hash-format"] || this.packIndex["hash-format"])
                     .then(hash => hash.toString())
             } else {
                 delete entry.hash
             }
+            if (!changeMap.delete(entry.file)) throw new Error("change map mismatch")
         })
-        await forAsync(Object.keys(this.packFilesModified), async filePath => {
-            const packFilePath = packIndexDirname + "/" + filePath
-            const fileHandle = await getFileHandle(this.packDirectoryHandle, packFilePath, { 'create': true })
-            const packFileBytes = encode(stringifyTOML(this.packFiles[filePath]))
-            writeFile(fileHandle, packFileBytes)
+        await forAsync(changeMap.entries(), async entry => {
+            let [filePath, data] = entry
+            const fileHandle = await getFileHandle(this.packDirectoryHandle, filePath, { "create": true })
+            if (!(data instanceof Uint8Array)) data = encode(stringifyTOML(data))
+            let hash = undefined
+            if (hashes) {
+                hash = await getHash(data, entry["hash-format"] || this.packIndex["hash-format"])
+                    .then(hash => hash.toString())
+            }
+            console.debug("writing " + filePath)
+            await writeFile(fileHandle, data)
             this.packIndex.files.push({
                 "file": filePath,
-                "metafile": true,
-                "hash": hashes ? (await getHash(packFileBytes, this.packIndex["hash-format"])).toString() : undefined
+                "metafile": !(data instanceof Uint8Array),
+                "hash": hash
             })
-            delete this.packFilesModified[filePath]
         })
+        this.packFiles.clearChanged()
         await getFileHandle(this.packDirectoryHandle, this.packMetaData.index.file, { "create": true }).then(async packIndexFileHandle => {
             const packIndexBytes = encode(stringifyTOML(this.packIndex))
             if (hashes) {
@@ -336,8 +335,10 @@ export class PackwizPack {
             } else {
                 delete this.packMetaData.index.hash
             }
+            console.debug("writing " + this.packMetaData.index.file)
             await writeFile(packIndexFileHandle, packIndexBytes)
         })
+        console.debug("writing "+this.packFileName)
         await getFileHandle(this.packDirectoryHandle, this.packFileName, { "create": true })
             .then(packFileHandle => writeFile(packFileHandle, encode(stringifyTOML(this.packMetaData))))
         console.log("changes saved")
@@ -353,23 +354,24 @@ export class PackwizPack {
         const workerlessPool = new WorkerlessPool()
         const indexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
         indexPathSegments.pop()
-        const indexDirname = indexPathSegments.join("/")
-        const indexDirectoryHandle = await getDirectoryHandle(this.packDirectoryHandle, indexDirname)
         const hashPaths = new Map<number, Path>()
         await forAsync(this.packIndex.files, async entry => {
             if (entry.metafile) return
-            const fileHandle = await getFileHandle(indexDirectoryHandle, entry.file)
-            const file = await fileHandle.getFile()
-            if (file.size === 0) return
-            const bytes = await file.bytes()
+            if(!(entry.file.endsWith(".zip")||entry.file.endsWith(".jar"))) return
+            const bytes = this.packFiles.get(entry.file) as Uint8Array<ArrayBuffer>
             const hash = await workerlessPool.run(getHash, bytes, "murmur2") as number
             hashPaths.set(hash, entry.file)
         })
         workerlessPool.terminate()
+        if (hashPaths.size === 0) {
+            console.log("no files to check")
+            return this
+        }
         console.log("requesting info on " + hashPaths.size + " files")
         const curseforge = new CurseforgeV1Client(apiKey)
         const result = await curseforge.getFingerprintsMatchesByGameId(minecrafCurseforgeGameID, [...hashPaths.keys()])
         console.debug("got " + result.exactMatches.length + " matches")
+        if (result.exactMatches.length === 0) return this
         const modIDs: number[] = []
         for (const match of result.exactMatches) modIDs.push(match.file.modId)
         const mods = new Map<number, Mod>()
@@ -382,11 +384,10 @@ export class PackwizPack {
             const filePathSegments = filePath.split(pathSeperatorRegex)
             const fileDirname = filePathSegments.join("/")
             const newFilePath = fileDirname + "/" + mod.slug + ".pw.toml"
-            this.packFilesModified[filePath] = "delete"
-            this.packFilesModified[newFilePath] = "write"
+            this.packFiles.delete(filePath)
             for (const hash of match.file.hashes) {
                 if (hash.algo === 1) {
-                    this.packFiles[newFilePath] = {
+                    this.packFiles.set(newFilePath, {
                         "filename": match.file.fileName,
                         "name": mod.name,
                         "download": {
@@ -402,16 +403,17 @@ export class PackwizPack {
                         },
                         "option": {
                             "optional": false,
+                            "default": true,
                             "description": mod.summary
                         }
-                    }
+                    })
                     replacedFileCount++
                     break
                 }
             }
         })
         console.log("replaced " + replacedFileCount + " files")
-        return this.savePack()
+        return this
     }
     /**
      * Hashes all non-metafiles in the pack and makes a request to modrinth to find matching files.
@@ -421,18 +423,18 @@ export class PackwizPack {
         console.log("finding and replacing modrinth files")
         const indexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
         indexPathSegments.pop()
-        const indexDirname = indexPathSegments.join("/")
-        const indexDirectoryHandle = await getDirectoryHandle(this.packDirectoryHandle, indexDirname)
         const hashPaths = new Map<Hash, Path>()
         await forAsync(this.packIndex.files, async entry => {
             if (entry.metafile) return
-            const fileHandle = await getFileHandle(indexDirectoryHandle, entry.file)
-            const file = await fileHandle.getFile()
-            if (file.size === 0) return
-            const bytes = await file.bytes()
+            if (!(entry.file.endsWith(".zip") || entry.file.endsWith(".jar"))) return
+            const bytes = this.packFiles.get(entry.file) as Uint8Array<ArrayBuffer>
             const hash = await getHash(bytes, "sha512") as string
             hashPaths.set(hash, entry.file)
         })
+        if (hashPaths.size === 0) {
+            console.log("no files to check")
+            return this
+        }
         console.log("requesting info on " + hashPaths.size + " files")
         const modrinth = new ModrinthV2Client()
         const matches = await modrinth.getProjectVersionsByHash([...hashPaths.keys()])
@@ -441,6 +443,7 @@ export class PackwizPack {
         const projects = new Map<string, Project>()
         for (const project of await modrinth.getProjects(projectIDs)) projects.set(project.id, project)
         console.log("got " + projectIDs.length + " matches")
+        if (projectIDs.length === 0) return this;
         let replacedFileCount = 0
         await forAsync(Object.entries(matches), async entry => {
             const [hash, match] = entry
@@ -450,8 +453,7 @@ export class PackwizPack {
             const filePathSegments = filePath.split(pathSeperatorRegex)
             const fileDirname = filePathSegments.join("/")
             const newFilePath = fileDirname + "/" + mod.slug + ".pw.toml"
-            this.packFilesModified[filePath] = "delete"
-            this.packFilesModified[newFilePath] = "write"
+            this.packFiles.delete(filePath)
             let side: Side = "both"
             if (mod.server_side === "unsupported") {
                 side = "client"
@@ -460,7 +462,7 @@ export class PackwizPack {
             }
             for (const file of match.files) {
                 if (file.hashes["sha512"] === hash) {
-                    this.packFiles[newFilePath] = {
+                    this.packFiles.set(newFilePath, {
                         "filename": file.filename,
                         "name": mod.title,
                         "download": {
@@ -478,13 +480,13 @@ export class PackwizPack {
                             "optional": false,
                             "description": mod.description
                         }
-                    }
+                    })
                 }
             }
             replacedFileCount++
         })
         console.log("replaced " + replacedFileCount + " files")
-        return this.savePack()
+        return this
     }
     /**
      * In order to comply with curseforge API requirements, direct download URLs to curseforge files cannot be saved into the pack.
@@ -494,10 +496,15 @@ export class PackwizPack {
     async cacheCurseforgeDownloadURLs(apiKey: string): Promise<PackwizPack> {
         console.log("caching curseforge download URLs")
         const fileIDPaths = new Map<number, Path>()
-        for (const [filePath, metafiledata] of Object.entries(this.packFiles)) {
+        for (const [filePath, metafiledata] of this.packFiles.entries()) {
+            if (metafiledata instanceof Uint8Array) continue
             if (!metafiledata.download.mode) continue
             fileIDPaths.set(metafiledata.update.curseforge["file-id"], filePath)
         }
+        if (fileIDPaths.size === 0) {
+            console.log("no files to check")
+            return this
+        }
         console.log("requesting info on " + fileIDPaths.size + " files")
         const curseforge = new CurseforgeV1Client(apiKey)
         let cachedURLcount = 0
@@ -505,9 +512,10 @@ export class PackwizPack {
             if (file.downloadUrl) {
                 cachedURLcount++
                 const filePath = fileIDPaths.get(file.id)
-                this.packFiles[filePath].download.url = file.downloadUrl
-                delete this.packFiles[filePath].download.mode
-                this.packFilesModified[filePath] = "write"
+                const packFileMetaData = this.packFiles.get(filePath) as PackFileMetaData
+                packFileMetaData.download.url = file.downloadUrl
+                delete packFileMetaData.download.mode
+                this.packFiles.set(filePath, packFileMetaData)
             }
         }
         console.log("cached " + cachedURLcount + " URLs")
@@ -522,10 +530,15 @@ export class PackwizPack {
     async mergeModrinthMetadata(): Promise<PackwizPack> {
         console.log("merging modrinth metadata with curseforge files")
         const hashPaths = new Map<Hash, Path>()
-        for (const [filePath, metafiledata] of Object.entries(this.packFiles)) {
+        for (const [filePath, metafiledata] of this.packFiles.entries()) {
+            if (metafiledata instanceof Uint8Array) continue
             if (metafiledata.update.modrinth) continue
             hashPaths.set(metafiledata.download.hash, filePath)
         }
+        if (hashPaths.size == 0) {
+            console.log("no files to check")
+            return this
+        }
         console.log("requesting info on " + hashPaths.size + " files")
         const modrinth = new ModrinthV2Client()
         const matches = await modrinth.getProjectVersionsByHash([...hashPaths.keys()])
@@ -548,7 +561,7 @@ export class PackwizPack {
             }
             for (const file of match.files) {
                 if (file.hashes["sha1"] === hash) {
-                    const metafiledata = this.packFiles[filePath]
+                    const metafiledata = this.packFiles.get(filePath) as PackFileMetaData
                     delete metafiledata.download.mode
                     metafiledata.download.url = file.url
                     metafiledata.side = side
@@ -556,10 +569,10 @@ export class PackwizPack {
                         "mod-id": mod.id,
                         "version": match.id
                     }
+                    this.packFiles.set(filePath, metafiledata)
                     break
                 }
             }
-            this.packFilesModified[filePath] = "write"
         }
         console.log("merged " + mergedFileCount + " files")
         return this