refactor

.gitignore

@@ -6,4 +6,5 @@ dist
 *.tgz
 docs
 bin
-*lock*
+*lock*
+.vscode

cli/scripts/test.ts

@@ -15,10 +15,8 @@ if (node) {
     spawnSync('packwiz', ['curseforge', 'import', '../modpack.zip'], { cwd: 'test' })
     console.log('> test on node')
     const start = performance.now()
-    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'cf', 'detect'])
-    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'cf', 'urls'])
-    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'mr', 'detect'])
-    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'mr', 'merge'])
+    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'detect', 'modrinth'])
+    spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'detect', 'curseforge'])
     spawnSync('npx', ['tsx', 'src/main.ts', '--pack-file=test/pack.toml', 'refresh', '--build'])
     const end = performance.now()
     times['node'] = (end - start) / 1000
@@ -31,10 +29,8 @@ if (deno) {
     spawnSync('packwiz', ['curseforge', 'import', '../modpack.zip'], { cwd: 'test' })
     console.log('> test on deno')
     const start = performance.now()
-    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'cf', 'detect'])
-    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'cf', 'urls'])
-    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'mr', 'detect'])
-    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'mr', 'merge'])
+    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'detect', 'modrinth'])
+    spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'detect', 'curseforge'])
     spawnSync('deno', ['run', '--allow-all', 'src/main.ts', '--pack-file=test/pack.toml', 'refresh', '--build'])
     const end = performance.now()
     times['deno'] = (end - start) / 1000

cli/src/main.ts

@@ -2,7 +2,7 @@
 
 import { program } from "commander"
 import { config } from "dotenv"
-import packwiz from "@sugoidogo/packwiz-extras-lib"
+import { PackwizPack as packwiz, ModrinthClient, CurseforgeClient, type ApiClient } from "@sugoidogo/packwiz-extras-lib"
 import NodeFileSystemDirectoryHandle from "@sugoidogo/node-file-system-adapter"
 import { execSync } from "node:child_process"
 import { freemem } from "node:os"
@@ -16,104 +16,77 @@ try {
         const packFile: string = program.opts().packFile
         const packFilePathSegments = packFile.split(pathSeperatorRegex)
         const packFileBasename = packFilePathSegments.pop()
-        const packFileDirname = "./"+packFilePathSegments.join("/")
+        const packFileDirname = "./" + packFilePathSegments.join("/")
         return { packFileDirname, packFileBasename }
     }
 
-    async function getPack() {
-        const {packFileDirname,packFileBasename}=getPackPaths()
+    function getPackInit() {
+        const { packFileDirname, packFileBasename } = getPackPaths()
         const packDirectoryHandle = new NodeFileSystemDirectoryHandle(packFileDirname)
-        return packwiz.loadPack(packDirectoryHandle, packFileBasename)
+        return { packDirectoryHandle, packFileBasename }
     }
 
     program.name('packwiz-extras')
         .description('extra utilities for packwiz')
         .option('--pack-file <string>', 'The modpack file to use', 'pack.toml')
 
-    const curseforge = program.command('curseforge').alias('cf')
-        .description('manage curseforge files')
-
-    const cfKey: string[] = ['--api-key <string>',
-        'Your curseforge api key. This is required by curseforge to access their API. ' +
-        'Can also be provided via environment variable or .env file as CF_API_KEY',]
-
-    if (process.env.CF_API_KEY) {
-        curseforge.option(cfKey[0], cfKey[1])
-    } else {
-        curseforge.requiredOption(cfKey[0], cfKey[1])
-    }
-
-    curseforge.command('detect')
-        .description('detect and replace files availible on curseforge')
-        .action(async () => {
-            const apiKey: string = curseforge.opts().apiKey || process.env.CF_API_KEY
-            const pack = await getPack()
-            await pack.findAndReplaceCurseforgeFiles(apiKey)
-            await pack.savePack()
+    program.command("detect <apiName> [apiKey]")
+        .description("detect and replace files availible via <apiName> (either curseforge or modrinth). " +
+            "apiKey is required for curseforge.")
+        .action(async (apiName, apiKey) => {
+            let apiClient: ApiClient
+            switch (apiName) {
+                case "modrinth": {
+                    apiClient = new ModrinthClient(apiKey)
+                    break
+                }
+                case "curseforge": {
+                    if (!apiKey) apiKey = process.env["CF_API_KEY"]
+                    if (!apiKey) throw new Error("apiKey required for Curseforge")
+                    apiClient = new CurseforgeClient(apiKey)
+                    break
+                }
+                default: throw new Error("unknown api: " + apiName)
+            }
+            const { packDirectoryHandle, packFileBasename } = getPackInit()
+            const pack = await packwiz.loadPack(packDirectoryHandle, packFileBasename)
+            await pack.detect(apiClient)
+            await pack.writeChanges(packDirectoryHandle)
         })
 
-    curseforge.command('urls')
-        .description('cache curseforge download urls to speed up packwiz-installer')
-        .action(async () => {
-            const apiKey = curseforge.opts().apiKey || process.env.CF_API_KEY
-            const pack = await getPack()
-            await pack.cacheCurseforgeDownloadURLs(apiKey)
-            await pack.savePack()
-        })
-
-    const modrith = program.command('modrinth').alias('mr')
-        .description('manage modrinth files')
-
-    modrith.command('detect')
-        .description('detect and replace files availible on modrinth')
-        .action(async () => {
-            const pack = await getPack()
-            await pack.findAndReplaceModrinthFiles()
-            await pack.savePack()
-        })
-
-    modrith.command('merge')
-        .description('detect curseforge metafiles which are also availible on modrinth and merge their metadata')
-        .action(async () => {
-            const pack = await getPack()
-            await pack.mergeModrinthMetadata()
-            await pack.savePack()
-        })
-    
     program.command("refresh")
         .description("refresh the index file")
         .option("--build", "Only has an effect in no-internal-hashes mode:"
             + "\nwgenerates internal hashes for distribution with packwiz-installer")
         .action(async (options) => {
-            const { packFileDirname, packFileBasename } = getPackPaths()
-            const packDirectoryHandle = new NodeFileSystemDirectoryHandle(packFileDirname)
+            const { packDirectoryHandle, packFileBasename } = getPackInit()
             const pack = await packwiz.refreshPack(packDirectoryHandle, packFileBasename)
-            await pack.savePack(options.build)
+            await pack.writeChanges(packDirectoryHandle)
         })
-    
+
     program.command("test-server [java]")
         .description(
             "use docker to test the server side of the modpack."
             + "\nyou can specify the java version using tags like java17"
             + "\nthe full list of supported java versions can be found at the following url:"
             + "\nhttps://docker-minecraft-server.readthedocs.io/en/latest/versions/java/"
-    ).action(async (java) => {
-        if (!java) java = "stable"
-        const { packFileDirname, packFileBasename } = getPackPaths()
-        const pack = await getPack()
-        await pack.savePack(true)
-        let version=pack.packMetaData.versions.minecraft
-        let loader = "VANILLA"
-        let loader_version=version
-        for (const key of Object.keys(pack.packMetaData.versions)) {
-            if (key === "minecraft") continue
-            loader = key.toUpperCase()
-            loader_version=pack.packMetaData.versions[key]
-        }
-        execSync(`docker run -it -v ${packFileDirname}:/pack -e "PACKWIZ_URL=/pack/${packFileBasename}"`
-            + ` -e EULA=TRUE -e MAX_MEMORY=${freemem()} -e VERSION=${version} -e TYPE=${loader} -e ${loader}_VERSION=${loader_version}`
-            + ` docker.io/itzg/minecraft-server:${java}`, { "stdio": "inherit" })
-    })
+        ).action(async (java) => {
+            if (!java) java = "stable"
+            const { packDirectoryHandle, packFileBasename } = getPackInit()
+            const pack = await packwiz.refreshPack(packDirectoryHandle, packFileBasename, true)
+            await pack.writeChanges(packDirectoryHandle)
+            let version = pack.packMetaData.versions.minecraft
+            let loader = "VANILLA"
+            let loader_version = version
+            for (const key of Object.keys(pack.packMetaData.versions)) {
+                if (key === "minecraft") continue
+                loader = key.toUpperCase()
+                loader_version = pack.packMetaData.versions[key]
+            }
+            execSync(`docker run -it -v ${packDirectoryHandle.path}:/pack:z -e "PACKWIZ_URL=/pack/${packFileBasename}"`
+                + ` -e EULA=TRUE -e MAX_MEMORY=${freemem()} -e VERSION=${version} -e TYPE=${loader} -e ${loader}_VERSION=${loader_version}`
+                + ` docker.io/itzg/minecraft-server:${java}`, { "stdio": "inherit" })
+        })
 
     await program.parseAsync()
 } catch (error) {

lib/package.json

@@ -26,11 +26,11 @@
     "docs": "typedoc src/index.ts"
   },
   "dependencies": {
-    "@xmcl/curseforge": "^2.1.1",
-    "@xmcl/modrinth": "^2.3.1",
+    "@plus99/murmur-hash": "^1.0.2",
+    "@xmcl/curseforge": "^2.2.0",
+    "@xmcl/modrinth": "^2.5.0",
     "confbox": "^0.2.4",
     "md5": "^2.3.0",
-    "murmur2": "^1.0.2",
     "workerless": "^0.1.0"
   },
   "devDependencies": {

lib/src/ChangeTrackingMap.ts

@@ -3,23 +3,32 @@
  * and provides a method to retrieve just those changes via `getChanged()`
  * and reset change tracking via `clearChanged()`
  */
-export default
-    class ChangeTrackingMap<K, V> extends Map<K, V> {
-    private changedKeys=new Set<K>()
+export class ChangeTrackingMap<K, V extends Object> extends Map<K, V> {
+    #changedKeys: Set<K>
+    constructor(iterable?:Iterable<[K,V]>) {
+        super()
+        for (const [key, value] of iterable) {
+            super.set(key,value)
+        }
+        this.#changedKeys=new Set()
+    }
     clear(): void {
         for (const key of this.keys()) {
-            this.changedKeys.add(key)
+            this.#changedKeys.add(key)
         }
         super.clear()
     }
     delete(key: K): boolean {
-        const deleted=super.delete(key)
-        if(deleted) this.changedKeys.add(key)
+        const deleted = super.delete(key)
+        if (deleted) this.#changedKeys.add(key)
         return deleted
     }
     set(key: K, value: V): this {
-        super.set(key,value)
-        this.changedKeys.add(key)
+        if (!("bytes" in value)) {
+            value=structuredClone(value)
+        }
+        super.set(key, value)
+        this.#changedKeys.add(key)
         return this
     }
     /**
@@ -28,8 +37,8 @@ export default
      * @returns the change map
      */
     getChanged(): Map<K, V | null> {
-        const changeMap=new Map<K,V|null>()
-        for (const key of this.changedKeys) {
+        const changeMap = new Map<K, V | null>()
+        for (const key of this.#changedKeys) {
             if (this.has(key)) changeMap.set(key, this.get(key)!)
             else changeMap.set(key, null)
         }
@@ -43,6 +52,43 @@ export default
      * only resets the list of changed entries.
      */
     clearChanged() {
-        this.changedKeys=new Set()
+        this.#changedKeys = new Set()
     }
-}
+    forEach(callbackfn: (value: V, key: K, map: Map<K, V>) => void, thisArg?: any): void {
+        for (let [key, value] of super.entries()) {
+
+            if (!("bytes" in value)) {
+                value = structuredClone(value)
+            }
+            callbackfn(value, key, this)
+        }
+    }
+    get(key: K): V {
+        let value = super.get(key)
+        if (!("bytes" in value)) {
+            value = structuredClone(value)
+        }
+        return value
+    }
+    *entries(): MapIterator<[K, V]> {
+        for (let [key, value] of super.entries()) {
+            if (!("bytes" in value)) {
+                value = structuredClone(value)
+            }
+            yield [key, value]
+        }
+    }
+    *values(): MapIterator<V> {
+        for (let value of super.values()) {
+            if (!("bytes" in value)) {
+                value = structuredClone(value)
+            }
+            yield value
+        }
+    }
+    [Symbol.iterator](): MapIterator<[K, V]> {
+        return this.entries()
+    }
+}
+
+export default ChangeTrackingMap

lib/src/CurseforgeClient.ts

@@ -0,0 +1,52 @@
+import type { ApiClient, Hash, HashFormat, PackFileMetaData, Path } from "./types"
+import { CurseforgeV1Client } from "@xmcl/curseforge"
+
+const gameId = 432
+
+export class CurseforgeClient implements ApiClient {
+    client: CurseforgeV1Client
+    name="curseforge"
+    hashFormats:HashFormat[]=["murmur2"]
+    constructor(apiKey: string) {
+        this.client=new CurseforgeV1Client(apiKey)
+    }
+    async detect(hashes: Iterable<Hash>) {
+        const fingerprints:number[] = []
+        for (const hash of hashes) {
+            fingerprints.push(Number(hash))
+        }
+        const response = await this.client.getFingerprintsMatchesByGameId(gameId,fingerprints)
+        const matches = new Map<Path, PackFileMetaData>()
+        if(!response.exactMatches) return matches
+        for (const match of response.exactMatches) {
+            if (!match.file) continue
+            if (!match.file.displayName) continue
+            if (!match.file.fileName) continue
+            if (!match.file.hashes) continue
+            if (!match.file.fileFingerprint) continue
+            for (const hash of match.file.hashes) {
+                if (hash.algo === 1) {
+                    const packFileMetaData: PackFileMetaData = {
+                        "name": match.file.displayName,
+                        "filename": match.file.fileName,
+                        "download": {
+                            "hash-format": "sha1",
+                            "hash": hash.value,
+                            "mode": "metadata:curseforge"
+                        },
+                        "update": {
+                            "curseforge": {
+                                "file-id": match.file.id,
+                                "project-id": match.file.modId
+                            }
+                        }
+                    }
+                    matches.set(match.file.fileFingerprint.toString(),packFileMetaData)
+                }
+            }
+        }
+        return matches
+    }
+}
+
+export default CurseforgeClient

lib/src/Directory.ts

@@ -0,0 +1,75 @@
+import type { Path } from "./types"
+import type { FileSystemDirectoryHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions } from "@sugoidogo/importable-types-web"
+
+export const pathSeperatorRegex = /[\\\/]/
+
+export class Directory {
+    #directoryHandle: FileSystemDirectoryHandle
+    constructor(directoryHandle: FileSystemDirectoryHandle) {
+        this.#directoryHandle = directoryHandle
+    }
+
+    static async #getDirectoryHandle(directoryHandle: FileSystemDirectoryHandle, path: Path, options?: FileSystemGetDirectoryOptions) {
+        const pathSegments = path.split(pathSeperatorRegex)
+        for (const segment of pathSegments) {
+            if (!segment) continue
+            directoryHandle = await directoryHandle.getDirectoryHandle(segment, options)
+        }
+        return directoryHandle
+    }
+
+    static async #getFileHandle(directoryHandle: FileSystemDirectoryHandle, path: Path, options?: FileSystemGetFileOptions) {
+        const pathSegments = path.split(pathSeperatorRegex)
+        const basename = pathSegments.pop()
+        if (!basename) throw new Error("path cannot be empty or end in a slash")
+        const dirname = pathSegments.join('/')
+        directoryHandle = await this.#getDirectoryHandle(directoryHandle, dirname, options)
+        return directoryHandle.getFileHandle(basename, options)
+    }
+
+    async writeFile(path: string, bytes: Uint8Array<ArrayBuffer>) {
+        const fileHandle = await Directory.#getFileHandle(this.#directoryHandle, path, { "create": true })
+        const writable = await fileHandle.createWritable()
+        await writable.write(bytes)
+        await writable.close()
+    }
+
+    async readFile(path: string) {
+        const fileHandle = await Directory.#getFileHandle(this.#directoryHandle, path, { "create": false })
+        return fileHandle.getFile()
+    }
+
+    async delete(path: string) {
+        const pathSegments = path.split(pathSeperatorRegex)
+        const basename = pathSegments.pop()
+        if (!basename) throw new Error("path cannot be empty or end in a slash")
+        const dirname = pathSegments.join('/')
+        const directoryHandle = await Directory.#getDirectoryHandle(this.#directoryHandle, dirname)
+        return directoryHandle.removeEntry(basename, { "recursive": true })
+    }
+
+    async getDirectory(path: string) {
+        const directoryHandle = await Directory.#getDirectoryHandle(this.#directoryHandle, path, { "create": true })
+        return new Directory(directoryHandle)
+    }
+
+    async listFiles() {
+        const fileList = []
+        const queue = [this.#directoryHandle]
+        while (queue.length !== 0) {
+            const directoryHandle = queue.pop()!
+            for await (const handle of directoryHandle?.values()) {
+                if (handle.kind==="directory") {
+                    queue.push(handle)
+                    continue
+                }
+                const pathSegments = await this.#directoryHandle.resolve(handle)
+                if(!pathSegments) throw new Error("this should be impossible")
+                fileList.push(pathSegments.join("/"))
+            }
+        }
+        return fileList
+    }
+}
+
+export default Directory

lib/src/ModrinthClient.ts

@@ -0,0 +1,50 @@
+import type { ApiClient, Hash, HashFormat, PackFileMetaData, Side } from "./types";
+import { ModrinthV2Client, type Project } from "@xmcl/modrinth";
+
+export class ModrinthClient implements ApiClient {
+    hashFormats: HashFormat[] = ["sha512", "sha1"];
+    name="modrinth"
+    client: ModrinthV2Client
+    constructor(apiKey?: string) {
+        this.client=new ModrinthV2Client({"headers":{"Authorization":apiKey}})
+    }
+    async detect(hashes: Iterable<Hash>, hashFormat: HashFormat) {
+        const response = await this.client.getProjectVersionsByHash([...hashes],hashFormat)
+        const projectIDs=[]
+        for (const match of Object.values(response)) {
+            projectIDs.push(match.project_id)
+        }
+        const projects = await this.client.getProjects(projectIDs)
+        const projectsByID=new Map<string,Project>()
+        for (const project of projects) {
+            projectsByID.set(project.id,project)
+        }
+        const matches=new Map<Hash,PackFileMetaData>()
+        for (const [hash, match] of Object.entries(response)) {
+            const project=projectsByID.get(match.project_id)!
+            let side: Side = "both"
+            if (project.client_side === "unsupported" && project.server_side!=="unsupported") side = "server"
+            if (project.server_side === "unsupported" && project.client_side!=="unsupported") side = "client"
+            const packFileMetaData: PackFileMetaData = {
+                "name": match.name,
+                "filename": match.files[0].filename,
+                "side":side,
+                "download": {
+                    "hash-format": "sha1",
+                    "hash": match.files[0].hashes.sha1,
+                    "url": match.files[0].url
+                },
+                "update": {
+                    "modrinth": {
+                        "version": match.id,
+                        "mod-id": match.project_id
+                    }
+                }
+            }
+            matches.set(hash,packFileMetaData)
+        }
+        return matches
+    }
+}
+
+export default ModrinthClient

lib/src/PackIndexMap.ts

@@ -0,0 +1,40 @@
+import type { HashFormat, PackIndex, PackIndexEntry, Path } from "./types";
+
+export class PackIndexMap extends Map<Path, PackIndexEntry> {
+    #packIndex: PackIndex
+    get hashFormat(): HashFormat | undefined { return this.#packIndex["hash-format"] }
+    set hashFormat(hashFormat: HashFormat) { this.#packIndex["hash-format"] = hashFormat }
+    get packIndex() { return structuredClone(this.#packIndex) }
+    constructor(packwizIndex: PackIndex) {
+        super()
+        if (!packwizIndex.files) packwizIndex.files = []
+        for (const packwizIndexEntry of packwizIndex.files!) {
+            super.set(packwizIndexEntry.file, packwizIndexEntry)
+        }
+        this.#packIndex = packwizIndex
+    }
+    clear(): void {
+        while (this.#packIndex.files!.length !== 0) {
+            this.#packIndex.files!.pop()
+        }
+        super.clear()
+    }
+    delete(key: string): boolean {
+        const value = super.get(key)
+        if (!value) return false
+        const index = this.#packIndex.files!.indexOf(value)
+        this.#packIndex.files!.splice(index, 1)
+        return super.delete(key)
+    }
+    set(key: string, value: PackIndexEntry): this {
+        const pre = this.get(key)
+        if (pre) {
+            const index = this.#packIndex.files!.indexOf(pre)
+            this.#packIndex.files![index] = value
+        } else this.#packIndex.files!.push(value)
+        super.set(key,value)
+        return this
+    }
+}
+
+export default PackIndexMap

lib/src/forAsync.ts

@@ -1,8 +1,9 @@
-
 export async function forAsync<T, U>(iterable: Iterable<T> | AsyncIterable<T>, callback: (value: T) => Promise<U> | U): Promise<Array<U>> {
     const promises: Array<Promise<U> | U> = []
     for await (const iterableValue of iterable) {
         promises.push(callback(iterableValue))
     }
     return Promise.all(promises)
-}
+}
+
+export default forAsync

lib/src/index.ts

@@ -1,5 +1,8 @@
-import { PackwizPack } from './packwiz.ts'
+import PackwizPack from './packwiz.ts'
+import CurseforgeClient from "./CurseforgeClient.ts"
+import ModrinthClient from "./ModrinthClient.ts"
 
 export type * from './types.ts'
 export * from './packwiz.ts'
+export { CurseforgeClient, ModrinthClient, PackwizPack }
 export default PackwizPack

lib/src/packwiz.ts

@@ -1,67 +1,19 @@
-import type { Blob, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, FileSystemRemoveOptions, FileSystemWriteChunkType, Response } from "@sugoidogo/importable-types-web"
-import type { Hash, HashFormat, PackFileMetaData, PackIndex, PackMetaData, Path, Side, Url } from "./types"
-import { parseTOML, stringifyTOML } from "confbox"
-import { forAsync } from "./forAsync.ts"
-import WorkerlessPool from "workerless"
-import { CurseforgeV1Client, type Mod } from "@xmcl/curseforge"
-import { ModrinthV2Client, type Project } from "@xmcl/modrinth"
+import type { Path, PackMetaData, PackIndex, HashFormat, Hash, PackFileMetaData, PackIndexEntry, ApiClient } from "./types.ts"
+import type { Response, Blob, FileSystemDirectoryHandle, URL } from "@sugoidogo/importable-types-web"
 import ChangeTrackingMap from "./ChangeTrackingMap.ts"
-
-const pathSeperatorRegex = /[\\\/]/
-const minecrafCurseforgeGameID = 432
-
-function encode(input: string) {
-    // for some reason, every time we re-serialize a TOML file it gains an extra newline
-    input = input.trimEnd() + "\n"
-    return new TextEncoder().encode(input)
-}
-function decode(input: Uint8Array<ArrayBuffer>) { return new TextDecoder().decode(input) }
-
-function assertOK(response: Response) {
-    if (!response.ok) {
-        throw new Error(response.statusText, { "cause": response })
-    }
-    return response as Response & { "ok": true }
-}
-
-async function getDirectoryHandle(directoryHandle: FileSystemDirectoryHandle, path: Path, options?: FileSystemGetDirectoryOptions) {
-    const pathSegments = path.split(pathSeperatorRegex)
-    for (const segment of pathSegments) {
-        if (!segment) continue
-        directoryHandle = await directoryHandle.getDirectoryHandle(segment, options)
-    }
-    return directoryHandle
-}
-
-async function getFileHandle(directoryHandle: FileSystemDirectoryHandle, path: Path, options?: FileSystemGetFileOptions) {
-    const pathSegments = path.split(pathSeperatorRegex)
-    const basename = pathSegments.pop()
-    const dirname = pathSegments.join('/')
-    directoryHandle = await getDirectoryHandle(directoryHandle, dirname, options)
-    return directoryHandle.getFileHandle(basename, options)
-}
-
-async function writeFile(fileHandle: FileSystemFileHandle, data: FileSystemWriteChunkType) {
-    const writable = await fileHandle.createWritable()
-    await writable.write(data)
-    await writable.close()
-}
-
-async function deleteFiles(directoryHandle: FileSystemDirectoryHandle, path: Path, options?: FileSystemRemoveOptions) {
-    const pathSegments = path.split(pathSeperatorRegex)
-    const basename = pathSegments.pop()
-    const dirname = pathSegments.join('/')
-    directoryHandle = await getDirectoryHandle(directoryHandle, dirname)
-    return directoryHandle.removeEntry(basename, options)
-}
+import { PackIndexMap } from "./PackIndexMap.ts"
+import { parseTOML, stringifyTOML } from "confbox"
+import forAsync from "./forAsync.ts"
+import Directory from "./Directory.ts"
+import WorkerlessPool from "workerless"
 
 async function getHash(data: Uint8Array<ArrayBuffer>, format: HashFormat) {
-    if (format.startsWith('sha')) { // copied from https://developer.mozilla.org/en-US/docs/Web/API/Web_Crypto_API/Non-cryptographic_uses_of_subtle_crypto#hashing_a_file
+    if (format.startsWith('sha')) {
+        // copied from https://developer.mozilla.org/en-US/docs/Web/API/Web_Crypto_API/Non-cryptographic_uses_of_subtle_crypto#hashing_a_file
         const hashAsArrayBuffer = await crypto.subtle.digest("SHA-" + format.substring(3), data);
         const uint8ViewOfHash = new Uint8Array(hashAsArrayBuffer);
-        // @ts-ignore
-        if (uint8ViewOfHash.toHex) {
-            // @ts-ignore
+        // modified to satisfy typescript with missing definitions
+        if ("toHex" in uint8ViewOfHash && typeof uint8ViewOfHash.toHex === "function") {
             return uint8ViewOfHash.toHex() as string;
         }
         const hashAsString = Array.from(uint8ViewOfHash)
@@ -70,522 +22,309 @@ async function getHash(data: Uint8Array<ArrayBuffer>, format: HashFormat) {
         return hashAsString;
     }
     if (format === "murmur2") {
-        const { default: murmurHash } = await import('murmur2')
-        return murmurHash(data, 1, true) as number
+        const whitespaceBytes = [9, 10, 13, 32]
+        data = data.filter((byte) => !whitespaceBytes.includes(byte))
+        const { murmur2_32: murmur2 } = await import('@plus99/murmur-hash')
+        return murmur2(data, 1).toString()
     }
     if (format === "md5") {
-        const md5 = await import('md5')
+        //@ts-ignore
+        const { default: md5 } = await import('md5')
         return md5(data)
     }
     throw new Error('unknown hash format: ' + format)
 }
 
-async function assertHashMatch(bytes: Uint8Array<ArrayBuffer>, hashFormat: HashFormat, hash: Hash, errorFileName?: string) {
+async function hashMatch(bytes: Uint8Array<ArrayBuffer>, hashFormat: HashFormat, hash: Hash, errorFileName?: string) {
     const bytesHash = await getHash(bytes, hashFormat)
-    if (bytesHash.toString() !== hash) {
+    if (bytesHash !== hash) {
         if (errorFileName) throw new Error("hash verification failed on " + errorFileName)
         return false
     }
     return true
 }
 
-/**
- * This object contains all the functionality of packwiz-extras, and is the default export of this library.
- * 
- * All instancing methods (new PackwizPack, loadPack, refreshPack, fetchPack) require a FileSystemDirectoryHandle as their first argument.
- * 
- * In web browsers, you can use [navigator.storage.getDirectory](https://developer.mozilla.org/en-US/docs/Web/API/StorageManager/getDirectory)
- * to use browser storage, or [showDirectoryPicker](https://developer.mozilla.org/en-US/docs/Web/API/Window/showDirectoryPicker)
- * to allow the user to select a folder on the host system.
- * 
- * In node, you can use [my node file system adapter](https://gitea.sugoidogo.com/sugoidogo/-/packages/npm/@sugoidogo%2Fnode-file-system-adapter)
- * (or any pollyfill/ponyfill that provides the Web File System API) to grant this library access to a host folder.
- */
-export class PackwizPack {
-    /** The directory in which all pack files are stored */
-    packDirectoryHandle: FileSystemDirectoryHandle
-    /** The name of the {@link PackMetaData} file */
-    packFileName: Path
+const textEncoder = new TextEncoder()
+function encode(input: string) {
+    // for some reason, every time we re-serialize a TOML file it gains an extra newline
+    input = input.trimEnd() + "\n"
+    return textEncoder.encode(input)
+}
+
+function assertOK(response: Response) {
+    if (!response.ok) throw new Error(response.status + " " + response.statusText, { "cause": response })
+    return response as Response & { ok: true }
+}
+
+type PackwizPackInit = {
+    packMetaFileName?: Path
     packMetaData: PackMetaData
     packIndex: PackIndex
-    /** This object stores the contents of {@link PackFileMetaData} files referenced in the {@link PackIndex} */
-    packFiles: ChangeTrackingMap<Path, PackFileMetaData | Uint8Array<ArrayBuffer>>
-    /** Creates a new, empty modpack */
-    constructor(packDirectoryHandle: FileSystemDirectoryHandle, packName: string, versions: PackMetaData["versions"]) {
-        this.packDirectoryHandle = packDirectoryHandle
-        this.packFileName = "pack.toml"
-        this.packMetaData = {
-            "name": packName,
-            "pack-format": "packwiz:1.1.0",
-            "index": {
-                "file": "index.toml",
-                "hash-format": "sha1"
-            },
-            "versions": versions
-        }
-        this.packIndex = {
-            "hash-format": "sha1",
-            "files": [] as any
-        }
-        this.packFiles = new ChangeTrackingMap()
-    }
-    /** Fetches a modpack from an HTTP/HTTPS url */
-    static async fetchPack(packDirectoryHandle: FileSystemDirectoryHandle, packUrl: Url): Promise<PackwizPack> {
-        console.debug("fetching pack from " + packUrl)
-        const { packMetaData, packFileName } = await fetch(packUrl).then(async response => {
-            const packFileText = await assertOK(response).text()
-            const packMetaData = parseTOML(packFileText) as PackMetaData
-            const packFileName = packUrl.split(pathSeperatorRegex).pop()
-            const packFileHandle = await getFileHandle(packDirectoryHandle, packFileName, { "create": true })
-            await writeFile(packFileHandle, packFileText)
-            return { packMetaData, packFileName }
-        })
-        const packIndexURL = new URL(packMetaData.index.file, packUrl)
-        const { packIndex, indexDirname: packIndexDirname } = await fetch(packIndexURL).then(async response => {
-            const packIndexBytes = await assertOK(response).bytes()
-            await assertHashMatch(packIndexBytes, packMetaData.index["hash-format"], packMetaData.index.hash, packMetaData.index.file)
-            const packIndex = parseTOML(decode(packIndexBytes)) as PackIndex
-            const packIndexFileHandle = await getFileHandle(packDirectoryHandle, packMetaData.index.file, { "create": true })
-            await writeFile(packIndexFileHandle, packIndexBytes)
-            const packIndexPathSegments = packIndexURL.pathname.split("/")
-            packIndexPathSegments.pop()
-            const indexDirname = packIndexPathSegments.join("/")
-            return { packIndex, indexDirname }
-        })
-        const pack = Object.assign(new PackwizPack(undefined, undefined, undefined), {
-            packDirectoryHandle,
-            packFileName,
-            packMetaData,
-            packIndex
-        })
-        await forAsync(packIndex.files, async entry => {
-            const fileHandle = await getFileHandle(packDirectoryHandle, packIndexDirname + "/" + entry.file, { "create": true })
-            const file = await fileHandle.getFile()
-            let fileBytes = await file.bytes()
-            if (await assertHashMatch(fileBytes, entry["hash-format"] || packIndex["hash-format"], entry.hash)) return
-            const fileURL = new URL(entry.file, packIndexURL)
-            fileBytes = await fetch(fileURL)
-                .then(response => assertOK(response).bytes())
-            await assertHashMatch(fileBytes, entry["hash-format"] || packIndex["hash-format"], entry.hash, entry.file)
-            await writeFile(fileHandle, fileBytes)
-            if (entry.metafile) pack.packFiles.set(entry.file, parseTOML(decode(fileBytes)))
-            else pack.packFiles.set(entry.file, fileBytes)
-        })
-        pack.packFiles.clearChanged()
-        console.log("pack fetched")
-        return pack
-    }
-    /** Loads a modpack from storage */
-    static async loadPack(packDirectoryHandle: FileSystemDirectoryHandle, packFileName = "pack.toml"): Promise<PackwizPack> {
-        console.debug("loading pack")
-        const packMetaData = await getFileHandle(packDirectoryHandle, packFileName).then(async packDirectoryHandle => {
-            const packFile = await packDirectoryHandle.getFile()
-            const packFileText = await packFile.text()
-            return parseTOML(packFileText) as PackMetaData
-        })
-        const { packIndex, packIndexDirname } = await getFileHandle(packDirectoryHandle, packMetaData.index.file).then(async indexFileHandle => {
-            const packIndexFile = await indexFileHandle.getFile()
-            const packIndexBytes = await packIndexFile.bytes()
-            if (!packMetaData.options || !packMetaData.options["no-internal-hashes"]) {
-                await assertHashMatch(packIndexBytes, packMetaData.index["hash-format"], packMetaData.index.hash, packMetaData.index.file)
+    packFiles?: Iterable<[Path, PackFileMetaData | Blob]>
+} | {
+    name: string
+    versions: PackMetaData["versions"]
+}
+
+export class PackwizPack {
+    packMetaFileName: Path
+    packMetaData: PackMetaData
+    packIndexMap: PackIndexMap
+    packFiles: ChangeTrackingMap<Path, PackFileMetaData | Blob>
+
+    constructor(init: PackwizPackInit) {
+        if ("name" in init) {
+            this.packMetaFileName = "pack.toml"
+            this.packMetaData = {
+                "pack-format": "packwiz:1.1.0",
+                "name": init.name,
+                "versions": init.versions,
+                "index": {
+                    "file": "index.toml",
+                    "hash-format": "sha512"
+                }
             }
-            const packIndexPathSegments = packMetaData.index.file.split(pathSeperatorRegex)
-            packIndexPathSegments.pop()
-            const packIndexDirname = packIndexPathSegments.join("/")
-            const packIndex = parseTOML(decode(packIndexBytes)) as PackIndex
-            return { packIndex, packIndexDirname }
-        })
-        const pack = Object.assign(new PackwizPack(undefined, undefined, undefined), {
-            packDirectoryHandle,
-            packFileName,
-            packMetaData,
-            packIndex
-        })
-        for (const entry of packIndex.files) {
-            const fileHandle = await getFileHandle(packDirectoryHandle, packIndexDirname + "/" + entry.file)
-            const file = await fileHandle.getFile()
-            const fileBytes = await file.bytes()
-            if (!packMetaData.options || !packMetaData.options["no-internal-hashes"]) {
-                await assertHashMatch(fileBytes, entry["hash-format"] || packIndex["hash-format"], entry.hash, entry.file)
-            }
-            if (entry.metafile) pack.packFiles.set(entry.file, parseTOML(decode(fileBytes)))
-            else pack.packFiles.set(entry.file, fileBytes)
+            this.packIndexMap = new PackIndexMap({
+                "hash-format": "sha512",
+                "files": []
+            })
+            this.packFiles = new ChangeTrackingMap()
+        } else {
+            this.packMetaFileName = init.packMetaFileName || "pack.toml"
+            this.packMetaData = init.packMetaData
+            this.packIndexMap = new PackIndexMap(init.packIndex)
+            this.packFiles = new ChangeTrackingMap(init.packFiles)
         }
-        pack.packFiles.clearChanged()
-        console.log("pack loaded")
-        return pack
     }
-    /** Loads a modpack from storage, ignoring and rebuilding the index so that changes made outside of packwiz(-extras) are included. */
-    static async refreshPack(packDirectoryHandle: FileSystemDirectoryHandle, packFileName = "pack.toml"): Promise<PackwizPack> {
-        console.log("refreshing pack")
-        const { packMetaData, packFileHandle } = await getFileHandle(packDirectoryHandle, packFileName).then(async packFileHandle => {
-            const packFile = await packFileHandle.getFile()
-            const packFileText = await packFile.text()
-            const packMetaData = parseTOML(packFileText) as PackMetaData
-            return { packMetaData, packFileHandle }
+
+    async writeChanges(directoryHandle: FileSystemDirectoryHandle) {
+        console.info("saving changes")
+        const packDirectory = new Directory(directoryHandle)
+        const packIndexDirectoryPath = this.packMetaData.index.file.split("/").slice(0, -1).join("/")
+        const indexDirectory = await packDirectory.getDirectory(packIndexDirectoryPath)
+        const useHashes = !(this.packMetaData.options && this.packMetaData.options["no-internal-hashes"])
+
+        const changes = await forAsync(this.packFiles.getChanged(), async ([localPath, newData]) => {
+            if (newData === null) {
+                this.packIndexMap.delete(localPath)
+                return indexDirectory.delete(localPath)
+            }
+            let indexEntry = this.packIndexMap.get(localPath)
+            if (!indexEntry) {
+                indexEntry = { file: localPath }
+                this.packIndexMap.set(localPath, indexEntry)
+            }
+
+            let bytes: Uint8Array<ArrayBuffer>
+            if (!(newData instanceof Blob)) bytes = encode(stringifyTOML(newData))
+            else bytes = await newData.bytes()
+
+            if (useHashes) {
+                const hashFormat = indexEntry["hash-format"] || this.packIndexMap.hashFormat
+                if (!hashFormat) throw new Error(`${this.packMetaData.index.file} is missing hash-format`)
+                indexEntry.hash = await getHash(bytes, hashFormat)
+            }
+
+            indexDirectory.writeFile(localPath, bytes)
         })
-        const { packIndex, packIndexDirname, indexFileHandle } = await getFileHandle(packDirectoryHandle, packMetaData.index.file).then(async indexFileHandle => {
-            const packIndexFile = await indexFileHandle.getFile()
-            const packIndexBytes = await packIndexFile.bytes()
-            const packIndexPathSegments = packMetaData.index.file.split(pathSeperatorRegex)
-            packIndexPathSegments.pop()
-            const packIndexDirname = packIndexPathSegments.join("/")
-            const packIndex = parseTOML(decode(packIndexBytes)) as PackIndex
-            packIndex.files = [] as any
-            return { packIndex, packIndexDirname, indexFileHandle }
+        if (changes.length === 0) return
+
+        const indexBytes = encode(stringifyTOML(this.packIndexMap.packIndex))
+        packDirectory.writeFile(this.packMetaData.index.file, indexBytes)
+
+        if (useHashes) {
+            this.packMetaData.index.hash = await getHash(indexBytes, this.packMetaData.index["hash-format"]!)
+        }
+
+        const metaBytes = encode(stringifyTOML(this.packMetaData))
+        packDirectory.writeFile(this.packMetaFileName, metaBytes)
+        console.info("changes saved")
+    }
+
+    static async fetchPack(url: string | URL) {
+        console.info("downloading modpack")
+        const packMetaData = await fetch(url).then(async response => {
+            return parseTOML(await assertOK(response).text()) as PackMetaData
         })
-        const pack = Object.assign(new PackwizPack(undefined, undefined, undefined), {
-            packDirectoryHandle,
-            packFileName,
-            packMetaData,
-            packIndex
+        const useHashes = !(packMetaData.options && packMetaData.options["no-internal-hashes"])
+        if (typeof url == "string") url = new URL(url)
+        const packMetaPathSegments = url.pathname.split("/")
+        const packMetaFileName = packMetaPathSegments.pop()
+        const packRoot = packMetaPathSegments.join("/")
+        const packIndexPath = [packRoot, packMetaData.index.file].join("/")
+        url.pathname = packIndexPath
+        const packIndex = await fetch(url).then(async response => {
+            const blob = await assertOK(response).blob()
+            if (useHashes) {
+                const { file, hash, "hash-format": hashFormat } = packMetaData.index
+                if (!hashFormat) throw new Error("index hash format missing")
+                if (!hash) throw new Error("index hash missing")
+                const bytes = await blob.bytes()
+                await hashMatch(bytes, hashFormat, hash, file)
+            }
+            return parseTOML(await blob.text()) as PackIndex
         })
-        const indexDirectoryHandle = await getDirectoryHandle(packDirectoryHandle, packIndexDirname)
-        const directoryHandles = [indexDirectoryHandle]
-        do {
-            const directoryHandle = directoryHandles.pop()
-            await forAsync(directoryHandle.values(), async handle => {
-                if (handle.kind === "directory") {
-                    directoryHandles.push(handle)
+        if (!packIndex.files || packIndex.files.length === 0) {
+            return new PackwizPack({ packMetaData, packIndex, packMetaFileName })
+        }
+        const packIndexRoot = url.pathname.split("/").slice(0, -1).join("/")
+        const packFiles = new Map()
+        for (const indexEntry of packIndex.files) {
+            url.pathname = [packIndexRoot, indexEntry.file].join("/")
+            const blob = await fetch(url).then(async response => {
+                return assertOK(response).blob()
+            })
+            if (useHashes) {
+                const bytes = await blob.bytes()
+                const hashFormat = indexEntry["hash-format"] || packIndex["hash-format"]
+                if (!hashFormat) throw new Error("hash format missing from index")
+                const hash = indexEntry.hash
+                if (!hash) throw new Error(indexEntry.file + " hash missing from index")
+                await hashMatch(bytes, hashFormat, hash, indexEntry.file)
+            }
+            if (!indexEntry.metafile) {
+                packFiles.set(indexEntry.file, blob)
+                continue
+            }
+            const packFileMetaData = parseTOML(await blob.text())
+            packFiles.set(indexEntry.file, packFileMetaData)
+        }
+        console.info("downloaded modpack")
+        return new PackwizPack({ packMetaData, packIndex, packMetaFileName, packFiles })
+    }
+
+    static async loadPack(directoryHandle: FileSystemDirectoryHandle, packMetaFileName: Path) {
+        console.info("loading modpack")
+        const packDirectory = new Directory(directoryHandle)
+        const packMetaData = await packDirectory.readFile(packMetaFileName).then(async file => {
+            return parseTOML(await file.text()) as PackMetaData
+        })
+        const useHashes = !(packMetaData.options && packMetaData.options["no-internal-hashes"])
+
+        const packIndex = await packDirectory.readFile(packMetaData.index.file).then(async file => {
+            if (useHashes) {
+                const { hash, "hash-format": hashFormat } = packMetaData.index
+                if (!hash) throw new Error("index hash missing from " + packMetaFileName)
+                if (!hashFormat) throw new Error("index hash-format missing from " + packMetaFileName)
+                const bytes = await file.bytes()
+                await hashMatch(bytes, hashFormat, hash, packMetaData.index.file)
+            }
+            return parseTOML(await file.text()) as PackIndex
+        })
+
+        if (!packIndex.files || packIndex.files.length === 0) {
+            return new PackwizPack({ packMetaFileName, packMetaData, packIndex })
+        }
+
+        const packIndexDirectoryPath = packMetaData.index.file.split("/").slice(0, -1).join("/")
+        const indexDirectory = await packDirectory.getDirectory(packIndexDirectoryPath)
+
+        const packFiles = new Map<Path, PackFileMetaData | Blob>()
+        for (const indexEntry of packIndex.files) {
+            const file = await indexDirectory.readFile(indexEntry.file)
+            if (useHashes) {
+                const hashFormat = indexEntry["hash-format"] || packIndex["hash-format"]
+                if (!hashFormat) throw new Error("hash-format missing from " + packMetaData.index.file)
+                const hash = indexEntry.hash
+                if (!hash) throw new Error(indexEntry.file + " hash missing from " + packMetaData.index.file)
+                const bytes = await file.bytes()
+                await hashMatch(bytes, hashFormat, hash, indexEntry.file)
+            }
+            if (!indexEntry.metafile) {
+                packFiles.set(indexEntry.file, file)
+                continue
+            }
+            const packFileMetaData = parseTOML(await file.text()) as PackFileMetaData
+            packFiles.set(indexEntry.file, packFileMetaData)
+        }
+        console.info("modpack loaded")
+        return new PackwizPack({ packMetaFileName, packMetaData, packIndex, packFiles })
+    }
+
+    static async refreshPack(directoryHandle: FileSystemDirectoryHandle, packMetaFileName: Path, useHashes?: boolean) {
+        console.info("refreshing modpack")
+        const packDirectory = new Directory(directoryHandle)
+        const packMetaData = await packDirectory.readFile(packMetaFileName).then(async file => {
+            return parseTOML(await file.text()) as PackMetaData
+        })
+        if (useHashes === undefined) useHashes = !(packMetaData.options && packMetaData.options["no-internal-hashes"])
+
+        const packIndex = await packDirectory.readFile(packMetaData.index.file).then(async file => {
+            return parseTOML(await file.text()) as PackIndex
+        })
+        packIndex.files = []
+
+        const packIndexPathSegments = packMetaData.index.file.split("/")
+        const packIndexFileName = packIndexPathSegments.at(-1)
+        const packIndexDirectoryPath = packIndexPathSegments.slice(0, -1).join("/")
+        const indexDirectory = await packDirectory.getDirectory(packIndexDirectoryPath)
+
+        const packFiles = new Map<Path, PackFileMetaData | Blob>()
+        for (const path of await indexDirectory.listFiles()) {
+            if (path === packIndexFileName) continue
+            const indexEntry: PackIndexEntry = { "file": path, metafile: path.endsWith(".pw.toml") }
+            const file = await indexDirectory.readFile(path)
+            if (useHashes) {
+                const hashFormat = packIndex["hash-format"]
+                if (!hashFormat) throw new Error("hash-format missing from " + packMetaData.index.file)
+                const bytes = await file.bytes()
+                indexEntry.hash = await getHash(bytes, hashFormat)
+            }
+            packIndex.files.push(indexEntry)
+            if (!indexEntry.metafile) {
+                packFiles.set(path, file)
+                continue
+            }
+            const packFileMetaData = parseTOML(await file.text()) as PackFileMetaData
+            packFiles.set(path, packFileMetaData)
+        }
+
+        console.info("modpack refreshed")
+        return new PackwizPack({ packMetaFileName, packMetaData, packIndex, packFiles })
+    }
+
+    async detect(apiClient: ApiClient) {
+        console.info("detecting " + apiClient.name + " mods")
+        let modCount = 0
+        for (const hashFormat of apiClient.hashFormats) {
+            const pathsByHash = new Map<Hash, Path>()
+            const workerlessPool = new WorkerlessPool()
+            await forAsync(this.packFiles, async ([path, data]) => {
+                if ("bytes" in data) {
+                    const bytes = await data.bytes()
+                    const hash = await workerlessPool.run(getHash, bytes, hashFormat)
+                    pathsByHash.set(hash, path)
                     return
                 }
-                if (await handle.isSameEntry(indexFileHandle) || await handle.isSameEntry(packFileHandle)) return
-                const pathSegments = await indexDirectoryHandle.resolve(handle)
-                const filePath = pathSegments.join("/")
-                const isMetafile = handle.name.endsWith(".pw.toml")
-                pack.packIndex.files.push({
-                    "file": filePath,
-                    "metafile": isMetafile
-                })
-                const file = await handle.getFile()
-                const fileBytes = await file.bytes()
-                if (isMetafile) pack.packFiles.set(filePath, parseTOML(decode(fileBytes)))
-                else pack.packFiles.set(filePath, fileBytes)
+                //@ts-ignore
+                if (data.update && data.update[apiClient.name]) return
+                if (!("download" in data)) throw new Error(path + " is fucked", { "cause": data })
+                if (data.download["hash-format"] === hashFormat) {
+                    pathsByHash.set(path, data.download.hash)
+                }
             })
-        } while (directoryHandles.length !== 0)
-        pack.packFiles.clearChanged()
-        console.log("pack refreshed")
-        return pack
-    }
-    /** 
-     * Saves the in-memory modpack changes to storage.
-     * You must call this function after making any changes to the pack in order to preserve those changes.
-     * @param hashes whether to calculate and include file hashes in the output.
-     * You should only change this to true when saving the pack for serving over HTTP, where hashes are required.
-     * Otherwise, this option is automatically set from pack options.
-     */
-    async savePack(hashes?: boolean): Promise<PackwizPack> {
-        console.log("saving changes")
-        if (typeof hashes !== "boolean") {
-            hashes = !this.packMetaData.options || !this.packMetaData.options["no-internal-hashes"]
-        }
-        const packIndexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
-        packIndexPathSegments.pop()
-        const packIndexDirname = packIndexPathSegments.join("/") || "."
-        const changeMap = this.packFiles.getChanged()
-        let deletedFileCount = 0
-        await forAsync(Array(this.packIndex.files.length).keys(), async index => {
-            index -= deletedFileCount
-            const entry = this.packIndex.files[index]
-            let data = changeMap.get(entry.file)
-            if (data === undefined) {
-                if (hashes && !entry.hash) {
-                    data = this.packFiles.get(entry.file)
-                    if (!(data instanceof Uint8Array)) data = encode(stringifyTOML(data))
-                    entry.hash = await getHash(data, entry["hash-format"] || this.packIndex["hash-format"])
-                        .then(hash => hash.toString())
-                } else if (!hashes && entry.hash) delete entry.hash
-                return
-            }
-            const filePath = packIndexDirname + "/" + entry.file
-            if (data === null) {
-                this.packIndex.files.splice(index, 1)
-                deletedFileCount++
-                console.debug("deleting " + filePath)
-                await deleteFiles(this.packDirectoryHandle, filePath)
-                if (!changeMap.delete(entry.file)) throw new Error("change map mismatch")
-                return
-            }
-            const fileHandle = await getFileHandle(this.packDirectoryHandle, filePath, { "create": true })
-            if (!(data instanceof Uint8Array)) data = encode(stringifyTOML(data))
-            console.debug("writing " + filePath)
-            await writeFile(fileHandle, data)
-            if (hashes) {
-                entry.hash = await getHash(data, entry["hash-format"] || this.packIndex["hash-format"])
-                    .then(hash => hash.toString())
-            } else {
-                delete entry.hash
-            }
-            if (!changeMap.delete(entry.file)) throw new Error("change map mismatch")
-        })
-        await forAsync(changeMap.entries(), async entry => {
-            let [filePath, data] = entry
-            const fileHandle = await getFileHandle(this.packDirectoryHandle, filePath, { "create": true })
-            if (!(data instanceof Uint8Array)) data = encode(stringifyTOML(data))
-            let hash = undefined
-            if (hashes) {
-                hash = await getHash(data, entry["hash-format"] || this.packIndex["hash-format"])
-                    .then(hash => hash.toString())
-            }
-            console.debug("writing " + filePath)
-            await writeFile(fileHandle, data)
-            this.packIndex.files.push({
-                "file": filePath,
-                "metafile": !(data instanceof Uint8Array),
-                "hash": hash
-            })
-        })
-        this.packFiles.clearChanged()
-        await getFileHandle(this.packDirectoryHandle, this.packMetaData.index.file, { "create": true }).then(async packIndexFileHandle => {
-            const packIndexBytes = encode(stringifyTOML(this.packIndex))
-            if (hashes) {
-                this.packMetaData.index.hash = await getHash(packIndexBytes, this.packMetaData.index["hash-format"])
-                    .then(hash => hash.toString())
-            } else {
-                delete this.packMetaData.index.hash
-            }
-            console.debug("writing " + this.packMetaData.index.file)
-            await writeFile(packIndexFileHandle, packIndexBytes)
-        })
-        console.debug("writing "+this.packFileName)
-        await getFileHandle(this.packDirectoryHandle, this.packFileName, { "create": true })
-            .then(packFileHandle => writeFile(packFileHandle, encode(stringifyTOML(this.packMetaData))))
-        console.log("changes saved")
-        return this
-    }
-    /** 
-     * Hashes all non-metafiles in the pack and makes a request to curseforge to find matching files.
-     * Any file that matches is removed from the modpack and replaced with a metafile referencing the curseforge file.
-     * @param apiKey required to access the curseforge api, can be retreived from [the curseforge console](https://console.curseforge.com/#/api-keys)
-     */
-    async findAndReplaceCurseforgeFiles(apiKey: string): Promise<PackwizPack> {
-        console.log("finding and replacing curseforge files")
-        const workerlessPool = new WorkerlessPool()
-        const indexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
-        indexPathSegments.pop()
-        const hashPaths = new Map<number, Path>()
-        await forAsync(this.packIndex.files, async entry => {
-            if (entry.metafile) return
-            if(!(entry.file.endsWith(".zip")||entry.file.endsWith(".jar"))) return
-            const bytes = this.packFiles.get(entry.file) as Uint8Array<ArrayBuffer>
-            const hash = await workerlessPool.run(getHash, bytes, "murmur2") as number
-            hashPaths.set(hash, entry.file)
-        })
-        workerlessPool.terminate()
-        if (hashPaths.size === 0) {
-            console.log("no files to check")
-            return this
-        }
-        console.log("requesting info on " + hashPaths.size + " files")
-        const curseforge = new CurseforgeV1Client(apiKey)
-        const result = await curseforge.getFingerprintsMatchesByGameId(minecrafCurseforgeGameID, [...hashPaths.keys()])
-        console.debug("got " + result.exactMatches.length + " matches")
-        if (result.exactMatches.length === 0) return this
-        const modIDs: number[] = []
-        for (const match of result.exactMatches) modIDs.push(match.file.modId)
-        const mods = new Map<number, Mod>()
-        for (const mod of await curseforge.getMods(modIDs)) mods.set(mod.id, mod)
-        let replacedFileCount = 0
-        await forAsync(result.exactMatches, async match => {
-            const filePath = hashPaths.get(match.file.fileFingerprint)
-            if (!filePath) return
-            const mod = mods.get(match.file.modId)
-            const filePathSegments = filePath.split(pathSeperatorRegex)
-            filePathSegments.pop()
-            const fileDirname = filePathSegments.join("/")
-            const newFilePath = fileDirname + "/" + mod.slug + ".pw.toml"
-            this.packFiles.delete(filePath)
-            for (const hash of match.file.hashes) {
-                if (hash.algo === 1) {
-                    this.packFiles.set(newFilePath, {
-                        "filename": match.file.fileName,
-                        "name": mod.name,
-                        "download": {
-                            "hash": hash.value,
-                            "hash-format": "sha1",
-                            "mode": "metadata:curseforge"
-                        },
-                        "update": {
-                            "curseforge": {
-                                "file-id": match.file.id,
-                                "project-id": match.file.modId
-                            }
-                        },
-                        "option": {
-                            "optional": false,
-                            "default": true,
-                            "description": mod.summary
-                        }
-                    })
-                    replacedFileCount++
-                    break
+            workerlessPool.terminate()
+            for (const [hash, packFileMetaData] of await apiClient.detect(pathsByHash.keys(), hashFormat)) {
+                const path = pathsByHash.get(hash)
+                if (!path) {
+                    console.debug("unrecognized match: " + packFileMetaData.filename)
+                    continue
                 }
-            }
-        })
-        console.log("replaced " + replacedFileCount + " files")
-        return this
-    }
-    /**
-     * Hashes all non-metafiles in the pack and makes a request to modrinth to find matching files.
-     * Any file that matches is removed from the modpack and replaced with a metafile referencing the modrinth file.
-     */
-    async findAndReplaceModrinthFiles(): Promise<PackwizPack> {
-        console.log("finding and replacing modrinth files")
-        const indexPathSegments = this.packMetaData.index.file.split(pathSeperatorRegex)
-        indexPathSegments.pop()
-        const hashPaths = new Map<Hash, Path>()
-        await forAsync(this.packIndex.files, async entry => {
-            if (entry.metafile) return
-            if (!(entry.file.endsWith(".zip") || entry.file.endsWith(".jar"))) return
-            const bytes = this.packFiles.get(entry.file) as Uint8Array<ArrayBuffer>
-            const hash = await getHash(bytes, "sha512") as string
-            hashPaths.set(hash, entry.file)
-        })
-        if (hashPaths.size === 0) {
-            console.log("no files to check")
-            return this
-        }
-        console.log("requesting info on " + hashPaths.size + " files")
-        const modrinth = new ModrinthV2Client()
-        const matches = await modrinth.getProjectVersionsByHash([...hashPaths.keys()])
-        const projectIDs: string[] = []
-        for (const match of Object.values(matches)) projectIDs.push(match.project_id)
-        const projects = new Map<string, Project>()
-        for (const project of await modrinth.getProjects(projectIDs)) projects.set(project.id, project)
-        console.log("got " + projectIDs.length + " matches")
-        if (projectIDs.length === 0) return this;
-        let replacedFileCount = 0
-        await forAsync(Object.entries(matches), async entry => {
-            const [hash, match] = entry
-            const filePath = hashPaths.get(hash)
-            if (!filePath) return
-            const mod = projects.get(match.project_id)
-            const filePathSegments = filePath.split(pathSeperatorRegex)
-            filePathSegments.pop()
-            const fileDirname = filePathSegments.join("/")
-            const newFilePath = fileDirname + "/" + mod.slug + ".pw.toml"
-            this.packFiles.delete(filePath)
-            let side: Side = "both"
-            if (mod.server_side === "unsupported") {
-                side = "client"
-            } else if (mod.client_side === "unsupported") {
-                side = "server"
-            }
-            for (const file of match.files) {
-                if (file.hashes["sha512"] === hash) {
-                    this.packFiles.set(newFilePath, {
-                        "filename": file.filename,
-                        "name": mod.title,
-                        "download": {
-                            "hash": file.hashes["sha1"],
-                            "hash-format": "sha1",
-                            "url": file.url
-                        },
-                        "update": {
-                            "modrinth": {
-                                "mod-id": mod.id,
-                                "version": match.id
-                            }
-                        },
-                        "option": {
-                            "optional": false,
-                            "description": mod.description
-                        }
-                    })
+                const data = this.packFiles.get(path)
+                if (!data) throw new Error("continuity error")
+                if ("bytes" in data) {
+                    this.packFiles.delete(path)
+                    this.packFiles.set(path + ".pw.toml", packFileMetaData)
+                    modCount++
+                    continue
                 }
+                Object.assign(data, packFileMetaData)
+                this.packFiles.set(path, data)
+                modCount++
             }
-            replacedFileCount++
-        })
-        console.log("replaced " + replacedFileCount + " files")
-        return this
+        }
+        console.info("detected " + modCount + " mods")
     }
-    /**
-     * In order to comply with curseforge API requirements, direct download URLs to curseforge files cannot be saved into the pack.
-     * Instead, you may use this function beforehand to cache the download URLs in memory before attempting to install the pack.
-     * @param apiKey required to access the curseforge api, can be retreived from [the curseforge console](https://console.curseforge.com/#/api-keys)
-     */
-    async cacheCurseforgeDownloadURLs(apiKey: string): Promise<PackwizPack> {
-        console.log("caching curseforge download URLs")
-        const fileIDPaths = new Map<number, Path>()
-        for (const [filePath, metafiledata] of this.packFiles.entries()) {
-            if (metafiledata instanceof Uint8Array) continue
-            if (!metafiledata.download.mode) continue
-            fileIDPaths.set(metafiledata.update.curseforge["file-id"], filePath)
-        }
-        if (fileIDPaths.size === 0) {
-            console.log("no files to check")
-            return this
-        }
-        console.log("requesting info on " + fileIDPaths.size + " files")
-        const curseforge = new CurseforgeV1Client(apiKey)
-        let cachedURLcount = 0
-        for (const file of await curseforge.getFiles([...fileIDPaths.keys()])) {
-            if (file.downloadUrl) {
-                cachedURLcount++
-                const filePath = fileIDPaths.get(file.id)
-                const packFileMetaData = this.packFiles.get(filePath) as PackFileMetaData
-                packFileMetaData.download.url = file.downloadUrl
-                delete packFileMetaData.download.mode
-                this.packFiles.set(filePath, packFileMetaData)
-            }
-        }
-        console.log("cached " + cachedURLcount + " URLs")
-        return this
-    }
-    /**
-     * Uses the hashes stored in any non-modrinth metafiles to find files that are also availible from modrinth,
-     * then adds modrinth metadata (client/server side, download url, etc) to that file.
-     * 
-     * Useful for metafiles that don't already have that meatadata availible, or for exporting your pack to multiple platforms.
-     */
-    async mergeModrinthMetadata(): Promise<PackwizPack> {
-        console.log("merging modrinth metadata with curseforge files")
-        const hashPaths = new Map<Hash, Path>()
-        for (const [filePath, metafiledata] of this.packFiles.entries()) {
-            if (metafiledata instanceof Uint8Array) continue
-            if (metafiledata.update.modrinth) continue
-            hashPaths.set(metafiledata.download.hash, filePath)
-        }
-        if (hashPaths.size == 0) {
-            console.log("no files to check")
-            return this
-        }
-        console.log("requesting info on " + hashPaths.size + " files")
-        const modrinth = new ModrinthV2Client()
-        const matches = await modrinth.getProjectVersionsByHash([...hashPaths.keys()])
-        const projectIDs: string[] = []
-        for (const match of Object.values(matches)) projectIDs.push(match.project_id)
-        const projects = new Map<string, Project>()
-        for (const project of await modrinth.getProjects(projectIDs)) projects.set(project.id, project)
-        let mergedFileCount = 0
-        for (const record of Object.entries(matches)) {
-            const [hash, match] = record
-            const filePath = hashPaths.get(hash)
-            if (!filePath) return
-            mergedFileCount++
-            const mod = projects.get(match.project_id)
-            let side: Side = "both"
-            if (mod.server_side === "unsupported") {
-                side = "client"
-            } else if (mod.client_side === "unsupported") {
-                side = "server"
-            }
-            for (const file of match.files) {
-                if (file.hashes["sha1"] === hash) {
-                    const metafiledata = this.packFiles.get(filePath) as PackFileMetaData
-                    delete metafiledata.download.mode
-                    metafiledata.download.url = file.url
-                    metafiledata.side = side
-                    metafiledata.update.modrinth = {
-                        "mod-id": mod.id,
-                        "version": match.id
-                    }
-                    this.packFiles.set(filePath, metafiledata)
-                    break
-                }
-            }
-        }
-        console.log("merged " + mergedFileCount + " files")
-        return this
-    }
-}
+}
+
+export default PackwizPack

lib/src/types.ts

@@ -1,3 +1,5 @@
+import type { Blob } from "@sugoidogo/importable-types-web"
+
 /**
  * The hash algorithm used to determine if a file is valid.
  * All functions listed must be supported by tools implementing the packwiz pack format.
@@ -178,41 +180,42 @@ export interface PackIndex {
      * The files listed in this index.
      * If it is not defined, defaults to an empty list.
      */
-    "files"?: [{
-        /**
-         * The path to the file to be downloaded,
-         * relative to this index file.
-         * @see {@link Path}
-         */
-        "file": Path
-        /** The hash of the specified file, as a string. @see {@link Hash} */
-        "hash"?: Hash
-        /**
-         * The name with which this file should be downloaded,
-         * instead of the filename specified in the path.
-         * Not compatible with metafile,
-         * and may not be very well supported.
-         */
-        "alias"?: Path
-        /**
-         * The hash format for the hash of the specified file.
-         * Defaults to the hash format specified in the index -
-         * ideally remove this value if it is equal to the hash format for the index to save space.
-         * @see {@link HashFormat}
-         */
-        "hash-format"?: HashFormat
-        /**
-         * True when this entry points to a .toml metadata file,
-         * which references a file outside the pack.
-         */
-        "metafile"?: boolean
-        /**
-         * When this is set to true,
-         * the file is not overwritten if it already exists,
-         * to preserve changes made by a user.
-         */
-        "preserve"?: boolean
-    }]
+    "files"?: PackIndexEntry[]
+}
+export interface PackIndexEntry {
+    /**
+     * The path to the file to be downloaded,
+     * relative to this index file.
+     * @see {@link Path}
+     */
+    "file": Path
+    /** The hash of the specified file, as a string. @see {@link Hash} */
+    "hash"?: Hash
+    /**
+     * The name with which this file should be downloaded,
+     * instead of the filename specified in the path.
+     * Not compatible with metafile,
+     * and may not be very well supported.
+     */
+    "alias"?: Path
+    /**
+     * The hash format for the hash of the specified file.
+     * Defaults to the hash format specified in the index -
+     * ideally remove this value if it is equal to the hash format for the index to save space.
+     * @see {@link HashFormat}
+     */
+    "hash-format"?: HashFormat
+    /**
+     * True when this entry points to a .toml metadata file,
+     * which references a file outside the pack.
+     */
+    "metafile"?: boolean
+    /**
+     * When this is set to true,
+     * the file is not overwritten if it already exists,
+     * to preserve changes made by a user.
+     */
+    "preserve"?: boolean
 }
 /**
  * A metadata file which references an external file from a URL.
@@ -319,3 +322,14 @@ export interface PackFileMetaData {
         }
     }
 }
+
+/**
+ * This interface allows packwiz-extras to support common functions of multiple mod hosting websites.
+ * Developers can implement this interface in their API client class,
+ * or as an adapter for a third party API.
+ */
+export interface ApiClient {
+    name:string
+    hashFormats: HashFormat[]
+    detect: (hashes: Iterable<Hash>, hashFormat: HashFormat) => Promise<Iterable<[Hash,PackFileMetaData]>>
+}