diff --git a/package-lock.json b/package-lock.json index 9365406..d26c0b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.2.0", "license": "LGPL-3.0-or-later", "dependencies": { + "@modrinth-ts/lib": "^2.1.0", "@sugoidogo/js-util": "^0.3.0", "@sugoidogo/node-file-system-adapter": "^1.0.1", "commander": "^14.0.2", @@ -1115,6 +1116,18 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@modrinth-ts/lib": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@modrinth-ts/lib/-/lib-2.1.0.tgz", + "integrity": "sha512-W+HGQgQSmK/bmQO2hjEQROwMRZUSH+uigke4Zat1DzeBjsKGIotEy2inuIXX+b0uh6SqRwb4grpbJKk/VPMT8Q==", + "license": "MIT", + "dependencies": { + "axios": "^1.13.1" + }, + "peerDependencies": { + "typescript": "^5.9.3" + } + }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -2142,7 +2155,6 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, "license": "Apache-2.0", "peer": true, "bin": { diff --git a/package.json b/package.json index 6cbdadb..33103ae 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "docs": "typedoc src/lib/packwiz.ts --tsconfig src/lib/tsconfig.json" }, "dependencies": { + "@modrinth-ts/lib": "^2.1.0", "@sugoidogo/js-util": "^0.3.0", "@sugoidogo/node-file-system-adapter": "^1.0.1", "commander": "^14.0.2", diff --git a/src/cli/tsconfig.json b/src/cli/tsconfig.json index 8c91052..758a72a 100644 --- a/src/cli/tsconfig.json +++ b/src/cli/tsconfig.json @@ -4,8 +4,9 @@ "module": "es2022", "moduleResolution": "node", "lib": ["ESNext"], - "noEmit": true, - "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, "types": [ "node" ] diff --git a/src/lib/packwiz.ts b/src/lib/packwiz.ts index 77fa12a..3f51bdb 100644 --- a/src/lib/packwiz.ts +++ b/src/lib/packwiz.ts @@ -92,9 +92,9 @@ export interface Pack { /** The path to the file that contains the index. @see {@link Path} */ "file": Path /** The hash format for the hash of the index file. @see {@link HashFormat} */ - "hash-format": HashFormat + "hash-format"?: HashFormat /** The hash of the index file, as a string. @see {@link Hash} */ - "hash": Hash + "hash"?: Hash } /** * The versions of components used by this modpack - usually Minecraft and the mod loader this pack uses. @@ -165,7 +165,7 @@ export interface Pack { */ export interface Index { /** The default hash format for every file in the index. @see {@link HashFormat} */ - "hash-format": HashFormat + "hash-format"?: HashFormat /** * The files listed in this index. * If it is not defined, defaults to an empty list. @@ -178,7 +178,7 @@ export interface Index { */ "file": Path /** The hash of the specified file, as a string. @see {@link Hash} */ - "hash": Hash + "hash"?: Hash /** * The name with which this file should be downloaded, * instead of the filename specified in the path. @@ -306,12 +306,87 @@ export interface Mod { } } +const textEncoder = new TextEncoder() + +class Modpack { + pack: Pack + index: Index + metafiles: { [key: Path]: Mod } + // initialize this modpack with default values + async init() { + this.index = { + 'hash-format': 'sha256' + } + this.pack = { + 'name': 'modpack', + 'pack-format': 'packwiz:1.1.0', + 'index': { + 'file': 'index.toml', + 'hash-format': 'sha256', + 'hash': await getHash(textEncoder.encode(stringifyTOML(this.index)), 'sha256') + }, + 'versions': { + 'minecraft': '1.21.11' // TODO get latest minecraft version + } + } + this.metafiles = {} + } + // load modpack from storage + async load(pack_directory_handle: FileSystemDirectoryHandle, pack_file_name = 'pack.toml') { + { + const pack_file_handle = await pack_directory_handle.getFileHandle(pack_file_name) + const file = await pack_file_handle.getFile() + const text = await file.text() + this.pack = parseTOML(text) + } + { + const { dirname, basename } = splitPath(this.pack.index.file) + const index_directory_handle = await getDirectoryHandle(pack_directory_handle, dirname) + const index_file_handle = await index_directory_handle.getFileHandle(basename) + const file = await index_file_handle.getFile() + const text = await file.text() + this.index = parseTOML(text) + await forAsync(this.index.files, async entry => { + if (!entry.metafile) return + const file_handle = await getFileHandle(index_directory_handle, entry.file) + const file = await file_handle.getFile() + const text = await file.text() + this.metafiles[entry.file] = parseTOML(text) + }) + } + } + // save modpack to storage + async save(pack_directory_handle: FileSystemDirectoryHandle, pack_file_name = 'pack.toml') { + { + const pack_file_handle = await pack_directory_handle.getFileHandle(pack_file_name, { create: true }) + const writeable = await pack_file_handle.createWritable() + await writeable.write(stringifyTOML(this.pack)) + await writeable.close() + } + { + const { dirname, basename } = splitPath(this.pack.index.file) + const index_directory_handle = await getDirectoryHandle(pack_directory_handle, dirname, { create: true }) + const index_file_handle = await index_directory_handle.getFileHandle(basename) + const writeable = await index_file_handle.createWritable() + writeable.write(stringifyTOML(this.index)) + writeable.close() + await forAsync(Object.entries(this.metafiles), async ([path,mod]) => { + const mod_file_handle = await getFileHandle(index_directory_handle, path, {create:true}) + const writeable = await mod_file_handle.createWritable() + await writeable.write(stringifyTOML(mod)) + await writeable.close() + }) + } + } +} + import WorkerlessPool from 'workerless' import { parseTOML, stringifyTOML } from 'confbox' import { CFV2Client } from 'curseforge-v2' import { forAsync } from '@sugoidogo/js-util' +import * as modrinth from '@modrinth-ts/lib/dist/index' -export async function getHash(data: Uint8Array, format:HashFormat): Promise { +export async function getHash(data: Uint8Array, format: HashFormat): Promise { if (format.startsWith('sha')) { // copied from https://developer.mozilla.org/en-US/docs/Web/API/Web_Crypto_API/Non-cryptographic_uses_of_subtle_crypto#hashing_a_file const hashAsArrayBuffer = await crypto.subtle.digest("SHA-" + format.substring(3), data); const uint8ViewOfHash = new Uint8Array(hashAsArrayBuffer); @@ -333,33 +408,33 @@ export async function getHash(data: Uint8Array, format:HashFormat): const md5 = await import('md5') return md5(data) } - throw new Error('unknown hash format: '+format) + throw new Error('unknown hash format: ' + format) } function splitPath(path: string) { - path=path.replaceAll('\\','/') + path = path.replaceAll('\\', '/') const path_segments = path.split('/') const basename = path_segments.pop() const dirname = path_segments.join('/') - return {dirname,basename} + return { dirname, basename } } -async function getDirectoryHandle(dir: FileSystemDirectoryHandle, path: string) { +async function getDirectoryHandle(dir: FileSystemDirectoryHandle, path: string, options?: FileSystemGetDirectoryOptions) { if (!path) { return dir } path = path.replaceAll('\\', '/') const path_segments = path.split('/') for (const segment of path_segments) { - dir = await dir.getDirectoryHandle(segment) + dir = await dir.getDirectoryHandle(segment, options) } return dir } -async function getFileHandle(dir: FileSystemDirectoryHandle, path: string, options?:FileSystemGetFileOptions) { +async function getFileHandle(dir: FileSystemDirectoryHandle, path: string, options?: FileSystemGetFileOptions) { const { dirname, basename } = splitPath(path) - dir = await getDirectoryHandle(dir, dirname) - return dir.getFileHandle(basename,options) + dir = await getDirectoryHandle(dir, dirname, options) + return dir.getFileHandle(basename, options) } async function removeEntry(dir: FileSystemDirectoryHandle, path: string) { @@ -374,19 +449,19 @@ async function removeEntry(dir: FileSystemDirectoryHandle, path: string) { * @param cfApiKey curseforge api key * @param size_min minimum size for a file to be checked */ -export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: string, pack_file_name='pack.toml', size_min = 4096): Promise { +export async function cfDetect(pack_dir: FileSystemDirectoryHandle, cfApiKey: string, pack_file_name = 'pack.toml', size_min = 4096): Promise { console.log('loading pack') - const pack_file_handle = await getFileHandle(pack_dir,pack_file_name) - const pack:Pack = await pack_file_handle.getFile() + const pack_file_handle = await getFileHandle(pack_dir, pack_file_name) + const pack: Pack = await pack_file_handle.getFile() .then(file => file.text()) .then(text => parseTOML(text)) console.log('loading index') const { dirname: index_dir_name, basename: index_file_name } = splitPath(pack.index.file) let index_dir: FileSystemDirectoryHandle; if (!index_dir_name) index_dir = pack_dir - else index_dir = await getDirectoryHandle(pack_dir,index_dir_name) - const index_file_handle = await getFileHandle(index_dir,index_file_name) - const index:Index = await index_file_handle.getFile() + else index_dir = await getDirectoryHandle(pack_dir, index_dir_name) + const index_file_handle = await getFileHandle(index_dir, index_file_name) + const index: Index = await index_file_handle.getFile() .then(file => file.text()) .then(text => parseTOML(text)) if (!index.files) { @@ -409,8 +484,8 @@ export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: str console.log('requesting matches from curseforge') const response = await new CFV2Client({ apiKey: cfApiKey }).getFingerprintMatches({ 'fingerprints': Array.from(file_hash_map.keys()) }) const textEncoder = new TextEncoder() - let resultCount=0 - await forAsync(Object.values(response.data.data.exactMatches),async function(match){ + let resultCount = 0 + await forAsync(Object.values(response.data.data.exactMatches), async function (match) { let file_path = file_hash_map.get(match.file.fileFingerprint) if (!file_path) { for (const module of match.file.modules) { @@ -444,7 +519,7 @@ export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: str const new_file_path = file_dir_name + '/' + encodeURI(match.file.displayName.toLowerCase().replaceAll(' ', '-') + '.pw.toml') const new_file_data = textEncoder.encode(stringifyTOML(mod)) const new_file_hash = await getHash(new_file_data, index['hash-format']) - await getFileHandle(index_dir,new_file_path,{'create':true}) + await getFileHandle(index_dir, new_file_path, { 'create': true }) .then(handle => handle.createWritable()) .then(stream => { stream.write(new_file_data) @@ -455,9 +530,9 @@ export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: str "metafile": true }) }) - await removeEntry(index_dir,file_path) + await removeEntry(index_dir, file_path) .then(() => { - for (let i = 0; i < index.files.length; i++){ + for (let i = 0; i < index.files.length; i++) { if (index.files[i].file != file_path) continue index.files.splice(i, 1) return @@ -469,7 +544,7 @@ export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: str } } }) - const index_file_data=textEncoder.encode(stringifyTOML(index)) + const index_file_data = textEncoder.encode(stringifyTOML(index)) const index_file_hash = await getHash(index_file_data, pack.index['hash-format']) await index_file_handle.createWritable() .then(async stream => { @@ -490,81 +565,124 @@ export async function cfDetect(pack_dir:FileSystemDirectoryHandle, cfApiKey: str * @param size_min minimum size for a file to be checked * @returns */ -export async function mrDetect(pack_url: string, mrApiKey?: string, size_min = 4096): Promise { +export async function mrDetect(pack_dir: FileSystemDirectoryHandle, mrApiKey?: string, pack_file_name = 'pack.toml', size_min = 4096) { console.log('loading pack') - const pack = await fetch_toml(pack_url) as Pack - const pack_dir = dirname(pack_url) - const index_url = `${pack_dir}/${pack.index.file}` + const pack_file_handle = await getFileHandle(pack_dir, pack_file_name) + const pack: Pack = await pack_file_handle.getFile() + .then(file => file.text()) + .then(text => parseTOML(text)) console.log('loading index') - const index = await fetch_toml(index_url) as Index - const index_dir = dirname(index_url) + const { dirname: index_dir_name, basename: index_file_name } = splitPath(pack.index.file) + let index_dir: FileSystemDirectoryHandle; + if (!index_dir_name) index_dir = pack_dir + else index_dir = await getDirectoryHandle(pack_dir, index_dir_name) + const index_file_handle = await getFileHandle(index_dir, index_file_name) + const index: Index = await index_file_handle.getFile() + .then(file => file.text()) + .then(text => parseTOML(text)) if (!index.files) { - console.warn(`${index_url} has no files indexed`) - return {} + console.warn(`modpack has no files indexed`) + return } const file_hash_map = new Map() console.log('hashing files') - await Promise.all(index.files.map(async file => { - if (file.metafile) return - const file_url = `${index_dir}/${file.file}` - const file_data = await fetch_bytes(file_url) - if (file_data.length < size_min) return - const hash=await sha1(file_data) - file_hash_map.set(hash, file_url) - })) - console.log('requesting matches from modrinth') - const matches = await fetch_json('https://api.modrinth.com/v2/version_files', { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'accept': 'application/json', - 'authorization': mrApiKey - }, - body: JSON.stringify({ - "algorithm": "sha1", - "hashes": Array.from(file_hash_map.keys()) - }) + await forAsync(index.files, async function (entry) { + if (entry.metafile) return + const file = await getFileHandle(index_dir, entry.file) + .then(handle => handle.getFile()) + if (file.size < size_min) return + const file_data = await file.bytes() + const hash = await getHash(file_data, "sha1") + file_hash_map.set(hash, entry.file) }) - const project_matches = new Map() - for (const match of Object.values(matches) as any[]) project_matches.set(match.project_id, match) - const projects = await fetch_json(`https://api.modrinth.com/v2/projects?ids=["${Array.from(project_matches.keys()).join('","')}"]`) - - const result: Result = { delete: [], write: {} } - for (const project of projects) { - const match = project_matches.get(project.id) - if (!match) { - console.debug("modrinth returned a project result we didn't ask for, skipping", project) - continue + console.log('requesting matches from modrinth') + const matches = await modrinth.getLatestVersionsFromHashes({ + 'auth': mrApiKey, + 'body': { + "algorithm": "sha1", + "hashes": Array.from(file_hash_map.keys()), + "game_versions": [], + "loaders": [] } - const file_url = file_hash_map.get(match.files[0].hashes.sha1) - if (!file_url) { - console.debug("modrinth returned a file result we didn't ask for, skipping", match) - continue + }) + const version_matches = new Map() + for (const match of Object.values(matches.data)) version_matches.set(match.project_id, match) + const projects = await modrinth.getProjects({ + 'auth': mrApiKey, + query: { + ids: JSON.stringify(Array.from(version_matches.keys())) + } + }) + const textEncoder = new TextEncoder() + let resultCount = 0 + await forAsync(projects.data, async function (project) { + const version = version_matches.get(project.id) + if (!version) { + console.debug("modrinth returned a project result we didn't ask for, skipping", project) + return + } + const file_path = file_hash_map.get(version.files[0].hashes.sha1) + if (!file_path) { + console.debug("modrinth returned a file result we didn't ask for, skipping", version) + return } let side: Side = "both" if (project.client_side === "unsupported") side = "server" if (project.server_side === "unsupported") side = "client" const mod: Mod = { "name": project.title, - "filename": match.files[0].filename, + "filename": version.files[0].filename, "side": side, "download": { "hash-format": "sha1", - "hash": match.files[0].hashes.sha1, - "url": match.files[0].url + "hash": version.files[0].hashes.sha1, + "url": version.files[0].url }, "update": { "modrinth": { - "version": match.id, - "mod-id": match.project_id + "version": version.id, + "mod-id": version.project_id } } } - result.delete.push(file_url) - result.write[index_dir + '/' +project.title.toLowerCase().replaceAll(' ', '-')] = mod - } - console.log(`found ${result.delete.length} matching files`) - return result + const new_file_path = splitPath(file_path).dirname + '/' + encodeURI(project.title.toLowerCase().replaceAll(' ', '-') + '.pw.toml') + const new_file_data = textEncoder.encode(stringifyTOML(mod)) + const new_file_hash = await getHash(new_file_data, index['hash-format']) + await getFileHandle(index_dir, new_file_path, { 'create': true }) + .then(handle => handle.createWritable()) + .then(stream => { + stream.write(new_file_data) + stream.close() + index.files.push({ + "file": new_file_path, + "hash": new_file_hash, + "metafile": true + }) + }) + await removeEntry(index_dir, file_path) + .then(() => { + for (let i = 0; i < index.files.length; i++) { + if (index.files[i].file != file_path) continue + index.files.splice(i, 1) + return + } + throw new Error("can't find removed file in index") + }) + resultCount++ + }) + const index_file_data = textEncoder.encode(stringifyTOML(index)) + const index_file_hash = await getHash(index_file_data, pack.index['hash-format']) + await index_file_handle.createWritable() + .then(async stream => { + await stream.write(index_file_data) + await stream.close() + pack.index.hash = index_file_hash.toString() + return pack_file_handle.createWritable() + }).then(async stream => { + await stream.write(stringifyTOML(pack)) + await stream.close() + }) + console.log(`found ${resultCount} matching files`) } /** * find download urls for curseforge files @@ -586,13 +704,13 @@ export async function cfUrl(pack_url: string, cfApiKey: string): Promise const mod_paths = new Map() console.log('loading curseforge files') await Promise.all(index.files.map(async file => { - if(!file.metafile) return + if (!file.metafile) return const file_url = `${index_dir}/${file.file}` const mod = await fetch_toml(file_url) as Mod if (!mod.update || !mod.update.curseforge) { return } - mod_paths.set(mod.update.curseforge['file-id'],{mod,file_url}) + mod_paths.set(mod.update.curseforge['file-id'], { mod, file_url }) })) console.log(`requesting ${mod_paths.size} download urls`) const response = await fetch_json('https://api.curseforge.com/v1/mods/files', { @@ -602,7 +720,7 @@ export async function cfUrl(pack_url: string, cfApiKey: string): Promise 'accept': 'application/json', 'x-api-key': cfApiKey }, - body: JSON.stringify({'fileIds':Array.from(mod_paths.keys())}) + body: JSON.stringify({ 'fileIds': Array.from(mod_paths.keys()) }) }) const result: Result = { write: {} } for (const file of response.data) { @@ -636,14 +754,14 @@ export async function mrMerge(pack_url: string, mrApiKey?: string): Promise() @@ -670,10 +788,10 @@ export async function mrMerge(pack_url: string, mrApiKey?: string): Promise