From 2fabd04a2e2dfbd34b129f2505f40e19c89d3a01 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:54:26 +0100 Subject: [PATCH] Remove Daedalus changes from frontend branch --- apps/daedalus_client/src/error.rs | 2 + apps/daedalus_client/src/fabric.rs | 393 +++++++++++++++----- apps/daedalus_client/src/forge.rs | 78 +++- apps/daedalus_client/src/main.rs | 249 +++++++++++-- apps/daedalus_client/src/metadata_groups.rs | 145 ++++++++ apps/daedalus_client/src/util.rs | 86 ++++- packages/daedalus/src/modded.rs | 82 +++- 7 files changed, 879 insertions(+), 156 deletions(-) create mode 100644 apps/daedalus_client/src/metadata_groups.rs diff --git a/apps/daedalus_client/src/error.rs b/apps/daedalus_client/src/error.rs index 6765341381..6afb321df4 100644 --- a/apps/daedalus_client/src/error.rs +++ b/apps/daedalus_client/src/error.rs @@ -27,6 +27,8 @@ pub enum ErrorKind { inner: Box, file: String, }, + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), #[error("Error acquiring semaphore: {0}")] Acquire(#[from] tokio::sync::AcquireError), #[error("Tracing error: {0}")] diff --git a/apps/daedalus_client/src/fabric.rs b/apps/daedalus_client/src/fabric.rs index 3d6f1eeb0c..8d9362bb73 100644 --- a/apps/daedalus_client/src/fabric.rs +++ b/apps/daedalus_client/src/fabric.rs @@ -1,3 +1,19 @@ +//! Fetches Fabric-compatible loader metadata. +//! +//! Fabric and Quilt both expose loader profiles for a concrete Minecraft +//! version, but Daedalus publishes templated profiles using +//! `${modrinth.gameVersion}`. A group is a set of Minecraft versions whose +//! upstream loader profiles have the same structure after the concrete +//! Minecraft version is replaced with `${modrinth.gameVersion}`. Fabric uses +//! one universal group, so its public profile paths stay as +//! `versions/{loader}.json`. Quilt has more than one group: versions before +//! 26.x include hashed/intermediary libraries, while 26.x versions do not. For +//! Quilt, Daedalus writes one templated profile per group at +//! `version-group/{group}/loader-version/{loader}`. + +use crate::metadata_groups::{ + UNIVERSAL_METADATA_GROUP, metadata_group_for_game_version, metadata_groups, +}; use crate::util::{download_file, fetch_json, format_url}; use crate::{ Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact, @@ -64,7 +80,112 @@ async fn fetch( &semaphore, ) .await?; + let all_loader_versions = fabric_manifest.loader.clone(); + let all_game_versions = fabric_manifest.game.clone(); + let metadata_groups = metadata_groups( + mod_loader, + all_game_versions.iter().map(|x| x.version.as_str()), + ); + if metadata_groups + .iter() + .any(|group| group.id != UNIVERSAL_METADATA_GROUP) + { + let loaders = all_loader_versions + .iter() + .filter(|x| !skip_versions.contains(&&*x.version)) + .collect::>(); + + let profile_requests = metadata_groups + .iter() + .flat_map(|group| { + loaders.iter().map(move |loader| ProfileRequest { + group: group.id.to_string(), + loader_profile_template_game_version: group + .loader_profile_template_game_version + .clone(), + game_versions: group.game_versions.clone(), + loader_version: loader.version.clone(), + url: format!( + "{}/versions/loader/{}/{}/profile/json", + meta_url, + group.loader_profile_template_game_version, + loader.version + ), + }) + }) + .collect::>(); + + fetch_metadata_profiles( + mod_loader, + format_version, + maven_url, + profile_requests, + &upload_files, + &mirror_artifacts, + &semaphore, + ) + .await?; + + let version_groups = metadata_groups + .iter() + .map(|group| daedalus::modded::VersionGroup { + id: group.id.to_string(), + loaders: loaders + .iter() + .map(|loader| { + let version_path = metadata_version_path( + mod_loader, + format_version, + &loader.version, + group.id, + ); + + daedalus::modded::LoaderVersion { + id: loader.version.clone(), + url: format_url(&version_path), + stable: loader.stable, + } + }) + .collect(), + }) + .collect(); + + let manifest = daedalus::modded::Manifest { + game_versions: all_game_versions + .into_iter() + .map(|game_version| { + let group = metadata_group_for_game_version( + &metadata_groups, + mod_loader, + &game_version.version, + ) + .expect("game version should have a metadata group"); + + daedalus::modded::Version { + id: game_version.version.clone(), + stable: game_version.stable, + version_group: Some(group.id.to_string()), + loaders: Vec::new(), + } + }) + .collect(), + version_groups, + }; + + upload_files.insert( + format!("{mod_loader}/v{format_version}/manifest.json"), + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), + content_type: Some("application/json".to_string()), + }, + ); + + return Ok(FetchResult { + upload_files, + mirror_artifacts, + }); + } // We check Modrinth's manifest to find newly added loader versions, // intermediary/mapping artifacts, and game versions. let ( @@ -125,8 +246,6 @@ async fn fetch( ) }; - const DUMMY_GAME_VERSION: &str = "1.21"; - if !fetch_intermediary_versions.is_empty() { for x in &fetch_intermediary_versions { insert_mirrored_artifact( @@ -140,94 +259,38 @@ async fn fetch( } if !fetch_fabric_versions.is_empty() { - let fabric_version_manifest_urls = fetch_fabric_versions + let universal_group = metadata_groups .iter() - .map(|x| { - format!( + .find(|group| group.id == UNIVERSAL_METADATA_GROUP) + .expect("fabric metadata should have a universal group"); + let profile_requests = fetch_fabric_versions + .iter() + .map(|loader| ProfileRequest { + group: universal_group.id.to_string(), + loader_profile_template_game_version: universal_group + .loader_profile_template_game_version + .clone(), + game_versions: universal_group.game_versions.clone(), + loader_version: loader.version.clone(), + url: format!( "{}/versions/loader/{}/{}/profile/json", - meta_url, DUMMY_GAME_VERSION, x.version - ) + meta_url, + universal_group.loader_profile_template_game_version, + loader.version + ), }) .collect::>(); - let fabric_version_manifests = futures::future::try_join_all( - fabric_version_manifest_urls - .iter() - .map(|x| download_file(x, None, &semaphore)), + + fetch_metadata_profiles( + mod_loader, + format_version, + maven_url, + profile_requests, + &upload_files, + &mirror_artifacts, + &semaphore, ) - .await? - .into_iter() - .map(|x| serde_json::from_slice(&x)) - .collect::, serde_json::Error>>()?; - - let patched_version_manifests = fabric_version_manifests - .into_iter() - .map(|mut version_info| { - for lib in &mut version_info.libraries { - let new_name = lib - .name - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - - // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries - if &*lib.name == "net.minecraft:launchwrapper:1.12" { - lib.url = Some( - "https://libraries.minecraft.net/".to_string(), - ); - } - - // If a library is not intermediary, we add it to mirror artifacts to be mirrored - if lib.name == new_name { - insert_mirrored_artifact( - &new_name, - None, - vec![ - lib.url - .clone() - .unwrap_or_else(|| maven_url.to_string()), - ], - false, - &mirror_artifacts, - )?; - } else { - lib.name = new_name; - } - - lib.url = Some(format_url("maven/")); - } - - version_info.id = version_info - .id - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - version_info.inherits_from = version_info - .inherits_from - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - - Ok(version_info) - }) - .collect::, Error>>()?; - let serialized_version_manifests = patched_version_manifests - .iter() - .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) - .collect::, serde_json::Error>>()?; - - serialized_version_manifests - .into_iter() - .enumerate() - .for_each(|(index, bytes)| { - let loader = fetch_fabric_versions[index]; - - let version_path = format!( - "{mod_loader}/v{format_version}/versions/{}.json", - loader.version - ); - - upload_files.insert( - version_path, - UploadFile { - file: bytes, - content_type: Some("application/json".to_string()), - }, - ); - }); + .await?; } if !fetch_fabric_versions.is_empty() @@ -240,17 +303,20 @@ async fn fetch( let loader_versions = daedalus::modded::Version { id: DUMMY_REPLACE_STRING.to_string(), stable: true, - loaders: fabric_manifest - .loader - .into_iter() + version_group: None, + loaders: all_loader_versions + .iter() + .filter(|x| !skip_versions.contains(&&*x.version)) .map(|x| { - let version_path = format!( - "{mod_loader}/v{format_version}/versions/{}.json", - x.version, + let version_path = metadata_version_path( + mod_loader, + format_version, + &x.version, + UNIVERSAL_METADATA_GROUP, ); daedalus::modded::LoaderVersion { - id: x.version, + id: x.version.clone(), url: format_url(&version_path), stable: x.stable, } @@ -260,14 +326,16 @@ async fn fetch( let manifest = daedalus::modded::Manifest { game_versions: std::iter::once(loader_versions) - .chain(fabric_manifest.game.into_iter().map(|x| { + .chain(all_game_versions.into_iter().map(|x| { daedalus::modded::Version { id: x.version, stable: x.stable, + version_group: None, loaders: vec![], } })) .collect(), + version_groups: Vec::new(), }; upload_files.insert( @@ -285,6 +353,145 @@ async fn fetch( }) } +struct ProfileRequest { + group: String, + loader_profile_template_game_version: String, + game_versions: Vec, + loader_version: String, + url: String, +} + +fn metadata_version_path( + mod_loader: &str, + format_version: usize, + loader_version: &str, + group: &str, +) -> String { + if group == UNIVERSAL_METADATA_GROUP { + format!("{mod_loader}/v{format_version}/versions/{loader_version}.json") + } else { + format!( + "{mod_loader}/v{format_version}/version-group/{group}/loader-version/{loader_version}" + ) + } +} + +async fn fetch_metadata_profiles( + mod_loader: &str, + format_version: usize, + maven_url: &str, + profile_requests: Vec, + upload_files: &DashMap, + mirror_artifacts: &DashMap, + semaphore: &Arc, +) -> Result<(), Error> { + let version_manifests = futures::future::try_join_all( + profile_requests + .iter() + .map(|x| download_file(&x.url, None, semaphore)), + ) + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; + + let patched_version_manifests = version_manifests + .into_iter() + .zip(profile_requests.iter()) + .map(|(mut version_info, request)| { + patch_version_info( + &mut version_info, + &request.loader_profile_template_game_version, + &request.game_versions, + maven_url, + mirror_artifacts, + )?; + + Ok(version_info) + }) + .collect::, Error>>()?; + let serialized_version_manifests = patched_version_manifests + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + + serialized_version_manifests + .into_iter() + .zip(profile_requests) + .for_each(|(bytes, request)| { + let version_path = metadata_version_path( + mod_loader, + format_version, + &request.loader_version, + &request.group, + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); + + Ok(()) +} + +fn patch_version_info( + version_info: &mut PartialVersionInfo, + game_version: &str, + game_versions: &[String], + maven_url: &str, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + for lib in &mut version_info.libraries { + let new_name = lib.name.replace(game_version, DUMMY_REPLACE_STRING); + + // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries + if &*lib.name == "net.minecraft:launchwrapper:1.12" { + lib.url = Some("https://libraries.minecraft.net/".to_string()); + } + let source_url = + lib.url.clone().unwrap_or_else(|| maven_url.to_string()); + + if lib.name == new_name { + insert_mirrored_artifact( + &new_name, + None, + vec![source_url], + false, + mirror_artifacts, + )?; + } else { + for concrete_game_version in game_versions { + let concrete_name = + lib.name.replace(game_version, concrete_game_version); + + insert_mirrored_artifact( + &concrete_name, + None, + vec![source_url.clone()], + false, + mirror_artifacts, + )?; + } + + lib.name = new_name; + } + + lib.url = Some(format_url("maven/")); + } + + version_info.id = + version_info.id.replace(game_version, DUMMY_REPLACE_STRING); + version_info.inherits_from = version_info + .inherits_from + .replace(game_version, DUMMY_REPLACE_STRING); + + Ok(()) +} + #[derive(Deserialize, Debug, Clone)] struct FabricVersions { pub loader: Vec, diff --git a/apps/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs index 3e9293281f..3ebdd6f428 100644 --- a/apps/daedalus_client/src/forge.rs +++ b/apps/daedalus_client/src/forge.rs @@ -12,7 +12,10 @@ use itertools::Itertools; use serde::Deserialize; use serde::de::DeserializeOwned; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; use tokio::sync::Semaphore; #[tracing::instrument(skip(semaphore))] @@ -243,12 +246,45 @@ async fn fetch( }; if !fetch_versions.is_empty() { - let forge_installers = futures::future::try_join_all( - fetch_versions - .iter() - .map(|x| download_file(&x.installer_url, None, &semaphore)), - ) - .await?; + let total_installers = fetch_versions.len(); + let downloaded_installers = Arc::new(AtomicUsize::new(0)); + + tracing::info!( + mod_loader, + total_files = total_installers, + "Downloading loader installers" + ); + + let forge_installers = + futures::future::try_join_all(fetch_versions.iter().map(|x| { + let downloaded_installers = downloaded_installers.clone(); + let semaphore = semaphore.clone(); + + async move { + let installer = + download_file(&x.installer_url, None, &semaphore) + .await?; + let downloaded = downloaded_installers + .fetch_add(1, Ordering::Relaxed) + + 1; + + if downloaded.is_multiple_of(100) + || downloaded == total_installers + { + tracing::info!( + mod_loader, + downloaded_files = downloaded, + remaining_files = + total_installers.saturating_sub(downloaded), + total_files = total_installers, + "Downloaded loader installers" + ); + } + + Ok::<_, Error>(installer) + } + })) + .await?; #[tracing::instrument(skip(raw, upload_files, mirror_artifacts))] async fn read_forge_installer( @@ -761,21 +797,23 @@ async fn fetch( .into_iter() .map(|(game_version, loaders)| { daedalus::modded::Version { - id: game_version, - stable: true, - loaders: loaders - .map(|x| daedalus::modded::LoaderVersion { - url: format_url(&format!( - "{mod_loader}/v{format_version}/versions/{}.json", - x.loader_version - )), - id: x.loader_version, - stable: false, - }) - .collect(), - } + id: game_version, + stable: true, + version_group: None, + loaders: loaders + .map(|x| daedalus::modded::LoaderVersion { + url: format_url(&format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.loader_version + )), + id: x.loader_version, + stable: false, + }) + .collect(), + } }) .collect(), + version_groups: Vec::new(), }; upload_files.insert( diff --git a/apps/daedalus_client/src/main.rs b/apps/daedalus_client/src/main.rs index 8b132a0c11..f9bda189ee 100644 --- a/apps/daedalus_client/src/main.rs +++ b/apps/daedalus_client/src/main.rs @@ -1,10 +1,14 @@ use crate::util::{ REQWEST_CLIENT, format_url, upload_file_to_bucket, - upload_url_to_bucket_mirrors, + upload_url_to_bucket_mirrors, write_file_to_local_output, + write_url_to_local_output_mirrors, }; use daedalus::get_path_from_artifact; use dashmap::{DashMap, DashSet}; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; use tokio::sync::Semaphore; use tracing_error::ErrorLayer; use tracing_subscriber::{EnvFilter, fmt, prelude::*}; @@ -12,6 +16,7 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*}; mod error; mod fabric; mod forge; +mod metadata_groups; mod minecraft; pub mod util; @@ -44,51 +49,92 @@ async fn main() -> Result<()> { )); let mut fetch_result = FetchResult::default(); + let only_loader = dotenvy::var("DAEDALUS_ONLY").ok(); - match minecraft::fetch(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"), + if should_fetch(only_loader.as_deref(), "minecraft") { + match minecraft::fetch(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"), + } } - match fabric::fetch_fabric(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"), + if should_fetch(only_loader.as_deref(), "fabric") { + match fabric::fetch_fabric(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"), + } } - match fabric::fetch_quilt(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"), + if should_fetch(only_loader.as_deref(), "quilt") { + match fabric::fetch_quilt(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"), + } } - match forge::fetch_neo(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"), + if should_fetch(only_loader.as_deref(), "neo") { + match forge::fetch_neo(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"), + } } - match forge::fetch_forge(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Forge fetch failed"), + if should_fetch(only_loader.as_deref(), "forge") { + match forge::fetch_forge(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Forge fetch failed"), + } } let FetchResult { upload_files, mirror_artifacts, } = fetch_result; + let upload_file_total = upload_files.len(); + let mirror_file_total = mirror_artifacts.len(); - futures::future::try_join_all(upload_files.iter().map(|entry| { - upload_file_to_bucket( - entry.key().clone(), - entry.value().file.clone(), - entry.value().content_type.clone(), - &semaphore, - ) - })) - .await?; + if dotenvy::var("LOCAL_OUTPUT_DIR").is_ok() { + let written_files = Arc::new(AtomicUsize::new(0)); - futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { - upload_url_to_bucket_mirrors( - format!("maven/{}", entry.key()), - entry + tracing::info!( + total_files = upload_file_total, + "Writing local metadata files" + ); + + futures::future::try_join_all(upload_files.iter().map(|entry| { + let path = entry.key().clone(); + let file = entry.value().file.clone(); + let written_files = written_files.clone(); + + async move { + write_file_to_local_output(&path, file).await?; + let written = written_files.fetch_add(1, Ordering::Relaxed) + 1; + + if written.is_multiple_of(100) || written == upload_file_total { + tracing::info!( + written_files = written, + remaining_files = + upload_file_total.saturating_sub(written), + total_files = upload_file_total, + "Wrote local metadata files" + ); + } + + Ok::<_, Error>(()) + } + })) + .await?; + + let written_mirror_files = Arc::new(AtomicUsize::new(0)); + + tracing::info!( + total_files = mirror_file_total, + "Writing local mirror files" + ); + + futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { + let path = format!("maven/{}", entry.key()); + let mirrors = entry .value() .mirrors .iter() @@ -99,12 +145,117 @@ async fn main() -> Result<()> { format!("{}{}", mirror.path, entry.key()) } }) - .collect(), - entry.value().sha1.clone(), - &semaphore, - ) - })) - .await?; + .collect(); + let sha1 = entry.value().sha1.clone(); + let written_mirror_files = written_mirror_files.clone(); + let semaphore = semaphore.clone(); + + async move { + write_url_to_local_output_mirrors( + path, mirrors, sha1, &semaphore, + ) + .await?; + let written = + written_mirror_files.fetch_add(1, Ordering::Relaxed) + 1; + + if written.is_multiple_of(100) || written == mirror_file_total { + tracing::info!( + written_files = written, + remaining_files = + mirror_file_total.saturating_sub(written), + total_files = mirror_file_total, + "Wrote local mirror files" + ); + } + + Ok::<_, Error>(()) + } + })) + .await?; + } else { + let uploaded_files = Arc::new(AtomicUsize::new(0)); + + tracing::info!( + total_files = upload_file_total, + "Uploading metadata files" + ); + + futures::future::try_join_all(upload_files.iter().map(|entry| { + let path = entry.key().clone(); + let file = entry.value().file.clone(); + let content_type = entry.value().content_type.clone(); + let uploaded_files = uploaded_files.clone(); + let semaphore = semaphore.clone(); + + async move { + upload_file_to_bucket(path, file, content_type, &semaphore) + .await?; + let uploaded = + uploaded_files.fetch_add(1, Ordering::Relaxed) + 1; + + if uploaded.is_multiple_of(100) || uploaded == upload_file_total + { + tracing::info!( + uploaded_files = uploaded, + remaining_files = + upload_file_total.saturating_sub(uploaded), + total_files = upload_file_total, + "Uploaded metadata files" + ); + } + + Ok::<_, Error>(()) + } + })) + .await?; + + let uploaded_mirror_files = Arc::new(AtomicUsize::new(0)); + + tracing::info!( + total_files = mirror_file_total, + "Uploading mirror files" + ); + + futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { + let path = format!("maven/{}", entry.key()); + let mirrors = entry + .value() + .mirrors + .iter() + .map(|mirror| { + if mirror.entire_url { + mirror.path.clone() + } else { + format!("{}{}", mirror.path, entry.key()) + } + }) + .collect(); + let sha1 = entry.value().sha1.clone(); + let uploaded_mirror_files = uploaded_mirror_files.clone(); + let semaphore = semaphore.clone(); + + async move { + upload_url_to_bucket_mirrors(path, mirrors, sha1, &semaphore) + .await?; + let uploaded = + uploaded_mirror_files.fetch_add(1, Ordering::Relaxed) + 1; + + if uploaded.is_multiple_of(100) || uploaded == mirror_file_total + { + tracing::info!( + uploaded_files = uploaded, + remaining_files = + mirror_file_total.saturating_sub(uploaded), + total_files = mirror_file_total, + "Uploaded mirror files" + ); + } + + Ok::<_, Error>(()) + } + })) + .await?; + } if dotenvy::var("CLOUDFLARE_INTEGRATION") .ok() @@ -151,6 +302,20 @@ async fn main() -> Result<()> { Ok(()) } +fn should_fetch(only_loader: Option<&str>, loader: &str) -> bool { + let Some(only_loader) = only_loader else { + return true; + }; + + only_loader.split(',').any(|entry| { + let entry = entry.trim(); + + entry.eq_ignore_ascii_case("all") + || entry.eq_ignore_ascii_case(loader) + || (loader == "neo" && entry.eq_ignore_ascii_case("neoforge")) + }) +} + pub struct UploadFile { file: bytes::Bytes, content_type: Option, @@ -248,11 +413,13 @@ fn check_env_vars() -> bool { failed |= check_var::("BASE_URL"); - failed |= check_var::("S3_ACCESS_TOKEN"); - failed |= check_var::("S3_SECRET"); - failed |= check_var::("S3_URL"); - failed |= check_var::("S3_REGION"); - failed |= check_var::("S3_BUCKET_NAME"); + if dotenvy::var("LOCAL_OUTPUT_DIR").is_err() { + failed |= check_var::("S3_ACCESS_TOKEN"); + failed |= check_var::("S3_SECRET"); + failed |= check_var::("S3_URL"); + failed |= check_var::("S3_REGION"); + failed |= check_var::("S3_BUCKET_NAME"); + } if dotenvy::var("CLOUDFLARE_INTEGRATION") .ok() diff --git a/apps/daedalus_client/src/metadata_groups.rs b/apps/daedalus_client/src/metadata_groups.rs new file mode 100644 index 0000000000..45b2c01b59 --- /dev/null +++ b/apps/daedalus_client/src/metadata_groups.rs @@ -0,0 +1,145 @@ +//! Determines how version info is generated for pairs of game and loader +//! versions. +//! +//! When a user installs a version of the game, they install two things: the +//! game (of some specific version), and a loader (of some other specific +//! version). Each combination of game and loader version requires a specific +//! configuration, like a specific set of libraries that must be downloaded and +//! run along with the game. However, some versions of the game or loader may +//! change configuration requirements without the other version being affected. +//! For example, pre-26.x game versions with Quilt require the Quilt `hashed` +//! libraries to also be downloaded. However, 26.x and later don't require the +//! `hashed` libraries, and don't even have a download for them. The problem is +//! that Quilt loader 0.30.0 can be used for both pre-26.x and 26.x - but our v0 +//! manifest files can't differentiate the two. The result is that you either +//! break compatibility for 0.30.0 game versions pre-26.x, or break 0.30.0 +//! on 26.x and later. +//! +//! To fix this, v1 introduces the concept of *version groups*: game versions +//! before 26.x are version group v1, and 26.x and later are v2. Then, we +//! parameterize our version info on both version group and loader version, +//! letting us specify the right configuration based on both game version and +//! loader version. +//! +//! Why not parameterize on game version and loader version directly? Most game +//! versions have the same configuration as their surrounding game versions, so +//! we'd end up with many duplicate configurations: the number of game versions +//! multiplied by the number of loader versions. +//! +//! This file lets you configure what game versions are grouped together. +//! +//! Each version group is templated from a specific game version - e.g. game +//! version 1.21 is used as the template file for 1.20, 1.19, etc. + +pub const UNIVERSAL_METADATA_GROUP: &str = "universal"; +pub const QUILT_LEGACY_METADATA_GROUP: &str = "v1"; +pub const QUILT_MODERN_METADATA_GROUP: &str = "v2"; + +pub struct MetadataGroup { + pub id: &'static str, + /// Minecraft version used to fetch and template this group's loader profiles. + pub loader_profile_template_game_version: String, + pub game_versions: Vec, +} + +pub fn metadata_groups<'a>( + mod_loader: &str, + game_versions: impl IntoIterator, +) -> Vec { + // Non-Quilt loaders don't need the concept of version groups, so we just + // make one "universal" group, and template it on 1.21. + if mod_loader != "quilt" { + return vec![MetadataGroup { + id: UNIVERSAL_METADATA_GROUP, + loader_profile_template_game_version: "1.21".to_string(), + game_versions: game_versions + .into_iter() + .map(|x| x.to_string()) + .collect(), + }]; + } + + let game_versions = game_versions.into_iter().collect::>(); + let legacy_game_versions = game_versions + .iter() + .copied() + .filter(|game_version| { + metadata_group_id_for_game_version(mod_loader, game_version) + == QUILT_LEGACY_METADATA_GROUP + }) + .collect::>(); + let modern_game_versions = game_versions + .iter() + .copied() + .filter(|game_version| { + metadata_group_id_for_game_version(mod_loader, game_version) + == QUILT_MODERN_METADATA_GROUP + }) + .collect::>(); + + let mut groups = Vec::new(); + + if !legacy_game_versions.is_empty() { + groups.push(MetadataGroup { + id: QUILT_LEGACY_METADATA_GROUP, + loader_profile_template_game_version: legacy_game_versions + .iter() + .find(|x| **x == "1.21") + .copied() + .unwrap_or(legacy_game_versions[0]) + .to_string(), + game_versions: legacy_game_versions + .iter() + .map(|x| x.to_string()) + .collect(), + }); + } + + if !modern_game_versions.is_empty() { + groups.push(MetadataGroup { + id: QUILT_MODERN_METADATA_GROUP, + loader_profile_template_game_version: modern_game_versions[0] + .to_string(), + game_versions: modern_game_versions + .iter() + .map(|x| x.to_string()) + .collect(), + }); + } + + groups +} + +pub fn metadata_group_for_game_version<'a>( + groups: &'a [MetadataGroup], + mod_loader: &str, + game_version: &str, +) -> Option<&'a MetadataGroup> { + let group_id = metadata_group_id_for_game_version(mod_loader, game_version); + + groups.iter().find(|group| group.id == group_id) +} + +fn metadata_group_id_for_game_version( + mod_loader: &str, + game_version: &str, +) -> &'static str { + if mod_loader == "quilt" && is_modern_quilt_game_version(game_version) { + QUILT_MODERN_METADATA_GROUP + } else if mod_loader == "quilt" { + QUILT_LEGACY_METADATA_GROUP + } else { + UNIVERSAL_METADATA_GROUP + } +} + +// Update these Quilt group boundaries if upstream loader profiles gain another +// structural incompatibility between Minecraft versions. +fn is_modern_quilt_game_version(game_version: &str) -> bool { + let major = game_version + .split(['.', 'w']) + .next() + .and_then(|x| x.parse::().ok()); + + major.is_some_and(|x| x >= 26) +} diff --git a/apps/daedalus_client/src/util.rs b/apps/daedalus_client/src/util.rs index 6d331d839e..0303e02bde 100644 --- a/apps/daedalus_client/src/util.rs +++ b/apps/daedalus_client/src/util.rs @@ -3,7 +3,11 @@ use bytes::Bytes; use s3::creds::Credentials; use s3::{Bucket, Region}; use serde::de::DeserializeOwned; -use std::sync::{Arc, LazyLock}; +use std::path::PathBuf; +use std::sync::{ + Arc, LazyLock, + atomic::{AtomicUsize, Ordering}, +}; use tokio::sync::Semaphore; static BUCKET: LazyLock = LazyLock::new(|| { @@ -55,6 +59,8 @@ pub static REQWEST_CLIENT: LazyLock = LazyLock::new(|| { .unwrap() }); +static DOWNLOADED_FILE_COUNT: AtomicUsize = AtomicUsize::new(0); + #[tracing::instrument(skip(bytes, semaphore))] pub async fn upload_file_to_bucket( path: String, @@ -135,6 +141,74 @@ pub async fn upload_url_to_bucket( Ok(()) } +pub async fn write_file_to_local_output( + path: &str, + bytes: Bytes, +) -> Result<(), Error> { + let output_path = local_output_path(path)?; + + if let Some(parent) = output_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + tokio::fs::write(output_path, bytes).await?; + + Ok(()) +} + +pub async fn write_url_to_local_output_mirrors( + output_path: String, + mirrors: Vec, + sha1: Option, + semaphore: &Arc, +) -> Result<(), Error> { + if mirrors.is_empty() { + return Err(ErrorKind::InvalidInput( + "No mirrors provided!".to_string(), + ) + .into()); + } + + for (index, mirror) in mirrors.iter().enumerate() { + let result = write_url_to_local_output( + output_path.clone(), + mirror.clone(), + sha1.clone(), + semaphore, + ) + .await; + + if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { + return result; + } + } + + unreachable!() +} + +async fn write_url_to_local_output( + path: String, + url: String, + sha1: Option, + semaphore: &Arc, +) -> Result<(), Error> { + let data = download_file(&url, sha1.as_deref(), semaphore).await?; + + write_file_to_local_output(&path, data).await?; + + Ok(()) +} + +fn local_output_path(path: &str) -> Result { + let output_dir = dotenvy::var("LOCAL_OUTPUT_DIR").map_err(|_| { + ErrorKind::InvalidInput( + "LOCAL_OUTPUT_DIR is required for local output".to_string(), + ) + })?; + + Ok(PathBuf::from(output_dir).join(path)) +} + #[tracing::instrument(skip(bytes))] pub async fn sha1_async(bytes: Bytes) -> Result { let hash = tokio::task::spawn_blocking(move || { @@ -182,6 +256,16 @@ pub async fn download_file( } } + let downloaded = DOWNLOADED_FILE_COUNT + .fetch_add(1, Ordering::Relaxed) + + 1; + if downloaded.is_multiple_of(100) { + tracing::info!( + downloaded_files = downloaded, + "Downloaded metadata files" + ); + } + return Ok(bytes); } else if attempt <= RETRIES { continue; diff --git a/packages/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs index c306b651ef..6d121b357e 100644 --- a/packages/daedalus/src/modded.rs +++ b/packages/daedalus/src/modded.rs @@ -10,10 +10,73 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// The latest version of the format the quilt model structs deserialize to -pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; +pub const CURRENT_QUILT_FORMAT_VERSION: usize = 1; /// The latest version of the format the neoforge model structs deserialize to pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; +/// Metadata for locating and caching a loader manifest. +#[derive(Debug, Clone)] +pub struct LoaderManifestMetadata { + /// The canonical loader name used in launcher-meta paths. + pub loader: String, + /// The latest manifest format version for this loader. + pub format_version: usize, + /// The cache key that includes the loader format version. + pub cache_key: String, + /// The launcher-meta path to the manifest. + pub path: String, +} + +/// Returns metadata for the latest manifest format for the provided loader. +pub fn loader_manifest_metadata(loader: &str) -> LoaderManifestMetadata { + let format_version = current_loader_manifest_format_version(loader); + let cache_key = format!("{loader}-v{format_version}"); + let path = format!("{loader}/v{format_version}/manifest.json"); + + LoaderManifestMetadata { + loader: loader.to_string(), + format_version, + cache_key, + path, + } +} + +/// Returns loader manifest metadata from a versioned cache key. +pub fn loader_manifest_metadata_from_cache_key( + cache_key: &str, +) -> LoaderManifestMetadata { + if let Some((loader, format_version)) = + cache_key.rsplit_once("-v").and_then(|(loader, version)| { + version + .parse::() + .ok() + .map(|version| (loader, version)) + }) + { + let cache_key = format!("{loader}-v{format_version}"); + let path = format!("{loader}/v{format_version}/manifest.json"); + + LoaderManifestMetadata { + loader: loader.to_string(), + format_version, + cache_key, + path, + } + } else { + loader_manifest_metadata(cache_key) + } +} + +fn current_loader_manifest_format_version(loader: &str) -> usize { + match loader { + "fabric" => CURRENT_FABRIC_FORMAT_VERSION, + "forge" => CURRENT_FORGE_FORMAT_VERSION, + "quilt" => CURRENT_QUILT_FORMAT_VERSION, + "neo" => CURRENT_NEOFORGE_FORMAT_VERSION, + _ => 0, + } +} + /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; @@ -188,19 +251,36 @@ pub fn merge_partial_version( pub struct Manifest { /// The game versions the mod loader supports pub game_versions: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + /// Groups of game versions that share compatible loader version profiles + pub version_groups: Vec, } #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] /// A game version of Minecraft pub struct Version { /// The minecraft version ID pub id: String, /// Whether the release is stable or not pub stable: bool, + #[serde(skip_serializing_if = "Option::is_none")] + /// The loader profile group for this Minecraft version + pub version_group: Option, /// A map that contains loader versions for the game version pub loaders: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +/// A group of Minecraft versions that share loader version profiles +pub struct VersionGroup { + /// The version group ID + pub id: String, + /// The loader versions for this version group + pub loaders: Vec, +} + #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion {