diff --git a/apps/daedalus_client/src/error.rs b/apps/daedalus_client/src/error.rs index 6afb321df4..6765341381 100644 --- a/apps/daedalus_client/src/error.rs +++ b/apps/daedalus_client/src/error.rs @@ -27,8 +27,6 @@ pub enum ErrorKind { inner: Box, file: String, }, - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), #[error("Error acquiring semaphore: {0}")] Acquire(#[from] tokio::sync::AcquireError), #[error("Tracing error: {0}")] diff --git a/apps/daedalus_client/src/fabric.rs b/apps/daedalus_client/src/fabric.rs index 8d9362bb73..3d6f1eeb0c 100644 --- a/apps/daedalus_client/src/fabric.rs +++ b/apps/daedalus_client/src/fabric.rs @@ -1,19 +1,3 @@ -//! Fetches Fabric-compatible loader metadata. -//! -//! Fabric and Quilt both expose loader profiles for a concrete Minecraft -//! version, but Daedalus publishes templated profiles using -//! `${modrinth.gameVersion}`. A group is a set of Minecraft versions whose -//! upstream loader profiles have the same structure after the concrete -//! Minecraft version is replaced with `${modrinth.gameVersion}`. Fabric uses -//! one universal group, so its public profile paths stay as -//! `versions/{loader}.json`. Quilt has more than one group: versions before -//! 26.x include hashed/intermediary libraries, while 26.x versions do not. For -//! Quilt, Daedalus writes one templated profile per group at -//! `version-group/{group}/loader-version/{loader}`. - -use crate::metadata_groups::{ - UNIVERSAL_METADATA_GROUP, metadata_group_for_game_version, metadata_groups, -}; use crate::util::{download_file, fetch_json, format_url}; use crate::{ Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact, @@ -80,112 +64,7 @@ async fn fetch( &semaphore, ) .await?; - let all_loader_versions = fabric_manifest.loader.clone(); - let all_game_versions = fabric_manifest.game.clone(); - let metadata_groups = metadata_groups( - mod_loader, - all_game_versions.iter().map(|x| x.version.as_str()), - ); - if metadata_groups - .iter() - .any(|group| group.id != UNIVERSAL_METADATA_GROUP) - { - let loaders = all_loader_versions - .iter() - .filter(|x| !skip_versions.contains(&&*x.version)) - .collect::>(); - - let profile_requests = metadata_groups - .iter() - .flat_map(|group| { - loaders.iter().map(move |loader| ProfileRequest { - group: group.id.to_string(), - loader_profile_template_game_version: group - .loader_profile_template_game_version - .clone(), - game_versions: group.game_versions.clone(), - loader_version: loader.version.clone(), - url: format!( - "{}/versions/loader/{}/{}/profile/json", - meta_url, - group.loader_profile_template_game_version, - loader.version - ), - }) - }) - .collect::>(); - - fetch_metadata_profiles( - mod_loader, - format_version, - maven_url, - profile_requests, - &upload_files, - &mirror_artifacts, - &semaphore, - ) - .await?; - - let version_groups = metadata_groups - .iter() - .map(|group| daedalus::modded::VersionGroup { - id: group.id.to_string(), - loaders: loaders - .iter() - .map(|loader| { - let version_path = metadata_version_path( - mod_loader, - format_version, - &loader.version, - group.id, - ); - - daedalus::modded::LoaderVersion { - id: loader.version.clone(), - url: format_url(&version_path), - stable: loader.stable, - } - }) - .collect(), - }) - .collect(); - - let manifest = daedalus::modded::Manifest { - game_versions: all_game_versions - .into_iter() - .map(|game_version| { - let group = metadata_group_for_game_version( - &metadata_groups, - mod_loader, - &game_version.version, - ) - .expect("game version should have a metadata group"); - - daedalus::modded::Version { - id: game_version.version.clone(), - stable: game_version.stable, - version_group: Some(group.id.to_string()), - loaders: Vec::new(), - } - }) - .collect(), - version_groups, - }; - - upload_files.insert( - format!("{mod_loader}/v{format_version}/manifest.json"), - UploadFile { - file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), - content_type: Some("application/json".to_string()), - }, - ); - - return Ok(FetchResult { - upload_files, - mirror_artifacts, - }); - } // We check Modrinth's manifest to find newly added loader versions, // intermediary/mapping artifacts, and game versions. let ( @@ -246,6 +125,8 @@ async fn fetch( ) }; + const DUMMY_GAME_VERSION: &str = "1.21"; + if !fetch_intermediary_versions.is_empty() { for x in &fetch_intermediary_versions { insert_mirrored_artifact( @@ -259,38 +140,94 @@ async fn fetch( } if !fetch_fabric_versions.is_empty() { - let universal_group = metadata_groups + let fabric_version_manifest_urls = fetch_fabric_versions .iter() - .find(|group| group.id == UNIVERSAL_METADATA_GROUP) - .expect("fabric metadata should have a universal group"); - let profile_requests = fetch_fabric_versions - .iter() - .map(|loader| ProfileRequest { - group: universal_group.id.to_string(), - loader_profile_template_game_version: universal_group - .loader_profile_template_game_version - .clone(), - game_versions: universal_group.game_versions.clone(), - loader_version: loader.version.clone(), - url: format!( + .map(|x| { + format!( "{}/versions/loader/{}/{}/profile/json", - meta_url, - universal_group.loader_profile_template_game_version, - loader.version - ), + meta_url, DUMMY_GAME_VERSION, x.version + ) }) .collect::>(); - - fetch_metadata_profiles( - mod_loader, - format_version, - maven_url, - profile_requests, - &upload_files, - &mirror_artifacts, - &semaphore, + let fabric_version_manifests = futures::future::try_join_all( + fabric_version_manifest_urls + .iter() + .map(|x| download_file(x, None, &semaphore)), ) - .await?; + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; + + let patched_version_manifests = fabric_version_manifests + .into_iter() + .map(|mut version_info| { + for lib in &mut version_info.libraries { + let new_name = lib + .name + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + + // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries + if &*lib.name == "net.minecraft:launchwrapper:1.12" { + lib.url = Some( + "https://libraries.minecraft.net/".to_string(), + ); + } + + // If a library is not intermediary, we add it to mirror artifacts to be mirrored + if lib.name == new_name { + insert_mirrored_artifact( + &new_name, + None, + vec![ + lib.url + .clone() + .unwrap_or_else(|| maven_url.to_string()), + ], + false, + &mirror_artifacts, + )?; + } else { + lib.name = new_name; + } + + lib.url = Some(format_url("maven/")); + } + + version_info.id = version_info + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + version_info.inherits_from = version_info + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + + Ok(version_info) + }) + .collect::, Error>>()?; + let serialized_version_manifests = patched_version_manifests + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + + serialized_version_manifests + .into_iter() + .enumerate() + .for_each(|(index, bytes)| { + let loader = fetch_fabric_versions[index]; + + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + loader.version + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); } if !fetch_fabric_versions.is_empty() @@ -303,20 +240,17 @@ async fn fetch( let loader_versions = daedalus::modded::Version { id: DUMMY_REPLACE_STRING.to_string(), stable: true, - version_group: None, - loaders: all_loader_versions - .iter() - .filter(|x| !skip_versions.contains(&&*x.version)) + loaders: fabric_manifest + .loader + .into_iter() .map(|x| { - let version_path = metadata_version_path( - mod_loader, - format_version, - &x.version, - UNIVERSAL_METADATA_GROUP, + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.version, ); daedalus::modded::LoaderVersion { - id: x.version.clone(), + id: x.version, url: format_url(&version_path), stable: x.stable, } @@ -326,16 +260,14 @@ async fn fetch( let manifest = daedalus::modded::Manifest { game_versions: std::iter::once(loader_versions) - .chain(all_game_versions.into_iter().map(|x| { + .chain(fabric_manifest.game.into_iter().map(|x| { daedalus::modded::Version { id: x.version, stable: x.stable, - version_group: None, loaders: vec![], } })) .collect(), - version_groups: Vec::new(), }; upload_files.insert( @@ -353,145 +285,6 @@ async fn fetch( }) } -struct ProfileRequest { - group: String, - loader_profile_template_game_version: String, - game_versions: Vec, - loader_version: String, - url: String, -} - -fn metadata_version_path( - mod_loader: &str, - format_version: usize, - loader_version: &str, - group: &str, -) -> String { - if group == UNIVERSAL_METADATA_GROUP { - format!("{mod_loader}/v{format_version}/versions/{loader_version}.json") - } else { - format!( - "{mod_loader}/v{format_version}/version-group/{group}/loader-version/{loader_version}" - ) - } -} - -async fn fetch_metadata_profiles( - mod_loader: &str, - format_version: usize, - maven_url: &str, - profile_requests: Vec, - upload_files: &DashMap, - mirror_artifacts: &DashMap, - semaphore: &Arc, -) -> Result<(), Error> { - let version_manifests = futures::future::try_join_all( - profile_requests - .iter() - .map(|x| download_file(&x.url, None, semaphore)), - ) - .await? - .into_iter() - .map(|x| serde_json::from_slice(&x)) - .collect::, serde_json::Error>>()?; - - let patched_version_manifests = version_manifests - .into_iter() - .zip(profile_requests.iter()) - .map(|(mut version_info, request)| { - patch_version_info( - &mut version_info, - &request.loader_profile_template_game_version, - &request.game_versions, - maven_url, - mirror_artifacts, - )?; - - Ok(version_info) - }) - .collect::, Error>>()?; - let serialized_version_manifests = patched_version_manifests - .iter() - .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) - .collect::, serde_json::Error>>()?; - - serialized_version_manifests - .into_iter() - .zip(profile_requests) - .for_each(|(bytes, request)| { - let version_path = metadata_version_path( - mod_loader, - format_version, - &request.loader_version, - &request.group, - ); - - upload_files.insert( - version_path, - UploadFile { - file: bytes, - content_type: Some("application/json".to_string()), - }, - ); - }); - - Ok(()) -} - -fn patch_version_info( - version_info: &mut PartialVersionInfo, - game_version: &str, - game_versions: &[String], - maven_url: &str, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { - for lib in &mut version_info.libraries { - let new_name = lib.name.replace(game_version, DUMMY_REPLACE_STRING); - - // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries - if &*lib.name == "net.minecraft:launchwrapper:1.12" { - lib.url = Some("https://libraries.minecraft.net/".to_string()); - } - let source_url = - lib.url.clone().unwrap_or_else(|| maven_url.to_string()); - - if lib.name == new_name { - insert_mirrored_artifact( - &new_name, - None, - vec![source_url], - false, - mirror_artifacts, - )?; - } else { - for concrete_game_version in game_versions { - let concrete_name = - lib.name.replace(game_version, concrete_game_version); - - insert_mirrored_artifact( - &concrete_name, - None, - vec![source_url.clone()], - false, - mirror_artifacts, - )?; - } - - lib.name = new_name; - } - - lib.url = Some(format_url("maven/")); - } - - version_info.id = - version_info.id.replace(game_version, DUMMY_REPLACE_STRING); - version_info.inherits_from = version_info - .inherits_from - .replace(game_version, DUMMY_REPLACE_STRING); - - Ok(()) -} - #[derive(Deserialize, Debug, Clone)] struct FabricVersions { pub loader: Vec, diff --git a/apps/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs index 3ebdd6f428..3e9293281f 100644 --- a/apps/daedalus_client/src/forge.rs +++ b/apps/daedalus_client/src/forge.rs @@ -12,10 +12,7 @@ use itertools::Itertools; use serde::Deserialize; use serde::de::DeserializeOwned; use std::collections::HashMap; -use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, -}; +use std::sync::Arc; use tokio::sync::Semaphore; #[tracing::instrument(skip(semaphore))] @@ -246,45 +243,12 @@ async fn fetch( }; if !fetch_versions.is_empty() { - let total_installers = fetch_versions.len(); - let downloaded_installers = Arc::new(AtomicUsize::new(0)); - - tracing::info!( - mod_loader, - total_files = total_installers, - "Downloading loader installers" - ); - - let forge_installers = - futures::future::try_join_all(fetch_versions.iter().map(|x| { - let downloaded_installers = downloaded_installers.clone(); - let semaphore = semaphore.clone(); - - async move { - let installer = - download_file(&x.installer_url, None, &semaphore) - .await?; - let downloaded = downloaded_installers - .fetch_add(1, Ordering::Relaxed) - + 1; - - if downloaded.is_multiple_of(100) - || downloaded == total_installers - { - tracing::info!( - mod_loader, - downloaded_files = downloaded, - remaining_files = - total_installers.saturating_sub(downloaded), - total_files = total_installers, - "Downloaded loader installers" - ); - } - - Ok::<_, Error>(installer) - } - })) - .await?; + let forge_installers = futures::future::try_join_all( + fetch_versions + .iter() + .map(|x| download_file(&x.installer_url, None, &semaphore)), + ) + .await?; #[tracing::instrument(skip(raw, upload_files, mirror_artifacts))] async fn read_forge_installer( @@ -797,23 +761,21 @@ async fn fetch( .into_iter() .map(|(game_version, loaders)| { daedalus::modded::Version { - id: game_version, - stable: true, - version_group: None, - loaders: loaders - .map(|x| daedalus::modded::LoaderVersion { - url: format_url(&format!( - "{mod_loader}/v{format_version}/versions/{}.json", - x.loader_version - )), - id: x.loader_version, - stable: false, - }) - .collect(), - } + id: game_version, + stable: true, + loaders: loaders + .map(|x| daedalus::modded::LoaderVersion { + url: format_url(&format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.loader_version + )), + id: x.loader_version, + stable: false, + }) + .collect(), + } }) .collect(), - version_groups: Vec::new(), }; upload_files.insert( diff --git a/apps/daedalus_client/src/main.rs b/apps/daedalus_client/src/main.rs index f9bda189ee..8b132a0c11 100644 --- a/apps/daedalus_client/src/main.rs +++ b/apps/daedalus_client/src/main.rs @@ -1,14 +1,10 @@ use crate::util::{ REQWEST_CLIENT, format_url, upload_file_to_bucket, - upload_url_to_bucket_mirrors, write_file_to_local_output, - write_url_to_local_output_mirrors, + upload_url_to_bucket_mirrors, }; use daedalus::get_path_from_artifact; use dashmap::{DashMap, DashSet}; -use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, -}; +use std::sync::Arc; use tokio::sync::Semaphore; use tracing_error::ErrorLayer; use tracing_subscriber::{EnvFilter, fmt, prelude::*}; @@ -16,7 +12,6 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*}; mod error; mod fabric; mod forge; -mod metadata_groups; mod minecraft; pub mod util; @@ -49,92 +44,51 @@ async fn main() -> Result<()> { )); let mut fetch_result = FetchResult::default(); - let only_loader = dotenvy::var("DAEDALUS_ONLY").ok(); - if should_fetch(only_loader.as_deref(), "minecraft") { - match minecraft::fetch(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"), - } + match minecraft::fetch(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"), } - if should_fetch(only_loader.as_deref(), "fabric") { - match fabric::fetch_fabric(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"), - } + match fabric::fetch_fabric(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"), } - if should_fetch(only_loader.as_deref(), "quilt") { - match fabric::fetch_quilt(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"), - } + match fabric::fetch_quilt(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"), } - if should_fetch(only_loader.as_deref(), "neo") { - match forge::fetch_neo(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"), - } + match forge::fetch_neo(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"), } - if should_fetch(only_loader.as_deref(), "forge") { - match forge::fetch_forge(semaphore.clone()).await { - Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), - Err(err) => tracing::warn!(error = %err, "Forge fetch failed"), - } + match forge::fetch_forge(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Forge fetch failed"), } let FetchResult { upload_files, mirror_artifacts, } = fetch_result; - let upload_file_total = upload_files.len(); - let mirror_file_total = mirror_artifacts.len(); - if dotenvy::var("LOCAL_OUTPUT_DIR").is_ok() { - let written_files = Arc::new(AtomicUsize::new(0)); + futures::future::try_join_all(upload_files.iter().map(|entry| { + upload_file_to_bucket( + entry.key().clone(), + entry.value().file.clone(), + entry.value().content_type.clone(), + &semaphore, + ) + })) + .await?; - tracing::info!( - total_files = upload_file_total, - "Writing local metadata files" - ); - - futures::future::try_join_all(upload_files.iter().map(|entry| { - let path = entry.key().clone(); - let file = entry.value().file.clone(); - let written_files = written_files.clone(); - - async move { - write_file_to_local_output(&path, file).await?; - let written = written_files.fetch_add(1, Ordering::Relaxed) + 1; - - if written.is_multiple_of(100) || written == upload_file_total { - tracing::info!( - written_files = written, - remaining_files = - upload_file_total.saturating_sub(written), - total_files = upload_file_total, - "Wrote local metadata files" - ); - } - - Ok::<_, Error>(()) - } - })) - .await?; - - let written_mirror_files = Arc::new(AtomicUsize::new(0)); - - tracing::info!( - total_files = mirror_file_total, - "Writing local mirror files" - ); - - futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { - let path = format!("maven/{}", entry.key()); - let mirrors = entry + futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { + upload_url_to_bucket_mirrors( + format!("maven/{}", entry.key()), + entry .value() .mirrors .iter() @@ -145,117 +99,12 @@ async fn main() -> Result<()> { format!("{}{}", mirror.path, entry.key()) } }) - .collect(); - let sha1 = entry.value().sha1.clone(); - let written_mirror_files = written_mirror_files.clone(); - let semaphore = semaphore.clone(); - - async move { - write_url_to_local_output_mirrors( - path, mirrors, sha1, &semaphore, - ) - .await?; - let written = - written_mirror_files.fetch_add(1, Ordering::Relaxed) + 1; - - if written.is_multiple_of(100) || written == mirror_file_total { - tracing::info!( - written_files = written, - remaining_files = - mirror_file_total.saturating_sub(written), - total_files = mirror_file_total, - "Wrote local mirror files" - ); - } - - Ok::<_, Error>(()) - } - })) - .await?; - } else { - let uploaded_files = Arc::new(AtomicUsize::new(0)); - - tracing::info!( - total_files = upload_file_total, - "Uploading metadata files" - ); - - futures::future::try_join_all(upload_files.iter().map(|entry| { - let path = entry.key().clone(); - let file = entry.value().file.clone(); - let content_type = entry.value().content_type.clone(); - let uploaded_files = uploaded_files.clone(); - let semaphore = semaphore.clone(); - - async move { - upload_file_to_bucket(path, file, content_type, &semaphore) - .await?; - let uploaded = - uploaded_files.fetch_add(1, Ordering::Relaxed) + 1; - - if uploaded.is_multiple_of(100) || uploaded == upload_file_total - { - tracing::info!( - uploaded_files = uploaded, - remaining_files = - upload_file_total.saturating_sub(uploaded), - total_files = upload_file_total, - "Uploaded metadata files" - ); - } - - Ok::<_, Error>(()) - } - })) - .await?; - - let uploaded_mirror_files = Arc::new(AtomicUsize::new(0)); - - tracing::info!( - total_files = mirror_file_total, - "Uploading mirror files" - ); - - futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { - let path = format!("maven/{}", entry.key()); - let mirrors = entry - .value() - .mirrors - .iter() - .map(|mirror| { - if mirror.entire_url { - mirror.path.clone() - } else { - format!("{}{}", mirror.path, entry.key()) - } - }) - .collect(); - let sha1 = entry.value().sha1.clone(); - let uploaded_mirror_files = uploaded_mirror_files.clone(); - let semaphore = semaphore.clone(); - - async move { - upload_url_to_bucket_mirrors(path, mirrors, sha1, &semaphore) - .await?; - let uploaded = - uploaded_mirror_files.fetch_add(1, Ordering::Relaxed) + 1; - - if uploaded.is_multiple_of(100) || uploaded == mirror_file_total - { - tracing::info!( - uploaded_files = uploaded, - remaining_files = - mirror_file_total.saturating_sub(uploaded), - total_files = mirror_file_total, - "Uploaded mirror files" - ); - } - - Ok::<_, Error>(()) - } - })) - .await?; - } + .collect(), + entry.value().sha1.clone(), + &semaphore, + ) + })) + .await?; if dotenvy::var("CLOUDFLARE_INTEGRATION") .ok() @@ -302,20 +151,6 @@ async fn main() -> Result<()> { Ok(()) } -fn should_fetch(only_loader: Option<&str>, loader: &str) -> bool { - let Some(only_loader) = only_loader else { - return true; - }; - - only_loader.split(',').any(|entry| { - let entry = entry.trim(); - - entry.eq_ignore_ascii_case("all") - || entry.eq_ignore_ascii_case(loader) - || (loader == "neo" && entry.eq_ignore_ascii_case("neoforge")) - }) -} - pub struct UploadFile { file: bytes::Bytes, content_type: Option, @@ -413,13 +248,11 @@ fn check_env_vars() -> bool { failed |= check_var::("BASE_URL"); - if dotenvy::var("LOCAL_OUTPUT_DIR").is_err() { - failed |= check_var::("S3_ACCESS_TOKEN"); - failed |= check_var::("S3_SECRET"); - failed |= check_var::("S3_URL"); - failed |= check_var::("S3_REGION"); - failed |= check_var::("S3_BUCKET_NAME"); - } + failed |= check_var::("S3_ACCESS_TOKEN"); + failed |= check_var::("S3_SECRET"); + failed |= check_var::("S3_URL"); + failed |= check_var::("S3_REGION"); + failed |= check_var::("S3_BUCKET_NAME"); if dotenvy::var("CLOUDFLARE_INTEGRATION") .ok() diff --git a/apps/daedalus_client/src/metadata_groups.rs b/apps/daedalus_client/src/metadata_groups.rs deleted file mode 100644 index 45b2c01b59..0000000000 --- a/apps/daedalus_client/src/metadata_groups.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! Determines how version info is generated for pairs of game and loader -//! versions. -//! -//! When a user installs a version of the game, they install two things: the -//! game (of some specific version), and a loader (of some other specific -//! version). Each combination of game and loader version requires a specific -//! configuration, like a specific set of libraries that must be downloaded and -//! run along with the game. However, some versions of the game or loader may -//! change configuration requirements without the other version being affected. -//! For example, pre-26.x game versions with Quilt require the Quilt `hashed` -//! libraries to also be downloaded. However, 26.x and later don't require the -//! `hashed` libraries, and don't even have a download for them. The problem is -//! that Quilt loader 0.30.0 can be used for both pre-26.x and 26.x - but our v0 -//! manifest files can't differentiate the two. The result is that you either -//! break compatibility for 0.30.0 game versions pre-26.x, or break 0.30.0 -//! on 26.x and later. -//! -//! To fix this, v1 introduces the concept of *version groups*: game versions -//! before 26.x are version group v1, and 26.x and later are v2. Then, we -//! parameterize our version info on both version group and loader version, -//! letting us specify the right configuration based on both game version and -//! loader version. -//! -//! Why not parameterize on game version and loader version directly? Most game -//! versions have the same configuration as their surrounding game versions, so -//! we'd end up with many duplicate configurations: the number of game versions -//! multiplied by the number of loader versions. -//! -//! This file lets you configure what game versions are grouped together. -//! -//! Each version group is templated from a specific game version - e.g. game -//! version 1.21 is used as the template file for 1.20, 1.19, etc. - -pub const UNIVERSAL_METADATA_GROUP: &str = "universal"; -pub const QUILT_LEGACY_METADATA_GROUP: &str = "v1"; -pub const QUILT_MODERN_METADATA_GROUP: &str = "v2"; - -pub struct MetadataGroup { - pub id: &'static str, - /// Minecraft version used to fetch and template this group's loader profiles. - pub loader_profile_template_game_version: String, - pub game_versions: Vec, -} - -pub fn metadata_groups<'a>( - mod_loader: &str, - game_versions: impl IntoIterator, -) -> Vec { - // Non-Quilt loaders don't need the concept of version groups, so we just - // make one "universal" group, and template it on 1.21. - if mod_loader != "quilt" { - return vec![MetadataGroup { - id: UNIVERSAL_METADATA_GROUP, - loader_profile_template_game_version: "1.21".to_string(), - game_versions: game_versions - .into_iter() - .map(|x| x.to_string()) - .collect(), - }]; - } - - let game_versions = game_versions.into_iter().collect::>(); - let legacy_game_versions = game_versions - .iter() - .copied() - .filter(|game_version| { - metadata_group_id_for_game_version(mod_loader, game_version) - == QUILT_LEGACY_METADATA_GROUP - }) - .collect::>(); - let modern_game_versions = game_versions - .iter() - .copied() - .filter(|game_version| { - metadata_group_id_for_game_version(mod_loader, game_version) - == QUILT_MODERN_METADATA_GROUP - }) - .collect::>(); - - let mut groups = Vec::new(); - - if !legacy_game_versions.is_empty() { - groups.push(MetadataGroup { - id: QUILT_LEGACY_METADATA_GROUP, - loader_profile_template_game_version: legacy_game_versions - .iter() - .find(|x| **x == "1.21") - .copied() - .unwrap_or(legacy_game_versions[0]) - .to_string(), - game_versions: legacy_game_versions - .iter() - .map(|x| x.to_string()) - .collect(), - }); - } - - if !modern_game_versions.is_empty() { - groups.push(MetadataGroup { - id: QUILT_MODERN_METADATA_GROUP, - loader_profile_template_game_version: modern_game_versions[0] - .to_string(), - game_versions: modern_game_versions - .iter() - .map(|x| x.to_string()) - .collect(), - }); - } - - groups -} - -pub fn metadata_group_for_game_version<'a>( - groups: &'a [MetadataGroup], - mod_loader: &str, - game_version: &str, -) -> Option<&'a MetadataGroup> { - let group_id = metadata_group_id_for_game_version(mod_loader, game_version); - - groups.iter().find(|group| group.id == group_id) -} - -fn metadata_group_id_for_game_version( - mod_loader: &str, - game_version: &str, -) -> &'static str { - if mod_loader == "quilt" && is_modern_quilt_game_version(game_version) { - QUILT_MODERN_METADATA_GROUP - } else if mod_loader == "quilt" { - QUILT_LEGACY_METADATA_GROUP - } else { - UNIVERSAL_METADATA_GROUP - } -} - -// Update these Quilt group boundaries if upstream loader profiles gain another -// structural incompatibility between Minecraft versions. -fn is_modern_quilt_game_version(game_version: &str) -> bool { - let major = game_version - .split(['.', 'w']) - .next() - .and_then(|x| x.parse::().ok()); - - major.is_some_and(|x| x >= 26) -} diff --git a/apps/daedalus_client/src/util.rs b/apps/daedalus_client/src/util.rs index 0303e02bde..6d331d839e 100644 --- a/apps/daedalus_client/src/util.rs +++ b/apps/daedalus_client/src/util.rs @@ -3,11 +3,7 @@ use bytes::Bytes; use s3::creds::Credentials; use s3::{Bucket, Region}; use serde::de::DeserializeOwned; -use std::path::PathBuf; -use std::sync::{ - Arc, LazyLock, - atomic::{AtomicUsize, Ordering}, -}; +use std::sync::{Arc, LazyLock}; use tokio::sync::Semaphore; static BUCKET: LazyLock = LazyLock::new(|| { @@ -59,8 +55,6 @@ pub static REQWEST_CLIENT: LazyLock = LazyLock::new(|| { .unwrap() }); -static DOWNLOADED_FILE_COUNT: AtomicUsize = AtomicUsize::new(0); - #[tracing::instrument(skip(bytes, semaphore))] pub async fn upload_file_to_bucket( path: String, @@ -141,74 +135,6 @@ pub async fn upload_url_to_bucket( Ok(()) } -pub async fn write_file_to_local_output( - path: &str, - bytes: Bytes, -) -> Result<(), Error> { - let output_path = local_output_path(path)?; - - if let Some(parent) = output_path.parent() { - tokio::fs::create_dir_all(parent).await?; - } - - tokio::fs::write(output_path, bytes).await?; - - Ok(()) -} - -pub async fn write_url_to_local_output_mirrors( - output_path: String, - mirrors: Vec, - sha1: Option, - semaphore: &Arc, -) -> Result<(), Error> { - if mirrors.is_empty() { - return Err(ErrorKind::InvalidInput( - "No mirrors provided!".to_string(), - ) - .into()); - } - - for (index, mirror) in mirrors.iter().enumerate() { - let result = write_url_to_local_output( - output_path.clone(), - mirror.clone(), - sha1.clone(), - semaphore, - ) - .await; - - if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { - return result; - } - } - - unreachable!() -} - -async fn write_url_to_local_output( - path: String, - url: String, - sha1: Option, - semaphore: &Arc, -) -> Result<(), Error> { - let data = download_file(&url, sha1.as_deref(), semaphore).await?; - - write_file_to_local_output(&path, data).await?; - - Ok(()) -} - -fn local_output_path(path: &str) -> Result { - let output_dir = dotenvy::var("LOCAL_OUTPUT_DIR").map_err(|_| { - ErrorKind::InvalidInput( - "LOCAL_OUTPUT_DIR is required for local output".to_string(), - ) - })?; - - Ok(PathBuf::from(output_dir).join(path)) -} - #[tracing::instrument(skip(bytes))] pub async fn sha1_async(bytes: Bytes) -> Result { let hash = tokio::task::spawn_blocking(move || { @@ -256,16 +182,6 @@ pub async fn download_file( } } - let downloaded = DOWNLOADED_FILE_COUNT - .fetch_add(1, Ordering::Relaxed) - + 1; - if downloaded.is_multiple_of(100) { - tracing::info!( - downloaded_files = downloaded, - "Downloaded metadata files" - ); - } - return Ok(bytes); } else if attempt <= RETRIES { continue; diff --git a/packages/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs index 6d121b357e..c306b651ef 100644 --- a/packages/daedalus/src/modded.rs +++ b/packages/daedalus/src/modded.rs @@ -10,73 +10,10 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// The latest version of the format the quilt model structs deserialize to -pub const CURRENT_QUILT_FORMAT_VERSION: usize = 1; +pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; /// The latest version of the format the neoforge model structs deserialize to pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; -/// Metadata for locating and caching a loader manifest. -#[derive(Debug, Clone)] -pub struct LoaderManifestMetadata { - /// The canonical loader name used in launcher-meta paths. - pub loader: String, - /// The latest manifest format version for this loader. - pub format_version: usize, - /// The cache key that includes the loader format version. - pub cache_key: String, - /// The launcher-meta path to the manifest. - pub path: String, -} - -/// Returns metadata for the latest manifest format for the provided loader. -pub fn loader_manifest_metadata(loader: &str) -> LoaderManifestMetadata { - let format_version = current_loader_manifest_format_version(loader); - let cache_key = format!("{loader}-v{format_version}"); - let path = format!("{loader}/v{format_version}/manifest.json"); - - LoaderManifestMetadata { - loader: loader.to_string(), - format_version, - cache_key, - path, - } -} - -/// Returns loader manifest metadata from a versioned cache key. -pub fn loader_manifest_metadata_from_cache_key( - cache_key: &str, -) -> LoaderManifestMetadata { - if let Some((loader, format_version)) = - cache_key.rsplit_once("-v").and_then(|(loader, version)| { - version - .parse::() - .ok() - .map(|version| (loader, version)) - }) - { - let cache_key = format!("{loader}-v{format_version}"); - let path = format!("{loader}/v{format_version}/manifest.json"); - - LoaderManifestMetadata { - loader: loader.to_string(), - format_version, - cache_key, - path, - } - } else { - loader_manifest_metadata(cache_key) - } -} - -fn current_loader_manifest_format_version(loader: &str) -> usize { - match loader { - "fabric" => CURRENT_FABRIC_FORMAT_VERSION, - "forge" => CURRENT_FORGE_FORMAT_VERSION, - "quilt" => CURRENT_QUILT_FORMAT_VERSION, - "neo" => CURRENT_NEOFORGE_FORMAT_VERSION, - _ => 0, - } -} - /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; @@ -251,36 +188,19 @@ pub fn merge_partial_version( pub struct Manifest { /// The game versions the mod loader supports pub game_versions: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - /// Groups of game versions that share compatible loader version profiles - pub version_groups: Vec, } #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] /// A game version of Minecraft pub struct Version { /// The minecraft version ID pub id: String, /// Whether the release is stable or not pub stable: bool, - #[serde(skip_serializing_if = "Option::is_none")] - /// The loader profile group for this Minecraft version - pub version_group: Option, /// A map that contains loader versions for the game version pub loaders: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -/// A group of Minecraft versions that share loader version profiles -pub struct VersionGroup { - /// The version group ID - pub id: String, - /// The loader versions for this version group - pub loaders: Vec, -} - #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion {