sugoidogo/modrinth
1
0
Fork 0
mirror of https://github.com/modrinth/code.git synced 2026-07-31 13:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Generate different Daedalus Quilt templates for 26.x/non-26.x (#6554)

Browse Source 
* Generate different Daedalus Quilt templates for 26.x/non-26.x

* add docs

* wip: daedalus version groups

* wip: app-side logic

* finish app-side logic, fix certain libs not included

* lints

* fmt

* fmt

* Split frontend changes out of Daedalus branch
This commit is contained in:
aecsocket
2026-07-07 16:45:33 +00:00
committed by GitHub
parent df8ebbd3e0
commit 5a1c2a46bc
7 changed files with 879 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/daedalus_client/src/error.rs
+2
View File
@@ -27,6 +27,8 @@ pub enum ErrorKind {
inner: Box<s3::error::S3Error>,
file: String,
},
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Error acquiring semaphore: {0}")]
Acquire(#[from] tokio::sync::AcquireError),
#[error("Tracing error: {0}")]
apps/daedalus_client/src/fabric.rs
+300 -93
View File
@@ -1,3 +1,19 @@
//! Fetches Fabric-compatible loader metadata.
//!
//! Fabric and Quilt both expose loader profiles for a concrete Minecraft
//! version, but Daedalus publishes templated profiles using
//! `${modrinth.gameVersion}`. A group is a set of Minecraft versions whose
//! upstream loader profiles have the same structure after the concrete
//! Minecraft version is replaced with `${modrinth.gameVersion}`. Fabric uses
//! one universal group, so its public profile paths stay as
//! `versions/{loader}.json`. Quilt has more than one group: versions before
//! 26.x include hashed/intermediary libraries, while 26.x versions do not. For
//! Quilt, Daedalus writes one templated profile per group at
//! `version-group/{group}/loader-version/{loader}`.
use crate::metadata_groups::{
UNIVERSAL_METADATA_GROUP, metadata_group_for_game_version, metadata_groups,
};
use crate::util::{download_file, fetch_json, format_url};
use crate::{
Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
@@ -64,7 +80,112 @@ async fn fetch(
&semaphore,
)
.await?;
let all_loader_versions = fabric_manifest.loader.clone();
let all_game_versions = fabric_manifest.game.clone();
let metadata_groups = metadata_groups(
mod_loader,
all_game_versions.iter().map(|x| x.version.as_str()),
);
if metadata_groups
.iter()
.any(|group| group.id != UNIVERSAL_METADATA_GROUP)
{
let loaders = all_loader_versions
.iter()
.filter(|x| !skip_versions.contains(&&*x.version))
.collect::<Vec<_>>();
let profile_requests = metadata_groups
.iter()
.flat_map(|group| {
loaders.iter().map(move |loader| ProfileRequest {
group: group.id.to_string(),
loader_profile_template_game_version: group
.loader_profile_template_game_version
.clone(),
game_versions: group.game_versions.clone(),
loader_version: loader.version.clone(),
url: format!(
"{}/versions/loader/{}/{}/profile/json",
meta_url,
group.loader_profile_template_game_version,
loader.version
),
})
})
.collect::<Vec<_>>();
fetch_metadata_profiles(
mod_loader,
format_version,
maven_url,
profile_requests,
&upload_files,
&mirror_artifacts,
&semaphore,
)
.await?;
let version_groups = metadata_groups
.iter()
.map(|group| daedalus::modded::VersionGroup {
id: group.id.to_string(),
loaders: loaders
.iter()
.map(|loader| {
let version_path = metadata_version_path(
mod_loader,
format_version,
&loader.version,
group.id,
);
daedalus::modded::LoaderVersion {
id: loader.version.clone(),
url: format_url(&version_path),
stable: loader.stable,
}
})
.collect(),
})
.collect();
let manifest = daedalus::modded::Manifest {
game_versions: all_game_versions
.into_iter()
.map(|game_version| {
let group = metadata_group_for_game_version(
&metadata_groups,
mod_loader,
&game_version.version,
)
.expect("game version should have a metadata group");
daedalus::modded::Version {
id: game_version.version.clone(),
stable: game_version.stable,
version_group: Some(group.id.to_string()),
loaders: Vec::new(),
}
})
.collect(),
version_groups,
};
upload_files.insert(
format!("{mod_loader}/v{format_version}/manifest.json"),
UploadFile {
file: bytes::Bytes::from(serde_json::to_vec(&manifest)?),
content_type: Some("application/json".to_string()),
},
);
return Ok(FetchResult {
upload_files,
mirror_artifacts,
});
}
// We check Modrinth's manifest to find newly added loader versions,
// intermediary/mapping artifacts, and game versions.
let (
@@ -125,8 +246,6 @@ async fn fetch(
)
};
const DUMMY_GAME_VERSION: &str = "1.21";
if !fetch_intermediary_versions.is_empty() {
for x in &fetch_intermediary_versions {
insert_mirrored_artifact(
@@ -140,94 +259,38 @@ async fn fetch(
}
if !fetch_fabric_versions.is_empty() {
let fabric_version_manifest_urls = fetch_fabric_versions
let universal_group = metadata_groups
.iter()
.map(|x| {
format!(
.find(|group| group.id == UNIVERSAL_METADATA_GROUP)
.expect("fabric metadata should have a universal group");
let profile_requests = fetch_fabric_versions
.iter()
.map(|loader| ProfileRequest {
group: universal_group.id.to_string(),
loader_profile_template_game_version: universal_group
.loader_profile_template_game_version
.clone(),
game_versions: universal_group.game_versions.clone(),
loader_version: loader.version.clone(),
url: format!(
"{}/versions/loader/{}/{}/profile/json",
meta_url, DUMMY_GAME_VERSION, x.version
)
meta_url,
universal_group.loader_profile_template_game_version,
loader.version
),
})
.collect::<Vec<_>>();
let fabric_version_manifests = futures::future::try_join_all(
fabric_version_manifest_urls
.iter()
.map(|x| download_file(x, None, &semaphore)),
fetch_metadata_profiles(
mod_loader,
format_version,
maven_url,
profile_requests,
&upload_files,
&mirror_artifacts,
&semaphore,
)
.await?
.into_iter()
.map(|x| serde_json::from_slice(&x))
.collect::<Result<Vec<PartialVersionInfo>, serde_json::Error>>()?;
let patched_version_manifests = fabric_version_manifests
.into_iter()
.map(|mut version_info| {
for lib in &mut version_info.libraries {
let new_name = lib
.name
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
// Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
if &*lib.name == "net.minecraft:launchwrapper:1.12" {
lib.url = Some(
"https://libraries.minecraft.net/".to_string(),
);
}
// If a library is not intermediary, we add it to mirror artifacts to be mirrored
if lib.name == new_name {
insert_mirrored_artifact(
&new_name,
None,
vec![
lib.url
.clone()
.unwrap_or_else(|| maven_url.to_string()),
],
false,
&mirror_artifacts,
)?;
} else {
lib.name = new_name;
}
lib.url = Some(format_url("maven/"));
}
version_info.id = version_info
.id
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
version_info.inherits_from = version_info
.inherits_from
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
Ok(version_info)
})
.collect::<Result<Vec<_>, Error>>()?;
let serialized_version_manifests = patched_version_manifests
.iter()
.map(|x| serde_json::to_vec(x).map(bytes::Bytes::from))
.collect::<Result<Vec<_>, serde_json::Error>>()?;
serialized_version_manifests
.into_iter()
.enumerate()
.for_each(|(index, bytes)| {
let loader = fetch_fabric_versions[index];
let version_path = format!(
"{mod_loader}/v{format_version}/versions/{}.json",
loader.version
);
upload_files.insert(
version_path,
UploadFile {
file: bytes,
content_type: Some("application/json".to_string()),
},
);
});
.await?;
}
if !fetch_fabric_versions.is_empty()
@@ -240,17 +303,20 @@ async fn fetch(
let loader_versions = daedalus::modded::Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: fabric_manifest
.loader
.into_iter()
version_group: None,
loaders: all_loader_versions
.iter()
.filter(|x| !skip_versions.contains(&&*x.version))
.map(|x| {
let version_path = format!(
"{mod_loader}/v{format_version}/versions/{}.json",
x.version,
let version_path = metadata_version_path(
mod_loader,
format_version,
&x.version,
UNIVERSAL_METADATA_GROUP,
);
daedalus::modded::LoaderVersion {
id: x.version,
id: x.version.clone(),
url: format_url(&version_path),
stable: x.stable,
}
@@ -260,14 +326,16 @@ async fn fetch(
let manifest = daedalus::modded::Manifest {
game_versions: std::iter::once(loader_versions)
.chain(fabric_manifest.game.into_iter().map(|x| {
.chain(all_game_versions.into_iter().map(|x| {
daedalus::modded::Version {
id: x.version,
stable: x.stable,
version_group: None,
loaders: vec![],
}
}))
.collect(),
version_groups: Vec::new(),
};
upload_files.insert(
@@ -285,6 +353,145 @@ async fn fetch(
})
}
struct ProfileRequest {
group: String,
loader_profile_template_game_version: String,
game_versions: Vec<String>,
loader_version: String,
url: String,
}
fn metadata_version_path(
mod_loader: &str,
format_version: usize,
loader_version: &str,
group: &str,
) -> String {
if group == UNIVERSAL_METADATA_GROUP {
format!("{mod_loader}/v{format_version}/versions/{loader_version}.json")
} else {
format!(
"{mod_loader}/v{format_version}/version-group/{group}/loader-version/{loader_version}"
)
}
}
async fn fetch_metadata_profiles(
mod_loader: &str,
format_version: usize,
maven_url: &str,
profile_requests: Vec<ProfileRequest>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
semaphore: &Arc<Semaphore>,
) -> Result<(), Error> {
let version_manifests = futures::future::try_join_all(
profile_requests
.iter()
.map(|x| download_file(&x.url, None, semaphore)),
)
.await?
.into_iter()
.map(|x| serde_json::from_slice(&x))
.collect::<Result<Vec<PartialVersionInfo>, serde_json::Error>>()?;
let patched_version_manifests = version_manifests
.into_iter()
.zip(profile_requests.iter())
.map(|(mut version_info, request)| {
patch_version_info(
&mut version_info,
&request.loader_profile_template_game_version,
&request.game_versions,
maven_url,
mirror_artifacts,
)?;
Ok(version_info)
})
.collect::<Result<Vec<_>, Error>>()?;
let serialized_version_manifests = patched_version_manifests
.iter()
.map(|x| serde_json::to_vec(x).map(bytes::Bytes::from))
.collect::<Result<Vec<_>, serde_json::Error>>()?;
serialized_version_manifests
.into_iter()
.zip(profile_requests)
.for_each(|(bytes, request)| {
let version_path = metadata_version_path(
mod_loader,
format_version,
&request.loader_version,
&request.group,
);
upload_files.insert(
version_path,
UploadFile {
file: bytes,
content_type: Some("application/json".to_string()),
},
);
});
Ok(())
}
fn patch_version_info(
version_info: &mut PartialVersionInfo,
game_version: &str,
game_versions: &[String],
maven_url: &str,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
for lib in &mut version_info.libraries {
let new_name = lib.name.replace(game_version, DUMMY_REPLACE_STRING);
// Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
if &*lib.name == "net.minecraft:launchwrapper:1.12" {
lib.url = Some("https://libraries.minecraft.net/".to_string());
}
let source_url =
lib.url.clone().unwrap_or_else(|| maven_url.to_string());
if lib.name == new_name {
insert_mirrored_artifact(
&new_name,
None,
vec![source_url],
false,
mirror_artifacts,
)?;
} else {
for concrete_game_version in game_versions {
let concrete_name =
lib.name.replace(game_version, concrete_game_version);
insert_mirrored_artifact(
&concrete_name,
None,
vec![source_url.clone()],
false,
mirror_artifacts,
)?;
}
lib.name = new_name;
}
lib.url = Some(format_url("maven/"));
}
version_info.id =
version_info.id.replace(game_version, DUMMY_REPLACE_STRING);
version_info.inherits_from = version_info
.inherits_from
.replace(game_version, DUMMY_REPLACE_STRING);
Ok(())
}
#[derive(Deserialize, Debug, Clone)]
struct FabricVersions {
pub loader: Vec<FabricLoaderVersion>,
apps/daedalus_client/src/forge.rs
+58 -20
View File
@@ -12,7 +12,10 @@ use itertools::Itertools;
use serde::Deserialize;
use serde::de::DeserializeOwned;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::Semaphore;
#[tracing::instrument(skip(semaphore))]
@@ -243,12 +246,45 @@ async fn fetch(
};
if !fetch_versions.is_empty() {
let forge_installers = futures::future::try_join_all(
fetch_versions
.iter()
.map(|x| download_file(&x.installer_url, None, &semaphore)),
)
.await?;
let total_installers = fetch_versions.len();
let downloaded_installers = Arc::new(AtomicUsize::new(0));
tracing::info!(
mod_loader,
total_files = total_installers,
"Downloading loader installers"
);
let forge_installers =
futures::future::try_join_all(fetch_versions.iter().map(|x| {
let downloaded_installers = downloaded_installers.clone();
let semaphore = semaphore.clone();
async move {
let installer =
download_file(&x.installer_url, None, &semaphore)
.await?;
let downloaded = downloaded_installers
.fetch_add(1, Ordering::Relaxed)
+ 1;
if downloaded.is_multiple_of(100)
|| downloaded == total_installers
{
tracing::info!(
mod_loader,
downloaded_files = downloaded,
remaining_files =
total_installers.saturating_sub(downloaded),
total_files = total_installers,
"Downloaded loader installers"
);
}
Ok::<_, Error>(installer)
}
}))
.await?;
#[tracing::instrument(skip(raw, upload_files, mirror_artifacts))]
async fn read_forge_installer(
@@ -761,21 +797,23 @@ async fn fetch(
.into_iter()
.map(|(game_version, loaders)| {
daedalus::modded::Version {
id: game_version,
stable: true,
loaders: loaders
.map(|x| daedalus::modded::LoaderVersion {
url: format_url(&format!(
"{mod_loader}/v{format_version}/versions/{}.json",
x.loader_version
)),
id: x.loader_version,
stable: false,
})
.collect(),
}
id: game_version,
stable: true,
version_group: None,
loaders: loaders
.map(|x| daedalus::modded::LoaderVersion {
url: format_url(&format!(
"{mod_loader}/v{format_version}/versions/{}.json",
x.loader_version
)),
id: x.loader_version,
stable: false,
})
.collect(),
}
})
.collect(),
version_groups: Vec::new(),
};
upload_files.insert(
apps/daedalus_client/src/main.rs
+208 -41
View File
@@ -1,10 +1,14 @@
use crate::util::{
REQWEST_CLIENT, format_url, upload_file_to_bucket,
upload_url_to_bucket_mirrors,
upload_url_to_bucket_mirrors, write_file_to_local_output,
write_url_to_local_output_mirrors,
};
use daedalus::get_path_from_artifact;
use dashmap::{DashMap, DashSet};
use std::sync::Arc;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::Semaphore;
use tracing_error::ErrorLayer;
use tracing_subscriber::{EnvFilter, fmt, prelude::*};
@@ -12,6 +16,7 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*};
mod error;
mod fabric;
mod forge;
mod metadata_groups;
mod minecraft;
pub mod util;
@@ -44,51 +49,92 @@ async fn main() -> Result<()> {
));
let mut fetch_result = FetchResult::default();
let only_loader = dotenvy::var("DAEDALUS_ONLY").ok();
match minecraft::fetch(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
if should_fetch(only_loader.as_deref(), "minecraft") {
match minecraft::fetch(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
}
}
match fabric::fetch_fabric(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
if should_fetch(only_loader.as_deref(), "fabric") {
match fabric::fetch_fabric(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
}
}
match fabric::fetch_quilt(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
if should_fetch(only_loader.as_deref(), "quilt") {
match fabric::fetch_quilt(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
}
}
match forge::fetch_neo(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
if should_fetch(only_loader.as_deref(), "neo") {
match forge::fetch_neo(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
}
}
match forge::fetch_forge(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
if should_fetch(only_loader.as_deref(), "forge") {
match forge::fetch_forge(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
}
}
let FetchResult {
upload_files,
mirror_artifacts,
} = fetch_result;
let upload_file_total = upload_files.len();
let mirror_file_total = mirror_artifacts.len();
futures::future::try_join_all(upload_files.iter().map(|entry| {
upload_file_to_bucket(
entry.key().clone(),
entry.value().file.clone(),
entry.value().content_type.clone(),
&semaphore,
)
}))
.await?;
if dotenvy::var("LOCAL_OUTPUT_DIR").is_ok() {
let written_files = Arc::new(AtomicUsize::new(0));
futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
upload_url_to_bucket_mirrors(
format!("maven/{}", entry.key()),
entry
tracing::info!(
total_files = upload_file_total,
"Writing local metadata files"
);
futures::future::try_join_all(upload_files.iter().map(|entry| {
let path = entry.key().clone();
let file = entry.value().file.clone();
let written_files = written_files.clone();
async move {
write_file_to_local_output(&path, file).await?;
let written = written_files.fetch_add(1, Ordering::Relaxed) + 1;
if written.is_multiple_of(100) || written == upload_file_total {
tracing::info!(
written_files = written,
remaining_files =
upload_file_total.saturating_sub(written),
total_files = upload_file_total,
"Wrote local metadata files"
);
}
Ok::<_, Error>(())
}
}))
.await?;
let written_mirror_files = Arc::new(AtomicUsize::new(0));
tracing::info!(
total_files = mirror_file_total,
"Writing local mirror files"
);
futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
let path = format!("maven/{}", entry.key());
let mirrors = entry
.value()
.mirrors
.iter()
@@ -99,12 +145,117 @@ async fn main() -> Result<()> {
format!("{}{}", mirror.path, entry.key())
}
})
.collect(),
entry.value().sha1.clone(),
&semaphore,
)
}))
.await?;
.collect();
let sha1 = entry.value().sha1.clone();
let written_mirror_files = written_mirror_files.clone();
let semaphore = semaphore.clone();
async move {
write_url_to_local_output_mirrors(
path, mirrors, sha1, &semaphore,
)
.await?;
let written =
written_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
if written.is_multiple_of(100) || written == mirror_file_total {
tracing::info!(
written_files = written,
remaining_files =
mirror_file_total.saturating_sub(written),
total_files = mirror_file_total,
"Wrote local mirror files"
);
}
Ok::<_, Error>(())
}
}))
.await?;
} else {
let uploaded_files = Arc::new(AtomicUsize::new(0));
tracing::info!(
total_files = upload_file_total,
"Uploading metadata files"
);
futures::future::try_join_all(upload_files.iter().map(|entry| {
let path = entry.key().clone();
let file = entry.value().file.clone();
let content_type = entry.value().content_type.clone();
let uploaded_files = uploaded_files.clone();
let semaphore = semaphore.clone();
async move {
upload_file_to_bucket(path, file, content_type, &semaphore)
.await?;
let uploaded =
uploaded_files.fetch_add(1, Ordering::Relaxed) + 1;
if uploaded.is_multiple_of(100) || uploaded == upload_file_total
{
tracing::info!(
uploaded_files = uploaded,
remaining_files =
upload_file_total.saturating_sub(uploaded),
total_files = upload_file_total,
"Uploaded metadata files"
);
}
Ok::<_, Error>(())
}
}))
.await?;
let uploaded_mirror_files = Arc::new(AtomicUsize::new(0));
tracing::info!(
total_files = mirror_file_total,
"Uploading mirror files"
);
futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
let path = format!("maven/{}", entry.key());
let mirrors = entry
.value()
.mirrors
.iter()
.map(|mirror| {
if mirror.entire_url {
mirror.path.clone()
} else {
format!("{}{}", mirror.path, entry.key())
}
})
.collect();
let sha1 = entry.value().sha1.clone();
let uploaded_mirror_files = uploaded_mirror_files.clone();
let semaphore = semaphore.clone();
async move {
upload_url_to_bucket_mirrors(path, mirrors, sha1, &semaphore)
.await?;
let uploaded =
uploaded_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
if uploaded.is_multiple_of(100) || uploaded == mirror_file_total
{
tracing::info!(
uploaded_files = uploaded,
remaining_files =
mirror_file_total.saturating_sub(uploaded),
total_files = mirror_file_total,
"Uploaded mirror files"
);
}
Ok::<_, Error>(())
}
}))
.await?;
}
if dotenvy::var("CLOUDFLARE_INTEGRATION")
.ok()
@@ -151,6 +302,20 @@ async fn main() -> Result<()> {
Ok(())
}
fn should_fetch(only_loader: Option<&str>, loader: &str) -> bool {
let Some(only_loader) = only_loader else {
return true;
};
only_loader.split(',').any(|entry| {
let entry = entry.trim();
entry.eq_ignore_ascii_case("all")
|| entry.eq_ignore_ascii_case(loader)
|| (loader == "neo" && entry.eq_ignore_ascii_case("neoforge"))
})
}
pub struct UploadFile {
file: bytes::Bytes,
content_type: Option<String>,
@@ -248,11 +413,13 @@ fn check_env_vars() -> bool {
failed |= check_var::<String>("BASE_URL");
failed |= check_var::<String>("S3_ACCESS_TOKEN");
failed |= check_var::<String>("S3_SECRET");
failed |= check_var::<String>("S3_URL");
failed |= check_var::<String>("S3_REGION");
failed |= check_var::<String>("S3_BUCKET_NAME");
if dotenvy::var("LOCAL_OUTPUT_DIR").is_err() {
failed |= check_var::<String>("S3_ACCESS_TOKEN");
failed |= check_var::<String>("S3_SECRET");
failed |= check_var::<String>("S3_URL");
failed |= check_var::<String>("S3_REGION");
failed |= check_var::<String>("S3_BUCKET_NAME");
}
if dotenvy::var("CLOUDFLARE_INTEGRATION")
.ok()
apps/daedalus_client/src/metadata_groups.rs
+145
View File
@@ -0,0 +1,145 @@
//! Determines how version info is generated for pairs of game and loader
//! versions.
//!
//! When a user installs a version of the game, they install two things: the
//! game (of some specific version), and a loader (of some other specific
//! version). Each combination of game and loader version requires a specific
//! configuration, like a specific set of libraries that must be downloaded and
//! run along with the game. However, some versions of the game or loader may
//! change configuration requirements without the other version being affected.
//! For example, pre-26.x game versions with Quilt require the Quilt `hashed`
//! libraries to also be downloaded. However, 26.x and later don't require the
//! `hashed` libraries, and don't even have a download for them. The problem is
//! that Quilt loader 0.30.0 can be used for both pre-26.x and 26.x - but our v0
//! manifest files can't differentiate the two. The result is that you either
//! break compatibility for 0.30.0 game versions pre-26.x, or break 0.30.0
//! on 26.x and later.
//!
//! To fix this, v1 introduces the concept of *version groups*: game versions
//! before 26.x are version group v1, and 26.x and later are v2. Then, we
//! parameterize our version info on both version group and loader version,
//! letting us specify the right configuration based on both game version and
//! loader version.
//!
//! Why not parameterize on game version and loader version directly? Most game
//! versions have the same configuration as their surrounding game versions, so
//! we'd end up with many duplicate configurations: the number of game versions
//! multiplied by the number of loader versions.
//!
//! This file lets you configure what game versions are grouped together.
//!
//! Each version group is templated from a specific game version - e.g. game
//! version 1.21 is used as the template file for 1.20, 1.19, etc.
pub const UNIVERSAL_METADATA_GROUP: &str = "universal";
pub const QUILT_LEGACY_METADATA_GROUP: &str = "v1";
pub const QUILT_MODERN_METADATA_GROUP: &str = "v2";
pub struct MetadataGroup {
pub id: &'static str,
/// Minecraft version used to fetch and template this group's loader profiles.
pub loader_profile_template_game_version: String,
pub game_versions: Vec<String>,
}
pub fn metadata_groups<'a>(
mod_loader: &str,
game_versions: impl IntoIterator<Item = &'a str>,
) -> Vec<MetadataGroup> {
// Non-Quilt loaders don't need the concept of version groups, so we just
// make one "universal" group, and template it on 1.21.
if mod_loader != "quilt" {
return vec![MetadataGroup {
id: UNIVERSAL_METADATA_GROUP,
loader_profile_template_game_version: "1.21".to_string(),
game_versions: game_versions
.into_iter()
.map(|x| x.to_string())
.collect(),
}];
}
let game_versions = game_versions.into_iter().collect::<Vec<_>>();
let legacy_game_versions = game_versions
.iter()
.copied()
.filter(|game_version| {
metadata_group_id_for_game_version(mod_loader, game_version)
== QUILT_LEGACY_METADATA_GROUP
})
.collect::<Vec<_>>();
let modern_game_versions = game_versions
.iter()
.copied()
.filter(|game_version| {
metadata_group_id_for_game_version(mod_loader, game_version)
== QUILT_MODERN_METADATA_GROUP
})
.collect::<Vec<_>>();
let mut groups = Vec::new();
if !legacy_game_versions.is_empty() {
groups.push(MetadataGroup {
id: QUILT_LEGACY_METADATA_GROUP,
loader_profile_template_game_version: legacy_game_versions
.iter()
.find(|x| **x == "1.21")
.copied()
.unwrap_or(legacy_game_versions[0])
.to_string(),
game_versions: legacy_game_versions
.iter()
.map(|x| x.to_string())
.collect(),
});
}
if !modern_game_versions.is_empty() {
groups.push(MetadataGroup {
id: QUILT_MODERN_METADATA_GROUP,
loader_profile_template_game_version: modern_game_versions[0]
.to_string(),
game_versions: modern_game_versions
.iter()
.map(|x| x.to_string())
.collect(),
});
}
groups
}
pub fn metadata_group_for_game_version<'a>(
groups: &'a [MetadataGroup],
mod_loader: &str,
game_version: &str,
) -> Option<&'a MetadataGroup> {
let group_id = metadata_group_id_for_game_version(mod_loader, game_version);
groups.iter().find(|group| group.id == group_id)
}
fn metadata_group_id_for_game_version(
mod_loader: &str,
game_version: &str,
) -> &'static str {
if mod_loader == "quilt" && is_modern_quilt_game_version(game_version) {
QUILT_MODERN_METADATA_GROUP
} else if mod_loader == "quilt" {
QUILT_LEGACY_METADATA_GROUP
} else {
UNIVERSAL_METADATA_GROUP
}
}
// Update these Quilt group boundaries if upstream loader profiles gain another
// structural incompatibility between Minecraft versions.
fn is_modern_quilt_game_version(game_version: &str) -> bool {
let major = game_version
.split(['.', 'w'])
.next()
.and_then(|x| x.parse::<usize>().ok());
major.is_some_and(|x| x >= 26)
}
apps/daedalus_client/src/util.rs
+85 -1
View File
@@ -3,7 +3,11 @@ use bytes::Bytes;
use s3::creds::Credentials;
use s3::{Bucket, Region};
use serde::de::DeserializeOwned;
use std::sync::{Arc, LazyLock};
use std::path::PathBuf;
use std::sync::{
Arc, LazyLock,
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::Semaphore;
static BUCKET: LazyLock<Bucket> = LazyLock::new(|| {
@@ -55,6 +59,8 @@ pub static REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
.unwrap()
});
static DOWNLOADED_FILE_COUNT: AtomicUsize = AtomicUsize::new(0);
#[tracing::instrument(skip(bytes, semaphore))]
pub async fn upload_file_to_bucket(
path: String,
@@ -135,6 +141,74 @@ pub async fn upload_url_to_bucket(
Ok(())
}
pub async fn write_file_to_local_output(
path: &str,
bytes: Bytes,
) -> Result<(), Error> {
let output_path = local_output_path(path)?;
if let Some(parent) = output_path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::write(output_path, bytes).await?;
Ok(())
}
pub async fn write_url_to_local_output_mirrors(
output_path: String,
mirrors: Vec<String>,
sha1: Option<String>,
semaphore: &Arc<Semaphore>,
) -> Result<(), Error> {
if mirrors.is_empty() {
return Err(ErrorKind::InvalidInput(
"No mirrors provided!".to_string(),
)
.into());
}
for (index, mirror) in mirrors.iter().enumerate() {
let result = write_url_to_local_output(
output_path.clone(),
mirror.clone(),
sha1.clone(),
semaphore,
)
.await;
if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
return result;
}
}
unreachable!()
}
async fn write_url_to_local_output(
path: String,
url: String,
sha1: Option<String>,
semaphore: &Arc<Semaphore>,
) -> Result<(), Error> {
let data = download_file(&url, sha1.as_deref(), semaphore).await?;
write_file_to_local_output(&path, data).await?;
Ok(())
}
fn local_output_path(path: &str) -> Result<PathBuf, Error> {
let output_dir = dotenvy::var("LOCAL_OUTPUT_DIR").map_err(|_| {
ErrorKind::InvalidInput(
"LOCAL_OUTPUT_DIR is required for local output".to_string(),
)
})?;
Ok(PathBuf::from(output_dir).join(path))
}
#[tracing::instrument(skip(bytes))]
pub async fn sha1_async(bytes: Bytes) -> Result<String, Error> {
let hash = tokio::task::spawn_blocking(move || {
@@ -182,6 +256,16 @@ pub async fn download_file(
}
}
let downloaded = DOWNLOADED_FILE_COUNT
.fetch_add(1, Ordering::Relaxed)
+ 1;
if downloaded.is_multiple_of(100) {
tracing::info!(
downloaded_files = downloaded,
"Downloaded metadata files"
);
}
return Ok(bytes);
} else if attempt <= RETRIES {
continue;
packages/daedalus/src/modded.rs
+81 -1
View File
@@ -10,10 +10,73 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
/// The latest version of the format the fabric model structs deserialize to
pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
/// The latest version of the format the quilt model structs deserialize to
pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0;
pub const CURRENT_QUILT_FORMAT_VERSION: usize = 1;
/// The latest version of the format the neoforge model structs deserialize to
pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0;
/// Metadata for locating and caching a loader manifest.
#[derive(Debug, Clone)]
pub struct LoaderManifestMetadata {
/// The canonical loader name used in launcher-meta paths.
pub loader: String,
/// The latest manifest format version for this loader.
pub format_version: usize,
/// The cache key that includes the loader format version.
pub cache_key: String,
/// The launcher-meta path to the manifest.
pub path: String,
}
/// Returns metadata for the latest manifest format for the provided loader.
pub fn loader_manifest_metadata(loader: &str) -> LoaderManifestMetadata {
let format_version = current_loader_manifest_format_version(loader);
let cache_key = format!("{loader}-v{format_version}");
let path = format!("{loader}/v{format_version}/manifest.json");
LoaderManifestMetadata {
loader: loader.to_string(),
format_version,
cache_key,
path,
}
}
/// Returns loader manifest metadata from a versioned cache key.
pub fn loader_manifest_metadata_from_cache_key(
cache_key: &str,
) -> LoaderManifestMetadata {
if let Some((loader, format_version)) =
cache_key.rsplit_once("-v").and_then(|(loader, version)| {
version
.parse::<usize>()
.ok()
.map(|version| (loader, version))
})
{
let cache_key = format!("{loader}-v{format_version}");
let path = format!("{loader}/v{format_version}/manifest.json");
LoaderManifestMetadata {
loader: loader.to_string(),
format_version,
cache_key,
path,
}
} else {
loader_manifest_metadata(cache_key)
}
}
fn current_loader_manifest_format_version(loader: &str) -> usize {
match loader {
"fabric" => CURRENT_FABRIC_FORMAT_VERSION,
"forge" => CURRENT_FORGE_FORMAT_VERSION,
"quilt" => CURRENT_QUILT_FORMAT_VERSION,
"neo" => CURRENT_NEOFORGE_FORMAT_VERSION,
_ => 0,
}
}
/// The dummy replace string library names, inheritsFrom, and version names should be replaced with
pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";
@@ -188,19 +251,36 @@ pub fn merge_partial_version(
pub struct Manifest {
/// The game versions the mod loader supports
pub game_versions: Vec<Version>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
/// Groups of game versions that share compatible loader version profiles
pub version_groups: Vec<VersionGroup>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
/// A game version of Minecraft
pub struct Version {
/// The minecraft version ID
pub id: String,
/// Whether the release is stable or not
pub stable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
/// The loader profile group for this Minecraft version
pub version_group: Option<String>,
/// A map that contains loader versions for the game version
pub loaders: Vec<LoaderVersion>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
/// A group of Minecraft versions that share loader version profiles
pub struct VersionGroup {
/// The version group ID
pub id: String,
/// The loader versions for this version group
pub loaders: Vec<LoaderVersion>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A version of a Minecraft mod loader
pub struct LoaderVersion {