Split frontend changes out of Daedalus branch

apps/daedalus_client/src/error.rs

@@ -27,6 +27,8 @@ pub enum ErrorKind {
         inner: Box<s3::error::S3Error>,
         file: String,
     },
+    #[error("I/O error: {0}")]
+    Io(#[from] std::io::Error),
     #[error("Error acquiring semaphore: {0}")]
     Acquire(#[from] tokio::sync::AcquireError),
     #[error("Tracing error: {0}")]

apps/daedalus_client/src/fabric.rs

@@ -1,3 +1,19 @@
+//! Fetches Fabric-compatible loader metadata.
+//!
+//! Fabric and Quilt both expose loader profiles for a concrete Minecraft
+//! version, but Daedalus publishes templated profiles using
+//! `${modrinth.gameVersion}`. A group is a set of Minecraft versions whose
+//! upstream loader profiles have the same structure after the concrete
+//! Minecraft version is replaced with `${modrinth.gameVersion}`. Fabric uses
+//! one universal group, so its public profile paths stay as
+//! `versions/{loader}.json`. Quilt has more than one group: versions before
+//! 26.x include hashed/intermediary libraries, while 26.x versions do not. For
+//! Quilt, Daedalus writes one templated profile per group at
+//! `version-group/{group}/loader-version/{loader}`.
+
+use crate::metadata_groups::{
+    UNIVERSAL_METADATA_GROUP, metadata_group_for_game_version, metadata_groups,
+};
 use crate::util::{download_file, fetch_json, format_url};
 use crate::{
     Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
@@ -64,7 +80,112 @@ async fn fetch(
         &semaphore,
     )
     .await?;
+    let all_loader_versions = fabric_manifest.loader.clone();
+    let all_game_versions = fabric_manifest.game.clone();
+    let metadata_groups = metadata_groups(
+        mod_loader,
+        all_game_versions.iter().map(|x| x.version.as_str()),
+    );
 
+    if metadata_groups
+        .iter()
+        .any(|group| group.id != UNIVERSAL_METADATA_GROUP)
+    {
+        let loaders = all_loader_versions
+            .iter()
+            .filter(|x| !skip_versions.contains(&&*x.version))
+            .collect::<Vec<_>>();
+
+        let profile_requests = metadata_groups
+            .iter()
+            .flat_map(|group| {
+                loaders.iter().map(move |loader| ProfileRequest {
+                    group: group.id.to_string(),
+                    loader_profile_template_game_version: group
+                        .loader_profile_template_game_version
+                        .clone(),
+                    game_versions: group.game_versions.clone(),
+                    loader_version: loader.version.clone(),
+                    url: format!(
+                        "{}/versions/loader/{}/{}/profile/json",
+                        meta_url,
+                        group.loader_profile_template_game_version,
+                        loader.version
+                    ),
+                })
+            })
+            .collect::<Vec<_>>();
+
+        fetch_metadata_profiles(
+            mod_loader,
+            format_version,
+            maven_url,
+            profile_requests,
+            &upload_files,
+            &mirror_artifacts,
+            &semaphore,
+        )
+        .await?;
+
+        let version_groups = metadata_groups
+            .iter()
+            .map(|group| daedalus::modded::VersionGroup {
+                id: group.id.to_string(),
+                loaders: loaders
+                    .iter()
+                    .map(|loader| {
+                        let version_path = metadata_version_path(
+                            mod_loader,
+                            format_version,
+                            &loader.version,
+                            group.id,
+                        );
+
+                        daedalus::modded::LoaderVersion {
+                            id: loader.version.clone(),
+                            url: format_url(&version_path),
+                            stable: loader.stable,
+                        }
+                    })
+                    .collect(),
+            })
+            .collect();
+
+        let manifest = daedalus::modded::Manifest {
+            game_versions: all_game_versions
+                .into_iter()
+                .map(|game_version| {
+                    let group = metadata_group_for_game_version(
+                        &metadata_groups,
+                        mod_loader,
+                        &game_version.version,
+                    )
+                    .expect("game version should have a metadata group");
+
+                    daedalus::modded::Version {
+                        id: game_version.version.clone(),
+                        stable: game_version.stable,
+                        version_group: Some(group.id.to_string()),
+                        loaders: Vec::new(),
+                    }
+                })
+                .collect(),
+            version_groups,
+        };
+
+        upload_files.insert(
+            format!("{mod_loader}/v{format_version}/manifest.json"),
+            UploadFile {
+                file: bytes::Bytes::from(serde_json::to_vec(&manifest)?),
+                content_type: Some("application/json".to_string()),
+            },
+        );
+
+        return Ok(FetchResult {
+            upload_files,
+            mirror_artifacts,
+        });
+    }
     // We check Modrinth's manifest to find newly added loader versions,
     // intermediary/mapping artifacts, and game versions.
     let (
@@ -125,8 +246,6 @@ async fn fetch(
         )
     };
 
-    const DUMMY_GAME_VERSION: &str = "1.21";
-
     if !fetch_intermediary_versions.is_empty() {
         for x in &fetch_intermediary_versions {
             insert_mirrored_artifact(
@@ -140,94 +259,38 @@ async fn fetch(
     }
 
     if !fetch_fabric_versions.is_empty() {
-        let fabric_version_manifest_urls = fetch_fabric_versions
+        let universal_group = metadata_groups
             .iter()
-            .map(|x| {
-                format!(
+            .find(|group| group.id == UNIVERSAL_METADATA_GROUP)
+            .expect("fabric metadata should have a universal group");
+        let profile_requests = fetch_fabric_versions
+            .iter()
+            .map(|loader| ProfileRequest {
+                group: universal_group.id.to_string(),
+                loader_profile_template_game_version: universal_group
+                    .loader_profile_template_game_version
+                    .clone(),
+                game_versions: universal_group.game_versions.clone(),
+                loader_version: loader.version.clone(),
+                url: format!(
                     "{}/versions/loader/{}/{}/profile/json",
-                    meta_url, DUMMY_GAME_VERSION, x.version
-                )
+                    meta_url,
+                    universal_group.loader_profile_template_game_version,
+                    loader.version
+                ),
             })
             .collect::<Vec<_>>();
-        let fabric_version_manifests = futures::future::try_join_all(
-            fabric_version_manifest_urls
-                .iter()
-                .map(|x| download_file(x, None, &semaphore)),
+
+        fetch_metadata_profiles(
+            mod_loader,
+            format_version,
+            maven_url,
+            profile_requests,
+            &upload_files,
+            &mirror_artifacts,
+            &semaphore,
         )
-        .await?
-        .into_iter()
-        .map(|x| serde_json::from_slice(&x))
-        .collect::<Result<Vec<PartialVersionInfo>, serde_json::Error>>()?;
-
-        let patched_version_manifests = fabric_version_manifests
-            .into_iter()
-            .map(|mut version_info| {
-                for lib in &mut version_info.libraries {
-                    let new_name = lib
-                        .name
-                        .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
-
-                    // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
-                    if &*lib.name == "net.minecraft:launchwrapper:1.12" {
-                        lib.url = Some(
-                            "https://libraries.minecraft.net/".to_string(),
-                        );
-                    }
-
-                    // If a library is not intermediary, we add it to mirror artifacts to be mirrored
-                    if lib.name == new_name {
-                        insert_mirrored_artifact(
-                            &new_name,
-                            None,
-                            vec![
-                                lib.url
-                                    .clone()
-                                    .unwrap_or_else(|| maven_url.to_string()),
-                            ],
-                            false,
-                            &mirror_artifacts,
-                        )?;
-                    } else {
-                        lib.name = new_name;
-                    }
-
-                    lib.url = Some(format_url("maven/"));
-                }
-
-                version_info.id = version_info
-                    .id
-                    .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
-                version_info.inherits_from = version_info
-                    .inherits_from
-                    .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
-
-                Ok(version_info)
-            })
-            .collect::<Result<Vec<_>, Error>>()?;
-        let serialized_version_manifests = patched_version_manifests
-            .iter()
-            .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from))
-            .collect::<Result<Vec<_>, serde_json::Error>>()?;
-
-        serialized_version_manifests
-            .into_iter()
-            .enumerate()
-            .for_each(|(index, bytes)| {
-                let loader = fetch_fabric_versions[index];
-
-                let version_path = format!(
-                    "{mod_loader}/v{format_version}/versions/{}.json",
-                    loader.version
-                );
-
-                upload_files.insert(
-                    version_path,
-                    UploadFile {
-                        file: bytes,
-                        content_type: Some("application/json".to_string()),
-                    },
-                );
-            });
+        .await?;
     }
 
     if !fetch_fabric_versions.is_empty()
@@ -240,17 +303,20 @@ async fn fetch(
         let loader_versions = daedalus::modded::Version {
             id: DUMMY_REPLACE_STRING.to_string(),
             stable: true,
-            loaders: fabric_manifest
-                .loader
-                .into_iter()
+            version_group: None,
+            loaders: all_loader_versions
+                .iter()
+                .filter(|x| !skip_versions.contains(&&*x.version))
                 .map(|x| {
-                    let version_path = format!(
-                        "{mod_loader}/v{format_version}/versions/{}.json",
-                        x.version,
+                    let version_path = metadata_version_path(
+                        mod_loader,
+                        format_version,
+                        &x.version,
+                        UNIVERSAL_METADATA_GROUP,
                     );
 
                     daedalus::modded::LoaderVersion {
-                        id: x.version,
+                        id: x.version.clone(),
                         url: format_url(&version_path),
                         stable: x.stable,
                     }
@@ -260,14 +326,16 @@ async fn fetch(
 
         let manifest = daedalus::modded::Manifest {
             game_versions: std::iter::once(loader_versions)
-                .chain(fabric_manifest.game.into_iter().map(|x| {
+                .chain(all_game_versions.into_iter().map(|x| {
                     daedalus::modded::Version {
                         id: x.version,
                         stable: x.stable,
+                        version_group: None,
                         loaders: vec![],
                     }
                 }))
                 .collect(),
+            version_groups: Vec::new(),
         };
 
         upload_files.insert(
@@ -285,6 +353,145 @@ async fn fetch(
     })
 }
 
+struct ProfileRequest {
+    group: String,
+    loader_profile_template_game_version: String,
+    game_versions: Vec<String>,
+    loader_version: String,
+    url: String,
+}
+
+fn metadata_version_path(
+    mod_loader: &str,
+    format_version: usize,
+    loader_version: &str,
+    group: &str,
+) -> String {
+    if group == UNIVERSAL_METADATA_GROUP {
+        format!("{mod_loader}/v{format_version}/versions/{loader_version}.json")
+    } else {
+        format!(
+            "{mod_loader}/v{format_version}/version-group/{group}/loader-version/{loader_version}"
+        )
+    }
+}
+
+async fn fetch_metadata_profiles(
+    mod_loader: &str,
+    format_version: usize,
+    maven_url: &str,
+    profile_requests: Vec<ProfileRequest>,
+    upload_files: &DashMap<String, UploadFile>,
+    mirror_artifacts: &DashMap<String, MirrorArtifact>,
+    semaphore: &Arc<Semaphore>,
+) -> Result<(), Error> {
+    let version_manifests = futures::future::try_join_all(
+        profile_requests
+            .iter()
+            .map(|x| download_file(&x.url, None, semaphore)),
+    )
+    .await?
+    .into_iter()
+    .map(|x| serde_json::from_slice(&x))
+    .collect::<Result<Vec<PartialVersionInfo>, serde_json::Error>>()?;
+
+    let patched_version_manifests = version_manifests
+        .into_iter()
+        .zip(profile_requests.iter())
+        .map(|(mut version_info, request)| {
+            patch_version_info(
+                &mut version_info,
+                &request.loader_profile_template_game_version,
+                &request.game_versions,
+                maven_url,
+                mirror_artifacts,
+            )?;
+
+            Ok(version_info)
+        })
+        .collect::<Result<Vec<_>, Error>>()?;
+    let serialized_version_manifests = patched_version_manifests
+        .iter()
+        .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from))
+        .collect::<Result<Vec<_>, serde_json::Error>>()?;
+
+    serialized_version_manifests
+        .into_iter()
+        .zip(profile_requests)
+        .for_each(|(bytes, request)| {
+            let version_path = metadata_version_path(
+                mod_loader,
+                format_version,
+                &request.loader_version,
+                &request.group,
+            );
+
+            upload_files.insert(
+                version_path,
+                UploadFile {
+                    file: bytes,
+                    content_type: Some("application/json".to_string()),
+                },
+            );
+        });
+
+    Ok(())
+}
+
+fn patch_version_info(
+    version_info: &mut PartialVersionInfo,
+    game_version: &str,
+    game_versions: &[String],
+    maven_url: &str,
+    mirror_artifacts: &DashMap<String, MirrorArtifact>,
+) -> Result<(), Error> {
+    for lib in &mut version_info.libraries {
+        let new_name = lib.name.replace(game_version, DUMMY_REPLACE_STRING);
+
+        // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries
+        if &*lib.name == "net.minecraft:launchwrapper:1.12" {
+            lib.url = Some("https://libraries.minecraft.net/".to_string());
+        }
+        let source_url =
+            lib.url.clone().unwrap_or_else(|| maven_url.to_string());
+
+        if lib.name == new_name {
+            insert_mirrored_artifact(
+                &new_name,
+                None,
+                vec![source_url],
+                false,
+                mirror_artifacts,
+            )?;
+        } else {
+            for concrete_game_version in game_versions {
+                let concrete_name =
+                    lib.name.replace(game_version, concrete_game_version);
+
+                insert_mirrored_artifact(
+                    &concrete_name,
+                    None,
+                    vec![source_url.clone()],
+                    false,
+                    mirror_artifacts,
+                )?;
+            }
+
+            lib.name = new_name;
+        }
+
+        lib.url = Some(format_url("maven/"));
+    }
+
+    version_info.id =
+        version_info.id.replace(game_version, DUMMY_REPLACE_STRING);
+    version_info.inherits_from = version_info
+        .inherits_from
+        .replace(game_version, DUMMY_REPLACE_STRING);
+
+    Ok(())
+}
+
 #[derive(Deserialize, Debug, Clone)]
 struct FabricVersions {
     pub loader: Vec<FabricLoaderVersion>,

apps/daedalus_client/src/forge.rs

@@ -12,7 +12,10 @@ use itertools::Itertools;
 use serde::Deserialize;
 use serde::de::DeserializeOwned;
 use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 use tokio::sync::Semaphore;
 
 #[tracing::instrument(skip(semaphore))]
@@ -243,12 +246,45 @@ async fn fetch(
     };
 
     if !fetch_versions.is_empty() {
-        let forge_installers = futures::future::try_join_all(
-            fetch_versions
-                .iter()
-                .map(|x| download_file(&x.installer_url, None, &semaphore)),
-        )
-        .await?;
+        let total_installers = fetch_versions.len();
+        let downloaded_installers = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            mod_loader,
+            total_files = total_installers,
+            "Downloading loader installers"
+        );
+
+        let forge_installers =
+            futures::future::try_join_all(fetch_versions.iter().map(|x| {
+                let downloaded_installers = downloaded_installers.clone();
+                let semaphore = semaphore.clone();
+
+                async move {
+                    let installer =
+                        download_file(&x.installer_url, None, &semaphore)
+                            .await?;
+                    let downloaded = downloaded_installers
+                        .fetch_add(1, Ordering::Relaxed)
+                        + 1;
+
+                    if downloaded.is_multiple_of(100)
+                        || downloaded == total_installers
+                    {
+                        tracing::info!(
+                            mod_loader,
+                            downloaded_files = downloaded,
+                            remaining_files =
+                                total_installers.saturating_sub(downloaded),
+                            total_files = total_installers,
+                            "Downloaded loader installers"
+                        );
+                    }
+
+                    Ok::<_, Error>(installer)
+                }
+            }))
+            .await?;
 
         #[tracing::instrument(skip(raw, upload_files, mirror_artifacts))]
         async fn read_forge_installer(
@@ -761,21 +797,23 @@ async fn fetch(
                 .into_iter()
                 .map(|(game_version, loaders)| {
                     daedalus::modded::Version {
-                    id: game_version,
-                    stable: true,
-                    loaders: loaders
-                        .map(|x| daedalus::modded::LoaderVersion {
-                            url: format_url(&format!(
-                        "{mod_loader}/v{format_version}/versions/{}.json",
-                        x.loader_version
-                    )),
-                            id: x.loader_version,
-                            stable: false,
-                        })
-                        .collect(),
-                }
+                        id: game_version,
+                        stable: true,
+                        version_group: None,
+                        loaders: loaders
+                            .map(|x| daedalus::modded::LoaderVersion {
+                                url: format_url(&format!(
+                                    "{mod_loader}/v{format_version}/versions/{}.json",
+                                    x.loader_version
+                                )),
+                                id: x.loader_version,
+                                stable: false,
+                            })
+                            .collect(),
+                    }
                 })
                 .collect(),
+            version_groups: Vec::new(),
         };
 
         upload_files.insert(

apps/daedalus_client/src/main.rs

@@ -1,10 +1,14 @@
 use crate::util::{
     REQWEST_CLIENT, format_url, upload_file_to_bucket,
-    upload_url_to_bucket_mirrors,
+    upload_url_to_bucket_mirrors, write_file_to_local_output,
+    write_url_to_local_output_mirrors,
 };
 use daedalus::get_path_from_artifact;
 use dashmap::{DashMap, DashSet};
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 use tokio::sync::Semaphore;
 use tracing_error::ErrorLayer;
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
@@ -12,6 +16,7 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 mod error;
 mod fabric;
 mod forge;
+mod metadata_groups;
 mod minecraft;
 pub mod util;
 
@@ -44,51 +49,92 @@ async fn main() -> Result<()> {
     ));
 
     let mut fetch_result = FetchResult::default();
+    let only_loader = dotenvy::var("DAEDALUS_ONLY").ok();
 
-    match minecraft::fetch(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
+    if should_fetch(only_loader.as_deref(), "minecraft") {
+        match minecraft::fetch(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
+        }
     }
 
-    match fabric::fetch_fabric(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
+    if should_fetch(only_loader.as_deref(), "fabric") {
+        match fabric::fetch_fabric(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
+        }
     }
 
-    match fabric::fetch_quilt(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
+    if should_fetch(only_loader.as_deref(), "quilt") {
+        match fabric::fetch_quilt(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
+        }
     }
 
-    match forge::fetch_neo(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
+    if should_fetch(only_loader.as_deref(), "neo") {
+        match forge::fetch_neo(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
+        }
     }
 
-    match forge::fetch_forge(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
+    if should_fetch(only_loader.as_deref(), "forge") {
+        match forge::fetch_forge(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
+        }
     }
 
     let FetchResult {
         upload_files,
         mirror_artifacts,
     } = fetch_result;
+    let upload_file_total = upload_files.len();
+    let mirror_file_total = mirror_artifacts.len();
 
-    futures::future::try_join_all(upload_files.iter().map(|entry| {
-        upload_file_to_bucket(
-            entry.key().clone(),
-            entry.value().file.clone(),
-            entry.value().content_type.clone(),
-            &semaphore,
-        )
-    }))
-    .await?;
+    if dotenvy::var("LOCAL_OUTPUT_DIR").is_ok() {
+        let written_files = Arc::new(AtomicUsize::new(0));
 
-    futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
-        upload_url_to_bucket_mirrors(
-            format!("maven/{}", entry.key()),
-            entry
+        tracing::info!(
+            total_files = upload_file_total,
+            "Writing local metadata files"
+        );
+
+        futures::future::try_join_all(upload_files.iter().map(|entry| {
+            let path = entry.key().clone();
+            let file = entry.value().file.clone();
+            let written_files = written_files.clone();
+
+            async move {
+                write_file_to_local_output(&path, file).await?;
+                let written = written_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if written.is_multiple_of(100) || written == upload_file_total {
+                    tracing::info!(
+                        written_files = written,
+                        remaining_files =
+                            upload_file_total.saturating_sub(written),
+                        total_files = upload_file_total,
+                        "Wrote local metadata files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+
+        let written_mirror_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = mirror_file_total,
+            "Writing local mirror files"
+        );
+
+        futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
+            let path = format!("maven/{}", entry.key());
+            let mirrors = entry
                 .value()
                 .mirrors
                 .iter()
@@ -99,12 +145,117 @@ async fn main() -> Result<()> {
                         format!("{}{}", mirror.path, entry.key())
                     }
                 })
-                .collect(),
-            entry.value().sha1.clone(),
-            &semaphore,
-        )
-    }))
-    .await?;
+                .collect();
+            let sha1 = entry.value().sha1.clone();
+            let written_mirror_files = written_mirror_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                write_url_to_local_output_mirrors(
+                    path, mirrors, sha1, &semaphore,
+                )
+                .await?;
+                let written =
+                    written_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if written.is_multiple_of(100) || written == mirror_file_total {
+                    tracing::info!(
+                        written_files = written,
+                        remaining_files =
+                            mirror_file_total.saturating_sub(written),
+                        total_files = mirror_file_total,
+                        "Wrote local mirror files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+    } else {
+        let uploaded_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = upload_file_total,
+            "Uploading metadata files"
+        );
+
+        futures::future::try_join_all(upload_files.iter().map(|entry| {
+            let path = entry.key().clone();
+            let file = entry.value().file.clone();
+            let content_type = entry.value().content_type.clone();
+            let uploaded_files = uploaded_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                upload_file_to_bucket(path, file, content_type, &semaphore)
+                    .await?;
+                let uploaded =
+                    uploaded_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if uploaded.is_multiple_of(100) || uploaded == upload_file_total
+                {
+                    tracing::info!(
+                        uploaded_files = uploaded,
+                        remaining_files =
+                            upload_file_total.saturating_sub(uploaded),
+                        total_files = upload_file_total,
+                        "Uploaded metadata files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+
+        let uploaded_mirror_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = mirror_file_total,
+            "Uploading mirror files"
+        );
+
+        futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
+            let path = format!("maven/{}", entry.key());
+            let mirrors = entry
+                .value()
+                .mirrors
+                .iter()
+                .map(|mirror| {
+                    if mirror.entire_url {
+                        mirror.path.clone()
+                    } else {
+                        format!("{}{}", mirror.path, entry.key())
+                    }
+                })
+                .collect();
+            let sha1 = entry.value().sha1.clone();
+            let uploaded_mirror_files = uploaded_mirror_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                upload_url_to_bucket_mirrors(path, mirrors, sha1, &semaphore)
+                    .await?;
+                let uploaded =
+                    uploaded_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if uploaded.is_multiple_of(100) || uploaded == mirror_file_total
+                {
+                    tracing::info!(
+                        uploaded_files = uploaded,
+                        remaining_files =
+                            mirror_file_total.saturating_sub(uploaded),
+                        total_files = mirror_file_total,
+                        "Uploaded mirror files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+    }
 
     if dotenvy::var("CLOUDFLARE_INTEGRATION")
         .ok()
@@ -151,6 +302,20 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
+fn should_fetch(only_loader: Option<&str>, loader: &str) -> bool {
+    let Some(only_loader) = only_loader else {
+        return true;
+    };
+
+    only_loader.split(',').any(|entry| {
+        let entry = entry.trim();
+
+        entry.eq_ignore_ascii_case("all")
+            || entry.eq_ignore_ascii_case(loader)
+            || (loader == "neo" && entry.eq_ignore_ascii_case("neoforge"))
+    })
+}
+
 pub struct UploadFile {
     file: bytes::Bytes,
     content_type: Option<String>,
@@ -248,11 +413,13 @@ fn check_env_vars() -> bool {
 
     failed |= check_var::<String>("BASE_URL");
 
-    failed |= check_var::<String>("S3_ACCESS_TOKEN");
-    failed |= check_var::<String>("S3_SECRET");
-    failed |= check_var::<String>("S3_URL");
-    failed |= check_var::<String>("S3_REGION");
-    failed |= check_var::<String>("S3_BUCKET_NAME");
+    if dotenvy::var("LOCAL_OUTPUT_DIR").is_err() {
+        failed |= check_var::<String>("S3_ACCESS_TOKEN");
+        failed |= check_var::<String>("S3_SECRET");
+        failed |= check_var::<String>("S3_URL");
+        failed |= check_var::<String>("S3_REGION");
+        failed |= check_var::<String>("S3_BUCKET_NAME");
+    }
 
     if dotenvy::var("CLOUDFLARE_INTEGRATION")
         .ok()

apps/daedalus_client/src/metadata_groups.rs

@@ -0,0 +1,145 @@
+//! Determines how version info is generated for pairs of game and loader
+//! versions.
+//!
+//! When a user installs a version of the game, they install two things: the
+//! game (of some specific version), and a loader (of some other specific
+//! version). Each combination of game and loader version requires a specific
+//! configuration, like a specific set of libraries that must be downloaded and
+//! run along with the game. However, some versions of the game or loader may
+//! change configuration requirements without the other version being affected.
+//! For example, pre-26.x game versions with Quilt require the Quilt `hashed`
+//! libraries to also be downloaded. However, 26.x and later don't require the
+//! `hashed` libraries, and don't even have a download for them. The problem is
+//! that Quilt loader 0.30.0 can be used for both pre-26.x and 26.x - but our v0
+//! manifest files can't differentiate the two. The result is that you either
+//! break compatibility for 0.30.0 game versions pre-26.x, or break 0.30.0
+//! on 26.x and later.
+//!
+//! To fix this, v1 introduces the concept of *version groups*: game versions
+//! before 26.x are version group v1, and 26.x and later are v2. Then, we
+//! parameterize our version info on both version group and loader version,
+//! letting us specify the right configuration based on both game version and
+//! loader version.
+//!
+//! Why not parameterize on game version and loader version directly? Most game
+//! versions have the same configuration as their surrounding game versions, so
+//! we'd end up with many duplicate configurations: the number of game versions
+//! multiplied by the number of loader versions.
+//!
+//! This file lets you configure what game versions are grouped together.
+//!
+//! Each version group is templated from a specific game version - e.g. game
+//! version 1.21 is used as the template file for 1.20, 1.19, etc.
+
+pub const UNIVERSAL_METADATA_GROUP: &str = "universal";
+pub const QUILT_LEGACY_METADATA_GROUP: &str = "v1";
+pub const QUILT_MODERN_METADATA_GROUP: &str = "v2";
+
+pub struct MetadataGroup {
+    pub id: &'static str,
+    /// Minecraft version used to fetch and template this group's loader profiles.
+    pub loader_profile_template_game_version: String,
+    pub game_versions: Vec<String>,
+}
+
+pub fn metadata_groups<'a>(
+    mod_loader: &str,
+    game_versions: impl IntoIterator<Item = &'a str>,
+) -> Vec<MetadataGroup> {
+    // Non-Quilt loaders don't need the concept of version groups, so we just
+    // make one "universal" group, and template it on 1.21.
+    if mod_loader != "quilt" {
+        return vec![MetadataGroup {
+            id: UNIVERSAL_METADATA_GROUP,
+            loader_profile_template_game_version: "1.21".to_string(),
+            game_versions: game_versions
+                .into_iter()
+                .map(|x| x.to_string())
+                .collect(),
+        }];
+    }
+
+    let game_versions = game_versions.into_iter().collect::<Vec<_>>();
+    let legacy_game_versions = game_versions
+        .iter()
+        .copied()
+        .filter(|game_version| {
+            metadata_group_id_for_game_version(mod_loader, game_version)
+                == QUILT_LEGACY_METADATA_GROUP
+        })
+        .collect::<Vec<_>>();
+    let modern_game_versions = game_versions
+        .iter()
+        .copied()
+        .filter(|game_version| {
+            metadata_group_id_for_game_version(mod_loader, game_version)
+                == QUILT_MODERN_METADATA_GROUP
+        })
+        .collect::<Vec<_>>();
+
+    let mut groups = Vec::new();
+
+    if !legacy_game_versions.is_empty() {
+        groups.push(MetadataGroup {
+            id: QUILT_LEGACY_METADATA_GROUP,
+            loader_profile_template_game_version: legacy_game_versions
+                .iter()
+                .find(|x| **x == "1.21")
+                .copied()
+                .unwrap_or(legacy_game_versions[0])
+                .to_string(),
+            game_versions: legacy_game_versions
+                .iter()
+                .map(|x| x.to_string())
+                .collect(),
+        });
+    }
+
+    if !modern_game_versions.is_empty() {
+        groups.push(MetadataGroup {
+            id: QUILT_MODERN_METADATA_GROUP,
+            loader_profile_template_game_version: modern_game_versions[0]
+                .to_string(),
+            game_versions: modern_game_versions
+                .iter()
+                .map(|x| x.to_string())
+                .collect(),
+        });
+    }
+
+    groups
+}
+
+pub fn metadata_group_for_game_version<'a>(
+    groups: &'a [MetadataGroup],
+    mod_loader: &str,
+    game_version: &str,
+) -> Option<&'a MetadataGroup> {
+    let group_id = metadata_group_id_for_game_version(mod_loader, game_version);
+
+    groups.iter().find(|group| group.id == group_id)
+}
+
+fn metadata_group_id_for_game_version(
+    mod_loader: &str,
+    game_version: &str,
+) -> &'static str {
+    if mod_loader == "quilt" && is_modern_quilt_game_version(game_version) {
+        QUILT_MODERN_METADATA_GROUP
+    } else if mod_loader == "quilt" {
+        QUILT_LEGACY_METADATA_GROUP
+    } else {
+        UNIVERSAL_METADATA_GROUP
+    }
+}
+
+// Update these Quilt group boundaries if upstream loader profiles gain another
+// structural incompatibility between Minecraft versions.
+fn is_modern_quilt_game_version(game_version: &str) -> bool {
+    let major = game_version
+        .split(['.', 'w'])
+        .next()
+        .and_then(|x| x.parse::<usize>().ok());
+
+    major.is_some_and(|x| x >= 26)
+}

apps/daedalus_client/src/util.rs

@@ -3,7 +3,11 @@ use bytes::Bytes;
 use s3::creds::Credentials;
 use s3::{Bucket, Region};
 use serde::de::DeserializeOwned;
-use std::sync::{Arc, LazyLock};
+use std::path::PathBuf;
+use std::sync::{
+    Arc, LazyLock,
+    atomic::{AtomicUsize, Ordering},
+};
 use tokio::sync::Semaphore;
 
 static BUCKET: LazyLock<Bucket> = LazyLock::new(|| {
@@ -55,6 +59,8 @@ pub static REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
         .unwrap()
 });
 
+static DOWNLOADED_FILE_COUNT: AtomicUsize = AtomicUsize::new(0);
+
 #[tracing::instrument(skip(bytes, semaphore))]
 pub async fn upload_file_to_bucket(
     path: String,
@@ -135,6 +141,74 @@ pub async fn upload_url_to_bucket(
     Ok(())
 }
 
+pub async fn write_file_to_local_output(
+    path: &str,
+    bytes: Bytes,
+) -> Result<(), Error> {
+    let output_path = local_output_path(path)?;
+
+    if let Some(parent) = output_path.parent() {
+        tokio::fs::create_dir_all(parent).await?;
+    }
+
+    tokio::fs::write(output_path, bytes).await?;
+
+    Ok(())
+}
+
+pub async fn write_url_to_local_output_mirrors(
+    output_path: String,
+    mirrors: Vec<String>,
+    sha1: Option<String>,
+    semaphore: &Arc<Semaphore>,
+) -> Result<(), Error> {
+    if mirrors.is_empty() {
+        return Err(ErrorKind::InvalidInput(
+            "No mirrors provided!".to_string(),
+        )
+        .into());
+    }
+
+    for (index, mirror) in mirrors.iter().enumerate() {
+        let result = write_url_to_local_output(
+            output_path.clone(),
+            mirror.clone(),
+            sha1.clone(),
+            semaphore,
+        )
+        .await;
+
+        if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
+            return result;
+        }
+    }
+
+    unreachable!()
+}
+
+async fn write_url_to_local_output(
+    path: String,
+    url: String,
+    sha1: Option<String>,
+    semaphore: &Arc<Semaphore>,
+) -> Result<(), Error> {
+    let data = download_file(&url, sha1.as_deref(), semaphore).await?;
+
+    write_file_to_local_output(&path, data).await?;
+
+    Ok(())
+}
+
+fn local_output_path(path: &str) -> Result<PathBuf, Error> {
+    let output_dir = dotenvy::var("LOCAL_OUTPUT_DIR").map_err(|_| {
+        ErrorKind::InvalidInput(
+            "LOCAL_OUTPUT_DIR is required for local output".to_string(),
+        )
+    })?;
+
+    Ok(PathBuf::from(output_dir).join(path))
+}
+
 #[tracing::instrument(skip(bytes))]
 pub async fn sha1_async(bytes: Bytes) -> Result<String, Error> {
     let hash = tokio::task::spawn_blocking(move || {
@@ -182,6 +256,16 @@ pub async fn download_file(
                         }
                     }
 
+                    let downloaded = DOWNLOADED_FILE_COUNT
+                        .fetch_add(1, Ordering::Relaxed)
+                        + 1;
+                    if downloaded.is_multiple_of(100) {
+                        tracing::info!(
+                            downloaded_files = downloaded,
+                            "Downloaded metadata files"
+                        );
+                    }
+
                     return Ok(bytes);
                 } else if attempt <= RETRIES {
                     continue;

packages/daedalus/src/modded.rs

@@ -10,10 +10,73 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
 /// The latest version of the format the fabric model structs deserialize to
 pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
 /// The latest version of the format the quilt model structs deserialize to
-pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0;
+pub const CURRENT_QUILT_FORMAT_VERSION: usize = 1;
 /// The latest version of the format the neoforge model structs deserialize to
 pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0;
 
+/// Metadata for locating and caching a loader manifest.
+#[derive(Debug, Clone)]
+pub struct LoaderManifestMetadata {
+    /// The canonical loader name used in launcher-meta paths.
+    pub loader: String,
+    /// The latest manifest format version for this loader.
+    pub format_version: usize,
+    /// The cache key that includes the loader format version.
+    pub cache_key: String,
+    /// The launcher-meta path to the manifest.
+    pub path: String,
+}
+
+/// Returns metadata for the latest manifest format for the provided loader.
+pub fn loader_manifest_metadata(loader: &str) -> LoaderManifestMetadata {
+    let format_version = current_loader_manifest_format_version(loader);
+    let cache_key = format!("{loader}-v{format_version}");
+    let path = format!("{loader}/v{format_version}/manifest.json");
+
+    LoaderManifestMetadata {
+        loader: loader.to_string(),
+        format_version,
+        cache_key,
+        path,
+    }
+}
+
+/// Returns loader manifest metadata from a versioned cache key.
+pub fn loader_manifest_metadata_from_cache_key(
+    cache_key: &str,
+) -> LoaderManifestMetadata {
+    if let Some((loader, format_version)) =
+        cache_key.rsplit_once("-v").and_then(|(loader, version)| {
+            version
+                .parse::<usize>()
+                .ok()
+                .map(|version| (loader, version))
+        })
+    {
+        let cache_key = format!("{loader}-v{format_version}");
+        let path = format!("{loader}/v{format_version}/manifest.json");
+
+        LoaderManifestMetadata {
+            loader: loader.to_string(),
+            format_version,
+            cache_key,
+            path,
+        }
+    } else {
+        loader_manifest_metadata(cache_key)
+    }
+}
+
+fn current_loader_manifest_format_version(loader: &str) -> usize {
+    match loader {
+        "fabric" => CURRENT_FABRIC_FORMAT_VERSION,
+        "forge" => CURRENT_FORGE_FORMAT_VERSION,
+        "quilt" => CURRENT_QUILT_FORMAT_VERSION,
+        "neo" => CURRENT_NEOFORGE_FORMAT_VERSION,
+        _ => 0,
+    }
+}
+
 /// The dummy replace string library names, inheritsFrom, and version names should be replaced with
 pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";
 
@@ -188,19 +251,36 @@ pub fn merge_partial_version(
 pub struct Manifest {
     /// The game versions the mod loader supports
     pub game_versions: Vec<Version>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    /// Groups of game versions that share compatible loader version profiles
+    pub version_groups: Vec<VersionGroup>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
 ///  A game version of Minecraft
 pub struct Version {
     /// The minecraft version ID
     pub id: String,
     /// Whether the release is stable or not
     pub stable: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    /// The loader profile group for this Minecraft version
+    pub version_group: Option<String>,
     /// A map that contains loader versions for the game version
     pub loaders: Vec<LoaderVersion>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+/// A group of Minecraft versions that share loader version profiles
+pub struct VersionGroup {
+    /// The version group ID
+    pub id: String,
+    /// The loader versions for this version group
+    pub loaders: Vec<LoaderVersion>,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 /// A version of a Minecraft mod loader
 pub struct LoaderVersion {