Generate different Daedalus Quilt templates for 26.x/non-26.x (#6554)

* Generate different Daedalus Quilt templates for 26.x/non-26.x

* add docs

* wip: daedalus version groups

* wip: app-side logic

* finish app-side logic, fix certain libs not included

* lints

* fmt

* fmt

* Split frontend changes out of Daedalus branch

apps/daedalus_client/src/main.rs

@@ -1,10 +1,14 @@
 use crate::util::{
     REQWEST_CLIENT, format_url, upload_file_to_bucket,
-    upload_url_to_bucket_mirrors,
+    upload_url_to_bucket_mirrors, write_file_to_local_output,
+    write_url_to_local_output_mirrors,
 };
 use daedalus::get_path_from_artifact;
 use dashmap::{DashMap, DashSet};
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 use tokio::sync::Semaphore;
 use tracing_error::ErrorLayer;
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
@@ -12,6 +16,7 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 mod error;
 mod fabric;
 mod forge;
+mod metadata_groups;
 mod minecraft;
 pub mod util;
 
@@ -44,51 +49,92 @@ async fn main() -> Result<()> {
     ));
 
     let mut fetch_result = FetchResult::default();
+    let only_loader = dotenvy::var("DAEDALUS_ONLY").ok();
 
-    match minecraft::fetch(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
+    if should_fetch(only_loader.as_deref(), "minecraft") {
+        match minecraft::fetch(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
+        }
     }
 
-    match fabric::fetch_fabric(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
+    if should_fetch(only_loader.as_deref(), "fabric") {
+        match fabric::fetch_fabric(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
+        }
     }
 
-    match fabric::fetch_quilt(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
+    if should_fetch(only_loader.as_deref(), "quilt") {
+        match fabric::fetch_quilt(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
+        }
     }
 
-    match forge::fetch_neo(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
+    if should_fetch(only_loader.as_deref(), "neo") {
+        match forge::fetch_neo(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
+        }
     }
 
-    match forge::fetch_forge(semaphore.clone()).await {
-        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-        Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
+    if should_fetch(only_loader.as_deref(), "forge") {
+        match forge::fetch_forge(semaphore.clone()).await {
+            Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+            Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
+        }
     }
 
     let FetchResult {
         upload_files,
         mirror_artifacts,
     } = fetch_result;
+    let upload_file_total = upload_files.len();
+    let mirror_file_total = mirror_artifacts.len();
 
-    futures::future::try_join_all(upload_files.iter().map(|entry| {
-        upload_file_to_bucket(
-            entry.key().clone(),
-            entry.value().file.clone(),
-            entry.value().content_type.clone(),
-            &semaphore,
-        )
-    }))
-    .await?;
+    if dotenvy::var("LOCAL_OUTPUT_DIR").is_ok() {
+        let written_files = Arc::new(AtomicUsize::new(0));
 
-    futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
-        upload_url_to_bucket_mirrors(
-            format!("maven/{}", entry.key()),
-            entry
+        tracing::info!(
+            total_files = upload_file_total,
+            "Writing local metadata files"
+        );
+
+        futures::future::try_join_all(upload_files.iter().map(|entry| {
+            let path = entry.key().clone();
+            let file = entry.value().file.clone();
+            let written_files = written_files.clone();
+
+            async move {
+                write_file_to_local_output(&path, file).await?;
+                let written = written_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if written.is_multiple_of(100) || written == upload_file_total {
+                    tracing::info!(
+                        written_files = written,
+                        remaining_files =
+                            upload_file_total.saturating_sub(written),
+                        total_files = upload_file_total,
+                        "Wrote local metadata files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+
+        let written_mirror_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = mirror_file_total,
+            "Writing local mirror files"
+        );
+
+        futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
+            let path = format!("maven/{}", entry.key());
+            let mirrors = entry
                 .value()
                 .mirrors
                 .iter()
@@ -99,12 +145,117 @@ async fn main() -> Result<()> {
                         format!("{}{}", mirror.path, entry.key())
                     }
                 })
-                .collect(),
-            entry.value().sha1.clone(),
-            &semaphore,
-        )
-    }))
-    .await?;
+                .collect();
+            let sha1 = entry.value().sha1.clone();
+            let written_mirror_files = written_mirror_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                write_url_to_local_output_mirrors(
+                    path, mirrors, sha1, &semaphore,
+                )
+                .await?;
+                let written =
+                    written_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if written.is_multiple_of(100) || written == mirror_file_total {
+                    tracing::info!(
+                        written_files = written,
+                        remaining_files =
+                            mirror_file_total.saturating_sub(written),
+                        total_files = mirror_file_total,
+                        "Wrote local mirror files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+    } else {
+        let uploaded_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = upload_file_total,
+            "Uploading metadata files"
+        );
+
+        futures::future::try_join_all(upload_files.iter().map(|entry| {
+            let path = entry.key().clone();
+            let file = entry.value().file.clone();
+            let content_type = entry.value().content_type.clone();
+            let uploaded_files = uploaded_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                upload_file_to_bucket(path, file, content_type, &semaphore)
+                    .await?;
+                let uploaded =
+                    uploaded_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if uploaded.is_multiple_of(100) || uploaded == upload_file_total
+                {
+                    tracing::info!(
+                        uploaded_files = uploaded,
+                        remaining_files =
+                            upload_file_total.saturating_sub(uploaded),
+                        total_files = upload_file_total,
+                        "Uploaded metadata files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+
+        let uploaded_mirror_files = Arc::new(AtomicUsize::new(0));
+
+        tracing::info!(
+            total_files = mirror_file_total,
+            "Uploading mirror files"
+        );
+
+        futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
+            let path = format!("maven/{}", entry.key());
+            let mirrors = entry
+                .value()
+                .mirrors
+                .iter()
+                .map(|mirror| {
+                    if mirror.entire_url {
+                        mirror.path.clone()
+                    } else {
+                        format!("{}{}", mirror.path, entry.key())
+                    }
+                })
+                .collect();
+            let sha1 = entry.value().sha1.clone();
+            let uploaded_mirror_files = uploaded_mirror_files.clone();
+            let semaphore = semaphore.clone();
+
+            async move {
+                upload_url_to_bucket_mirrors(path, mirrors, sha1, &semaphore)
+                    .await?;
+                let uploaded =
+                    uploaded_mirror_files.fetch_add(1, Ordering::Relaxed) + 1;
+
+                if uploaded.is_multiple_of(100) || uploaded == mirror_file_total
+                {
+                    tracing::info!(
+                        uploaded_files = uploaded,
+                        remaining_files =
+                            mirror_file_total.saturating_sub(uploaded),
+                        total_files = mirror_file_total,
+                        "Uploaded mirror files"
+                    );
+                }
+
+                Ok::<_, Error>(())
+            }
+        }))
+        .await?;
+    }
 
     if dotenvy::var("CLOUDFLARE_INTEGRATION")
         .ok()
@@ -151,6 +302,20 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
+fn should_fetch(only_loader: Option<&str>, loader: &str) -> bool {
+    let Some(only_loader) = only_loader else {
+        return true;
+    };
+
+    only_loader.split(',').any(|entry| {
+        let entry = entry.trim();
+
+        entry.eq_ignore_ascii_case("all")
+            || entry.eq_ignore_ascii_case(loader)
+            || (loader == "neo" && entry.eq_ignore_ascii_case("neoforge"))
+    })
+}
+
 pub struct UploadFile {
     file: bytes::Bytes,
     content_type: Option<String>,
@@ -248,11 +413,13 @@ fn check_env_vars() -> bool {
 
     failed |= check_var::<String>("BASE_URL");
 
-    failed |= check_var::<String>("S3_ACCESS_TOKEN");
-    failed |= check_var::<String>("S3_SECRET");
-    failed |= check_var::<String>("S3_URL");
-    failed |= check_var::<String>("S3_REGION");
-    failed |= check_var::<String>("S3_BUCKET_NAME");
+    if dotenvy::var("LOCAL_OUTPUT_DIR").is_err() {
+        failed |= check_var::<String>("S3_ACCESS_TOKEN");
+        failed |= check_var::<String>("S3_SECRET");
+        failed |= check_var::<String>("S3_URL");
+        failed |= check_var::<String>("S3_REGION");
+        failed |= check_var::<String>("S3_BUCKET_NAME");
+    }
 
     if dotenvy::var("CLOUDFLARE_INTEGRATION")
         .ok()