fix: show "inspecting modpack" (#6979)

* fix: show "inspecting modpack"

Shows when determining external files for modpacks which have many overrides/over gb of content files.

* fix: better progress indicator

apps/app-frontend/src/components/ui/AppActionBar.vue

@@ -345,7 +345,7 @@ function buildDownloadItems(): PopupNotificationProgressItem[] {
 			iconUrl: currentLoadingBarIconUrls.value[getLoadingBarKey(bar)] ?? null,
 			progress: getLoadingProgress(bar),
 			waiting: !bar.total || bar.total <= 0,
-			progressType: 'percentage',
+			progressType: bar.bar_type?.type === 'pack_import' ? 'bytes' : 'percentage',
 			progressCurrent: bar.current,
 			progressTotal: bar.total,
 		})),

packages/app-lib/src/api/pack/install_from.rs

@@ -1,5 +1,7 @@
 use crate::State;
 use crate::data::ModLoader;
+use crate::event::LoadingBarType;
+use crate::event::emit::{emit_loading, init_loading};
 use crate::install::{
     InstallErrorContext, InstallPhaseDetails, InstallPhaseId, InstallProgress,
     InstallProgressReporter,
@@ -10,7 +12,7 @@ use crate::state::{
 };
 use crate::util::fetch::{
     DownloadMeta, DownloadReason, FetchProgressFn, fetch,
-    fetch_advanced_with_progress, sha1_file_async,
+    fetch_advanced_with_progress, sha1_file_async_with_progress,
 };
 use path_util::SafeRelativeUtf8UnixPathBuf;
 use reqwest::Method;
@@ -18,7 +20,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::future::Future;
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::pin::Pin;
 
 #[derive(Serialize, Deserialize, Eq, PartialEq)]
@@ -153,6 +155,17 @@ pub struct CreatePack {
 
 const MAX_LOCAL_FILE_HASH_LOOKUP_SIZE: u64 = 1024 * 1024 * 1024;
 
+pub(crate) fn get_local_pack_instance(path: &Path) -> CreatePackInstance {
+    CreatePackInstance {
+        name: path
+            .file_stem()
+            .unwrap_or_default()
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct CreatePackDescription {
     pub icon: Option<PathBuf>,
@@ -182,17 +195,60 @@ pub async fn get_instance_from_pack(
             ..Default::default()
         }),
         CreatePackLocation::FromFile { path } => {
-            let file_name = path
+            let mut instance = get_local_pack_instance(&path);
-                .file_stem()
+            let file_size = tokio::fs::metadata(&path).await?.len();
-                .unwrap_or_default()
+            let hashes_archive = file_size <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE;
-                .to_string_lossy()
+            let archive_hashing_bytes =
-                .to_string();
+                if hashes_archive { file_size } else { 0 };
+            let pack_file = CreatePackFile::Path(path.clone());
+            let external_file_hashing_bytes =
+                super::install_mrpack::get_external_file_hashing_size_from_mrpack(
+                    &pack_file,
+                )
+                .await?;
+            let inspection_total_bytes = archive_hashing_bytes
+                .saturating_add(external_file_hashing_bytes)
+                .max(1);
+            let inspection = init_loading(
+                LoadingBarType::PackImport {
+                    pack_name: instance.name.clone(),
+                },
+                inspection_total_bytes as f64,
+                "Inspecting modpack",
+            )
+            .await
+            .ok();
+            let min_delta = (inspection_total_bytes / 200).max(256 * 1024);
+            let mut reported_bytes = 0_u64;
+            let mut report_progress =
+                |current: u64, offset: u64, message: &str| {
+                    let target = offset
+                        .saturating_add(current)
+                        .min(inspection_total_bytes);
+                    let increment = target.saturating_sub(reported_bytes);
+                    if target < inspection_total_bytes && increment < min_delta
+                    {
+                        return;
+                    }
 
-            let is_known_file = if tokio::fs::metadata(&path).await?.len()
+                    if let Some(inspection) = &inspection {
-                <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE
+                        let _ = emit_loading(
-            {
+                            inspection,
+                            increment as f64,
+                            Some(message),
+                        );
+                    }
+                    reported_bytes = target;
+                };
+
+            let is_known_file = if hashes_archive {
                 let state = State::get().await?;
-                let (_, hash) = sha1_file_async(&path).await?;
+                let (_, hash) =
+                    sha1_file_async_with_progress(&path, |current, _| {
+                        report_progress(current, 0, "Hashing local modpack");
+                        Ok(())
+                    })
+                    .await?;
                 match CachedEntry::get_file_many(
                     &[&hash],
                     Some(CacheBehaviour::StaleWhileRevalidateSkipOffline),
@@ -217,16 +273,26 @@ pub async fn get_instance_from_pack(
 
             let external_files_in_modpack =
                 super::install_mrpack::get_external_files_from_mrpack(
-                    &CreatePackFile::Path(path),
+                    &pack_file,
+                    |current, _| {
+                        report_progress(
+                            current,
+                            archive_hashing_bytes,
+                            "Inspecting modpack files",
+                        );
+                        Ok(())
+                    },
                 )
                 .await?;
+            report_progress(
+                inspection_total_bytes,
+                0,
+                "Finished inspecting modpack",
+            );
 
-            Ok(CreatePackInstance {
+            instance.unknown_file = !is_known_file;
-                name: file_name,
+            instance.external_files_in_modpack = external_files_in_modpack;
-                unknown_file: !is_known_file,
+            Ok(instance)
-                external_files_in_modpack,
-                ..Default::default()
-            })
         }
     }
 }

packages/app-lib/src/api/pack/install_mrpack.rs

@@ -41,6 +41,7 @@ use tokio::sync::Mutex;
 type ExtractProgressFn<'a> = dyn FnMut(u64) -> Pin<Box<dyn Future<Output = crate::Result<()>> + Send + 'a>>
     + Send
     + 'a;
+type HashProgressFn<'a> = dyn FnMut(u64) -> crate::Result<()> + Send + 'a;
 const MODPACK_CONTENT_DOWNLOAD_CONCURRENCY: usize = 4;
 
 #[derive(Clone)]
@@ -166,13 +167,16 @@ impl MrpackZipReader {
     async fn hash_entry(
         &mut self,
         index: usize,
+        progress: Option<&mut HashProgressFn<'_>>,
     ) -> crate::Result<(u64, String)> {
         match self {
             Self::Memory(reader) => {
-                hash_zip_entry(reader.reader_with_entry(index).await?).await
+                hash_zip_entry(reader.reader_with_entry(index).await?, progress)
+                    .await
             }
             Self::File(reader) => {
-                hash_zip_entry(reader.reader_with_entry(index).await?).await
+                hash_zip_entry(reader.reader_with_entry(index).await?, progress)
+                    .await
             }
         }
     }
@@ -209,6 +213,7 @@ impl MrpackZipReader {
 
 async fn hash_zip_entry<R>(
     mut reader: ZipEntryReader<'_, R, WithEntry<'_>>,
+    mut progress: Option<&mut HashProgressFn<'_>>,
 ) -> crate::Result<(u64, String)>
 where
     R: futures_lite::io::AsyncBufRead + Unpin,
@@ -228,6 +233,9 @@ where
 
         hasher.update(&buffer[..bytes_read]);
         size += bytes_read as u64;
+        if let Some(progress) = progress.as_mut() {
+            progress(bytes_read as u64)?;
+        }
     }
 
     if reader.compute_hash() != expected_crc32 {
@@ -239,6 +247,7 @@ where
 
 pub(crate) async fn get_external_files_from_mrpack(
     file: &CreatePackFile,
+    mut progress: impl FnMut(u64, u64) -> crate::Result<()> + Send,
 ) -> crate::Result<Vec<String>> {
     let mut zip_reader = MrpackZipReader::new(file).await?;
     let Some(manifest_idx) =
@@ -271,21 +280,35 @@ pub(crate) async fn get_external_files_from_mrpack(
         .enumerate()
         .filter_map(|(index, entry)| {
             let path = entry.filename().as_str().ok()?;
-            let relative_path = path
+            let relative_path = external_override_relative_path(path)?;
-                .strip_prefix("overrides/")
-                .or_else(|| path.strip_prefix("client-overrides/"))?;
-            if path.ends_with('/')
-                || ProjectType::get_from_parent_folder(relative_path).is_none()
-            {
-                return None;
-            }
             let file_name = relative_path.rsplit('/').next()?.to_string();
-            Some((index, file_name))
+            Some((index, file_name, entry.uncompressed_size()))
         })
         .collect::<Vec<_>>();
 
-    for (index, file_name) in override_entries {
+    let total_bytes = override_entries
-        let (_, hash) = zip_reader.hash_entry(index).await?;
+        .iter()
+        .map(|(_, _, size)| size)
+        .sum::<u64>();
+    let min_delta = (total_bytes / 200).max(256 * 1024);
+    let mut current_bytes = 0_u64;
+    let mut last_reported_bytes = 0_u64;
+    let mut report_progress = |bytes_read: u64| {
+        current_bytes = current_bytes.saturating_add(bytes_read);
+        if current_bytes >= total_bytes
+            || current_bytes.saturating_sub(last_reported_bytes) < min_delta
+        {
+            return Ok(());
+        }
+
+        last_reported_bytes = current_bytes;
+        progress(current_bytes.min(total_bytes), total_bytes)
+    };
+
+    for (index, file_name, _) in override_entries {
+        let (_, hash) = zip_reader
+            .hash_entry(index, Some(&mut report_progress))
+            .await?;
         candidates.push((file_name, hash));
     }
 
@@ -327,6 +350,31 @@ pub(crate) async fn get_external_files_from_mrpack(
     Ok(external_files)
 }
 
+pub(crate) async fn get_external_file_hashing_size_from_mrpack(
+    file: &CreatePackFile,
+) -> crate::Result<u64> {
+    let zip_reader = MrpackZipReader::new(file).await?;
+    Ok(zip_reader
+        .file()
+        .entries()
+        .iter()
+        .filter_map(|entry| {
+            let path = entry.filename().as_str().ok()?;
+            external_override_relative_path(path)
+                .map(|_| entry.uncompressed_size())
+        })
+        .sum())
+}
+
+fn external_override_relative_path(path: &str) -> Option<&str> {
+    let relative_path = path
+        .strip_prefix("overrides/")
+        .or_else(|| path.strip_prefix("client-overrides/"))?;
+    (!path.ends_with('/')
+        && ProjectType::get_from_parent_folder(relative_path).is_some())
+    .then_some(relative_path)
+}
+
 async fn extract_zip_entry<R>(
     mut reader: ZipEntryReader<'_, R, WithEntry<'_>>,
     path: &Path,
@@ -527,7 +575,7 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter(
             .collect();
 
         for index in override_entries {
-            let (_, hash) = zip_reader.hash_entry(index).await?;
+            let (_, hash) = zip_reader.hash_entry(index, None).await?;
             file_hashes.push(hash);
         }
 

packages/app-lib/src/event/emit.rs

@@ -71,7 +71,7 @@ pub async fn init_loading_unsafe(
             message: title.to_string(),
             total,
             current: 0.0,
-            last_sent: 0.0,
+            last_sent: -1.0,
             bar_type,
             #[cfg(feature = "cli")]
             cli_progress_bar: {

packages/app-lib/src/event/mod.rs

@@ -157,6 +157,9 @@ pub enum LoadingBarType {
         pack_id: Option<String>,
         pack_version: Option<String>,
     },
+    PackImport {
+        pack_name: String,
+    },
     MinecraftDownload {
         instance_id: String,
         instance_name: String,

packages/app-lib/src/install/runner.rs

@@ -16,6 +16,7 @@ use crate::ErrorKind;
 use crate::api::pack::install_from::{
     CreatePackLocation, generate_pack_from_file,
     generate_pack_from_version_id_with_reporter, get_instance_from_pack,
+    get_local_pack_instance,
 };
 use crate::api::pack::install_mrpack::install_zipped_mrpack_files_with_reporter;
 use crate::event::InstancePayloadType;
@@ -163,16 +164,6 @@ pub async fn retry_job(job_id: Uuid) -> crate::Result<InstallJobSnapshot> {
     job.state.progress.phase = InstallPhaseId::PreparingInstance;
     job.state.progress.progress = None;
     job.state.progress.details = InstallPhaseDetails::Empty;
-    if let Err(error) = prepare_initial_instance(&mut job.state, &state).await {
-        if let Err(cleanup_error) =
-            recovery::apply_cleanup(&job.state, &state).await
-        {
-            tracing::error!(
-                "Error cleaning up install job {job_id} retry preparation: {cleanup_error}"
-            );
-        }
-        return Err(error);
-    }
     job.state.record_event(InstallJobEventKind::JobQueued {
         kind: job.state.request.kind(),
     });
@@ -197,6 +188,42 @@ pub async fn retry_job(job_id: Uuid) -> crate::Result<InstallJobSnapshot> {
             return Err(error);
         }
     };
+    emit_install_job(&record.snapshot()).await?;
+
+    if let Err(error) = prepare_initial_instance(&mut job.state, &state).await {
+        let error_view = install_error_view(
+            job.state.progress.phase,
+            &error,
+            job.state.context.clone(),
+        );
+        if let Err(terminal_error) =
+            terminalize_failed_job(job_id, job.state, error_view, &state).await
+        {
+            tracing::error!(
+                "Failed to terminalize retried install job {job_id}: {terminal_error}"
+            );
+        }
+        return Err(error);
+    }
+    let record = match store::update_state(job_id, &job.state, &state).await {
+        Ok(record) => record,
+        Err(error) => {
+            let error_view = install_error_view(
+                job.state.progress.phase,
+                &error,
+                job.state.context.clone(),
+            );
+            if let Err(terminal_error) =
+                terminalize_failed_job(job_id, job.state, error_view, &state)
+                    .await
+            {
+                tracing::error!(
+                    "Failed to terminalize retried install job {job_id}: {terminal_error}"
+                );
+            }
+            return Err(error);
+        }
+    };
     if let Err(error) =
         lock_existing_instance_if_needed(&job.state, &state).await
     {
@@ -337,31 +364,39 @@ async fn start(request: InstallRequest) -> crate::Result<InstallJobSnapshot> {
     let state = State::get().await?;
     let id = Uuid::new_v4();
     let mut job_state = InstallJobState::new(request);
+    set_initial_display(&mut job_state);
+    let record =
+        store::insert(id, &job_state, InstallJobStatus::Queued, &state).await?;
+    emit_install_job(&record.snapshot()).await?;
+
     if let Err(error) = prepare_initial_instance(&mut job_state, &state).await {
-        if let Err(cleanup_error) =
+        let error_view = install_error_view(
-            recovery::apply_cleanup(&job_state, &state).await
+            job_state.progress.phase,
+            &error,
+            job_state.context.clone(),
+        );
+        if let Err(terminal_error) =
+            terminalize_failed_job(id, job_state, error_view, &state).await
         {
             tracing::error!(
-                "Error cleaning up install job preparation: {cleanup_error}"
+                "Failed to terminalize install job {id} after setup error: {terminal_error}"
             );
         }
         return Err(error);
     }
-    let record = match store::insert(
+    let record = match store::update_state(id, &job_state, &state).await {
-        id,
-        &job_state,
-        InstallJobStatus::Queued,
-        &state,
-    )
-    .await
-    {
         Ok(record) => record,
         Err(error) => {
-            if let Err(cleanup_error) =
+            let error_view = install_error_view(
-                recovery::apply_cleanup(&job_state, &state).await
+                job_state.progress.phase,
+                &error,
+                job_state.context.clone(),
+            );
+            if let Err(terminal_error) =
+                terminalize_failed_job(id, job_state, error_view, &state).await
             {
                 tracing::error!(
-                    "Error cleaning up untracked install job {id}: {cleanup_error}"
+                    "Failed to terminalize install job {id} after setup error: {terminal_error}"
                 );
             }
             return Err(error);
@@ -422,7 +457,12 @@ async fn prepare_initial_instance(
             location,
             post_install_edit,
         } => {
-            let preview = get_instance_from_pack(location).await?;
+            let preview = match location {
+                CreatePackLocation::FromFile { path } => {
+                    get_local_pack_instance(&path)
+                }
+                location => get_instance_from_pack(location).await?,
+            };
             let name = post_install_edit
                 .as_ref()
                 .and_then(|edit| edit.name.clone())
@@ -1443,6 +1483,26 @@ fn set_display(
     job_state.display = Some(InstallJobDisplay { title, icon });
 }
 
+fn set_initial_display(job_state: &mut InstallJobState) {
+    let display = match &job_state.request {
+        InstallRequest::CreateModpackInstance { location, .. } => {
+            match location {
+                CreatePackLocation::FromVersionId {
+                    title, icon_url, ..
+                } => Some((title.clone(), icon_url.clone())),
+                CreatePackLocation::FromFile { path } => {
+                    Some((get_local_pack_instance(path).name, None))
+                }
+            }
+        }
+        _ => None,
+    };
+
+    if let Some((title, icon)) = display {
+        set_display(job_state, title, icon);
+    }
+}
+
 fn install_error_view(
     phase: InstallPhaseId,
     error: &crate::Error,

packages/app-lib/src/util/fetch.rs

@@ -954,12 +954,24 @@ pub async fn sha1_async(bytes: Bytes) -> crate::Result<String> {
 
 pub async fn sha1_file_async(
     path: impl AsRef<Path>,
+) -> crate::Result<(u64, String)> {
+    sha1_file_async_with_progress(path, |_, _| Ok(())).await
+}
+
+pub async fn sha1_file_async_with_progress(
+    path: impl AsRef<Path>,
+    mut progress: impl FnMut(u64, u64) -> crate::Result<()>,
 ) -> crate::Result<(u64, String)> {
     let path = path.as_ref();
     // Local files can be multi-gigabyte .mrpacks, so hash them without materializing bytes.
     let mut file = File::open(path)
         .await
         .map_err(|e| IOError::with_path(e, path))?;
+    let total = file
+        .metadata()
+        .await
+        .map_err(|e| IOError::with_path(e, path))?
+        .len();
     let mut hasher = sha1_smol::Sha1::new();
     let mut size = 0;
     let mut buffer = vec![0; 262144];
@@ -975,6 +987,7 @@ pub async fn sha1_file_async(
 
         hasher.update(&buffer[..bytes_read]);
         size += bytes_read as u64;
+        progress(size, total)?;
     }
 
     Ok((size, hasher.digest().to_string()))