fix: show "inspecting modpack" (#6979)

* fix: show "inspecting modpack"

Shows when determining external files for modpacks which have many overrides/over gb of content files.

* fix: better progress indicator

packages/app-lib/src/api/pack/install_from.rs

@@ -1,5 +1,7 @@
 use crate::State;
 use crate::data::ModLoader;
+use crate::event::LoadingBarType;
+use crate::event::emit::{emit_loading, init_loading};
 use crate::install::{
     InstallErrorContext, InstallPhaseDetails, InstallPhaseId, InstallProgress,
     InstallProgressReporter,
@@ -10,7 +12,7 @@ use crate::state::{
 };
 use crate::util::fetch::{
     DownloadMeta, DownloadReason, FetchProgressFn, fetch,
-    fetch_advanced_with_progress, sha1_file_async,
+    fetch_advanced_with_progress, sha1_file_async_with_progress,
 };
 use path_util::SafeRelativeUtf8UnixPathBuf;
 use reqwest::Method;
@@ -18,7 +20,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::future::Future;
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::pin::Pin;
 
 #[derive(Serialize, Deserialize, Eq, PartialEq)]
@@ -153,6 +155,17 @@ pub struct CreatePack {
 
 const MAX_LOCAL_FILE_HASH_LOOKUP_SIZE: u64 = 1024 * 1024 * 1024;
 
+pub(crate) fn get_local_pack_instance(path: &Path) -> CreatePackInstance {
+    CreatePackInstance {
+        name: path
+            .file_stem()
+            .unwrap_or_default()
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct CreatePackDescription {
     pub icon: Option<PathBuf>,
@@ -182,17 +195,60 @@ pub async fn get_instance_from_pack(
             ..Default::default()
         }),
         CreatePackLocation::FromFile { path } => {
-            let file_name = path
-                .file_stem()
-                .unwrap_or_default()
-                .to_string_lossy()
-                .to_string();
+            let mut instance = get_local_pack_instance(&path);
+            let file_size = tokio::fs::metadata(&path).await?.len();
+            let hashes_archive = file_size <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE;
+            let archive_hashing_bytes =
+                if hashes_archive { file_size } else { 0 };
+            let pack_file = CreatePackFile::Path(path.clone());
+            let external_file_hashing_bytes =
+                super::install_mrpack::get_external_file_hashing_size_from_mrpack(
+                    &pack_file,
+                )
+                .await?;
+            let inspection_total_bytes = archive_hashing_bytes
+                .saturating_add(external_file_hashing_bytes)
+                .max(1);
+            let inspection = init_loading(
+                LoadingBarType::PackImport {
+                    pack_name: instance.name.clone(),
+                },
+                inspection_total_bytes as f64,
+                "Inspecting modpack",
+            )
+            .await
+            .ok();
+            let min_delta = (inspection_total_bytes / 200).max(256 * 1024);
+            let mut reported_bytes = 0_u64;
+            let mut report_progress =
+                |current: u64, offset: u64, message: &str| {
+                    let target = offset
+                        .saturating_add(current)
+                        .min(inspection_total_bytes);
+                    let increment = target.saturating_sub(reported_bytes);
+                    if target < inspection_total_bytes && increment < min_delta
+                    {
+                        return;
+                    }
 
-            let is_known_file = if tokio::fs::metadata(&path).await?.len()
-                <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE
-            {
+                    if let Some(inspection) = &inspection {
+                        let _ = emit_loading(
+                            inspection,
+                            increment as f64,
+                            Some(message),
+                        );
+                    }
+                    reported_bytes = target;
+                };
+
+            let is_known_file = if hashes_archive {
                 let state = State::get().await?;
-                let (_, hash) = sha1_file_async(&path).await?;
+                let (_, hash) =
+                    sha1_file_async_with_progress(&path, |current, _| {
+                        report_progress(current, 0, "Hashing local modpack");
+                        Ok(())
+                    })
+                    .await?;
                 match CachedEntry::get_file_many(
                     &[&hash],
                     Some(CacheBehaviour::StaleWhileRevalidateSkipOffline),
@@ -217,16 +273,26 @@ pub async fn get_instance_from_pack(
 
             let external_files_in_modpack =
                 super::install_mrpack::get_external_files_from_mrpack(
-                    &CreatePackFile::Path(path),
+                    &pack_file,
+                    |current, _| {
+                        report_progress(
+                            current,
+                            archive_hashing_bytes,
+                            "Inspecting modpack files",
+                        );
+                        Ok(())
+                    },
                 )
                 .await?;
+            report_progress(
+                inspection_total_bytes,
+                0,
+                "Finished inspecting modpack",
+            );
 
-            Ok(CreatePackInstance {
-                name: file_name,
-                unknown_file: !is_known_file,
-                external_files_in_modpack,
-                ..Default::default()
-            })
+            instance.unknown_file = !is_known_file;
+            instance.external_files_in_modpack = external_files_in_modpack;
+            Ok(instance)
         }
     }
 }

packages/app-lib/src/api/pack/install_mrpack.rs

@@ -41,6 +41,7 @@ use tokio::sync::Mutex;
 type ExtractProgressFn<'a> = dyn FnMut(u64) -> Pin<Box<dyn Future<Output = crate::Result<()>> + Send + 'a>>
     + Send
     + 'a;
+type HashProgressFn<'a> = dyn FnMut(u64) -> crate::Result<()> + Send + 'a;
 const MODPACK_CONTENT_DOWNLOAD_CONCURRENCY: usize = 4;
 
 #[derive(Clone)]
@@ -166,13 +167,16 @@ impl MrpackZipReader {
     async fn hash_entry(
         &mut self,
         index: usize,
+        progress: Option<&mut HashProgressFn<'_>>,
     ) -> crate::Result<(u64, String)> {
         match self {
             Self::Memory(reader) => {
-                hash_zip_entry(reader.reader_with_entry(index).await?).await
+                hash_zip_entry(reader.reader_with_entry(index).await?, progress)
+                    .await
             }
             Self::File(reader) => {
-                hash_zip_entry(reader.reader_with_entry(index).await?).await
+                hash_zip_entry(reader.reader_with_entry(index).await?, progress)
+                    .await
             }
         }
     }
@@ -209,6 +213,7 @@ impl MrpackZipReader {
 
 async fn hash_zip_entry<R>(
     mut reader: ZipEntryReader<'_, R, WithEntry<'_>>,
+    mut progress: Option<&mut HashProgressFn<'_>>,
 ) -> crate::Result<(u64, String)>
 where
     R: futures_lite::io::AsyncBufRead + Unpin,
@@ -228,6 +233,9 @@ where
 
         hasher.update(&buffer[..bytes_read]);
         size += bytes_read as u64;
+        if let Some(progress) = progress.as_mut() {
+            progress(bytes_read as u64)?;
+        }
     }
 
     if reader.compute_hash() != expected_crc32 {
@@ -239,6 +247,7 @@ where
 
 pub(crate) async fn get_external_files_from_mrpack(
     file: &CreatePackFile,
+    mut progress: impl FnMut(u64, u64) -> crate::Result<()> + Send,
 ) -> crate::Result<Vec<String>> {
     let mut zip_reader = MrpackZipReader::new(file).await?;
     let Some(manifest_idx) =
@@ -271,21 +280,35 @@ pub(crate) async fn get_external_files_from_mrpack(
         .enumerate()
         .filter_map(|(index, entry)| {
             let path = entry.filename().as_str().ok()?;
-            let relative_path = path
-                .strip_prefix("overrides/")
-                .or_else(|| path.strip_prefix("client-overrides/"))?;
-            if path.ends_with('/')
-                || ProjectType::get_from_parent_folder(relative_path).is_none()
-            {
-                return None;
-            }
+            let relative_path = external_override_relative_path(path)?;
             let file_name = relative_path.rsplit('/').next()?.to_string();
-            Some((index, file_name))
+            Some((index, file_name, entry.uncompressed_size()))
         })
         .collect::<Vec<_>>();
 
-    for (index, file_name) in override_entries {
-        let (_, hash) = zip_reader.hash_entry(index).await?;
+    let total_bytes = override_entries
+        .iter()
+        .map(|(_, _, size)| size)
+        .sum::<u64>();
+    let min_delta = (total_bytes / 200).max(256 * 1024);
+    let mut current_bytes = 0_u64;
+    let mut last_reported_bytes = 0_u64;
+    let mut report_progress = |bytes_read: u64| {
+        current_bytes = current_bytes.saturating_add(bytes_read);
+        if current_bytes >= total_bytes
+            || current_bytes.saturating_sub(last_reported_bytes) < min_delta
+        {
+            return Ok(());
+        }
+
+        last_reported_bytes = current_bytes;
+        progress(current_bytes.min(total_bytes), total_bytes)
+    };
+
+    for (index, file_name, _) in override_entries {
+        let (_, hash) = zip_reader
+            .hash_entry(index, Some(&mut report_progress))
+            .await?;
         candidates.push((file_name, hash));
     }
 
@@ -327,6 +350,31 @@ pub(crate) async fn get_external_files_from_mrpack(
     Ok(external_files)
 }
 
+pub(crate) async fn get_external_file_hashing_size_from_mrpack(
+    file: &CreatePackFile,
+) -> crate::Result<u64> {
+    let zip_reader = MrpackZipReader::new(file).await?;
+    Ok(zip_reader
+        .file()
+        .entries()
+        .iter()
+        .filter_map(|entry| {
+            let path = entry.filename().as_str().ok()?;
+            external_override_relative_path(path)
+                .map(|_| entry.uncompressed_size())
+        })
+        .sum())
+}
+
+fn external_override_relative_path(path: &str) -> Option<&str> {
+    let relative_path = path
+        .strip_prefix("overrides/")
+        .or_else(|| path.strip_prefix("client-overrides/"))?;
+    (!path.ends_with('/')
+        && ProjectType::get_from_parent_folder(relative_path).is_some())
+    .then_some(relative_path)
+}
+
 async fn extract_zip_entry<R>(
     mut reader: ZipEntryReader<'_, R, WithEntry<'_>>,
     path: &Path,
@@ -527,7 +575,7 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter(
             .collect();
 
         for index in override_entries {
-            let (_, hash) = zip_reader.hash_entry(index).await?;
+            let (_, hash) = zip_reader.hash_entry(index, None).await?;
             file_hashes.push(hash);
         }