diff --git a/apps/app-frontend/src/components/ui/AppActionBar.vue b/apps/app-frontend/src/components/ui/AppActionBar.vue index 33f0a207d3..380a78a5c5 100644 --- a/apps/app-frontend/src/components/ui/AppActionBar.vue +++ b/apps/app-frontend/src/components/ui/AppActionBar.vue @@ -345,7 +345,7 @@ function buildDownloadItems(): PopupNotificationProgressItem[] { iconUrl: currentLoadingBarIconUrls.value[getLoadingBarKey(bar)] ?? null, progress: getLoadingProgress(bar), waiting: !bar.total || bar.total <= 0, - progressType: 'percentage', + progressType: bar.bar_type?.type === 'pack_import' ? 'bytes' : 'percentage', progressCurrent: bar.current, progressTotal: bar.total, })), diff --git a/packages/app-lib/src/api/pack/install_from.rs b/packages/app-lib/src/api/pack/install_from.rs index c894b94ad2..d2b7af8702 100644 --- a/packages/app-lib/src/api/pack/install_from.rs +++ b/packages/app-lib/src/api/pack/install_from.rs @@ -1,5 +1,7 @@ use crate::State; use crate::data::ModLoader; +use crate::event::LoadingBarType; +use crate::event::emit::{emit_loading, init_loading}; use crate::install::{ InstallErrorContext, InstallPhaseDetails, InstallPhaseId, InstallProgress, InstallProgressReporter, @@ -10,7 +12,7 @@ use crate::state::{ }; use crate::util::fetch::{ DownloadMeta, DownloadReason, FetchProgressFn, fetch, - fetch_advanced_with_progress, sha1_file_async, + fetch_advanced_with_progress, sha1_file_async_with_progress, }; use path_util::SafeRelativeUtf8UnixPathBuf; use reqwest::Method; @@ -18,7 +20,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::future::Future; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::pin::Pin; #[derive(Serialize, Deserialize, Eq, PartialEq)] @@ -153,6 +155,17 @@ pub struct CreatePack { const MAX_LOCAL_FILE_HASH_LOOKUP_SIZE: u64 = 1024 * 1024 * 1024; +pub(crate) fn get_local_pack_instance(path: &Path) -> CreatePackInstance { + CreatePackInstance { + name: path + .file_stem() + .unwrap_or_default() + .to_string_lossy() + .to_string(), + ..Default::default() + } +} + #[derive(Clone, Debug)] pub struct CreatePackDescription { pub icon: Option, @@ -182,17 +195,60 @@ pub async fn get_instance_from_pack( ..Default::default() }), CreatePackLocation::FromFile { path } => { - let file_name = path - .file_stem() - .unwrap_or_default() - .to_string_lossy() - .to_string(); + let mut instance = get_local_pack_instance(&path); + let file_size = tokio::fs::metadata(&path).await?.len(); + let hashes_archive = file_size <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE; + let archive_hashing_bytes = + if hashes_archive { file_size } else { 0 }; + let pack_file = CreatePackFile::Path(path.clone()); + let external_file_hashing_bytes = + super::install_mrpack::get_external_file_hashing_size_from_mrpack( + &pack_file, + ) + .await?; + let inspection_total_bytes = archive_hashing_bytes + .saturating_add(external_file_hashing_bytes) + .max(1); + let inspection = init_loading( + LoadingBarType::PackImport { + pack_name: instance.name.clone(), + }, + inspection_total_bytes as f64, + "Inspecting modpack", + ) + .await + .ok(); + let min_delta = (inspection_total_bytes / 200).max(256 * 1024); + let mut reported_bytes = 0_u64; + let mut report_progress = + |current: u64, offset: u64, message: &str| { + let target = offset + .saturating_add(current) + .min(inspection_total_bytes); + let increment = target.saturating_sub(reported_bytes); + if target < inspection_total_bytes && increment < min_delta + { + return; + } - let is_known_file = if tokio::fs::metadata(&path).await?.len() - <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE - { + if let Some(inspection) = &inspection { + let _ = emit_loading( + inspection, + increment as f64, + Some(message), + ); + } + reported_bytes = target; + }; + + let is_known_file = if hashes_archive { let state = State::get().await?; - let (_, hash) = sha1_file_async(&path).await?; + let (_, hash) = + sha1_file_async_with_progress(&path, |current, _| { + report_progress(current, 0, "Hashing local modpack"); + Ok(()) + }) + .await?; match CachedEntry::get_file_many( &[&hash], Some(CacheBehaviour::StaleWhileRevalidateSkipOffline), @@ -217,16 +273,26 @@ pub async fn get_instance_from_pack( let external_files_in_modpack = super::install_mrpack::get_external_files_from_mrpack( - &CreatePackFile::Path(path), + &pack_file, + |current, _| { + report_progress( + current, + archive_hashing_bytes, + "Inspecting modpack files", + ); + Ok(()) + }, ) .await?; + report_progress( + inspection_total_bytes, + 0, + "Finished inspecting modpack", + ); - Ok(CreatePackInstance { - name: file_name, - unknown_file: !is_known_file, - external_files_in_modpack, - ..Default::default() - }) + instance.unknown_file = !is_known_file; + instance.external_files_in_modpack = external_files_in_modpack; + Ok(instance) } } } diff --git a/packages/app-lib/src/api/pack/install_mrpack.rs b/packages/app-lib/src/api/pack/install_mrpack.rs index 52946bd120..80d8c6f64b 100644 --- a/packages/app-lib/src/api/pack/install_mrpack.rs +++ b/packages/app-lib/src/api/pack/install_mrpack.rs @@ -41,6 +41,7 @@ use tokio::sync::Mutex; type ExtractProgressFn<'a> = dyn FnMut(u64) -> Pin> + Send + 'a>> + Send + 'a; +type HashProgressFn<'a> = dyn FnMut(u64) -> crate::Result<()> + Send + 'a; const MODPACK_CONTENT_DOWNLOAD_CONCURRENCY: usize = 4; #[derive(Clone)] @@ -166,13 +167,16 @@ impl MrpackZipReader { async fn hash_entry( &mut self, index: usize, + progress: Option<&mut HashProgressFn<'_>>, ) -> crate::Result<(u64, String)> { match self { Self::Memory(reader) => { - hash_zip_entry(reader.reader_with_entry(index).await?).await + hash_zip_entry(reader.reader_with_entry(index).await?, progress) + .await } Self::File(reader) => { - hash_zip_entry(reader.reader_with_entry(index).await?).await + hash_zip_entry(reader.reader_with_entry(index).await?, progress) + .await } } } @@ -209,6 +213,7 @@ impl MrpackZipReader { async fn hash_zip_entry( mut reader: ZipEntryReader<'_, R, WithEntry<'_>>, + mut progress: Option<&mut HashProgressFn<'_>>, ) -> crate::Result<(u64, String)> where R: futures_lite::io::AsyncBufRead + Unpin, @@ -228,6 +233,9 @@ where hasher.update(&buffer[..bytes_read]); size += bytes_read as u64; + if let Some(progress) = progress.as_mut() { + progress(bytes_read as u64)?; + } } if reader.compute_hash() != expected_crc32 { @@ -239,6 +247,7 @@ where pub(crate) async fn get_external_files_from_mrpack( file: &CreatePackFile, + mut progress: impl FnMut(u64, u64) -> crate::Result<()> + Send, ) -> crate::Result> { let mut zip_reader = MrpackZipReader::new(file).await?; let Some(manifest_idx) = @@ -271,21 +280,35 @@ pub(crate) async fn get_external_files_from_mrpack( .enumerate() .filter_map(|(index, entry)| { let path = entry.filename().as_str().ok()?; - let relative_path = path - .strip_prefix("overrides/") - .or_else(|| path.strip_prefix("client-overrides/"))?; - if path.ends_with('/') - || ProjectType::get_from_parent_folder(relative_path).is_none() - { - return None; - } + let relative_path = external_override_relative_path(path)?; let file_name = relative_path.rsplit('/').next()?.to_string(); - Some((index, file_name)) + Some((index, file_name, entry.uncompressed_size())) }) .collect::>(); - for (index, file_name) in override_entries { - let (_, hash) = zip_reader.hash_entry(index).await?; + let total_bytes = override_entries + .iter() + .map(|(_, _, size)| size) + .sum::(); + let min_delta = (total_bytes / 200).max(256 * 1024); + let mut current_bytes = 0_u64; + let mut last_reported_bytes = 0_u64; + let mut report_progress = |bytes_read: u64| { + current_bytes = current_bytes.saturating_add(bytes_read); + if current_bytes >= total_bytes + || current_bytes.saturating_sub(last_reported_bytes) < min_delta + { + return Ok(()); + } + + last_reported_bytes = current_bytes; + progress(current_bytes.min(total_bytes), total_bytes) + }; + + for (index, file_name, _) in override_entries { + let (_, hash) = zip_reader + .hash_entry(index, Some(&mut report_progress)) + .await?; candidates.push((file_name, hash)); } @@ -327,6 +350,31 @@ pub(crate) async fn get_external_files_from_mrpack( Ok(external_files) } +pub(crate) async fn get_external_file_hashing_size_from_mrpack( + file: &CreatePackFile, +) -> crate::Result { + let zip_reader = MrpackZipReader::new(file).await?; + Ok(zip_reader + .file() + .entries() + .iter() + .filter_map(|entry| { + let path = entry.filename().as_str().ok()?; + external_override_relative_path(path) + .map(|_| entry.uncompressed_size()) + }) + .sum()) +} + +fn external_override_relative_path(path: &str) -> Option<&str> { + let relative_path = path + .strip_prefix("overrides/") + .or_else(|| path.strip_prefix("client-overrides/"))?; + (!path.ends_with('/') + && ProjectType::get_from_parent_folder(relative_path).is_some()) + .then_some(relative_path) +} + async fn extract_zip_entry( mut reader: ZipEntryReader<'_, R, WithEntry<'_>>, path: &Path, @@ -527,7 +575,7 @@ pub(crate) async fn install_zipped_mrpack_files_with_reporter( .collect(); for index in override_entries { - let (_, hash) = zip_reader.hash_entry(index).await?; + let (_, hash) = zip_reader.hash_entry(index, None).await?; file_hashes.push(hash); } diff --git a/packages/app-lib/src/event/emit.rs b/packages/app-lib/src/event/emit.rs index 64969cd727..bc82d9fafa 100644 --- a/packages/app-lib/src/event/emit.rs +++ b/packages/app-lib/src/event/emit.rs @@ -71,7 +71,7 @@ pub async fn init_loading_unsafe( message: title.to_string(), total, current: 0.0, - last_sent: 0.0, + last_sent: -1.0, bar_type, #[cfg(feature = "cli")] cli_progress_bar: { diff --git a/packages/app-lib/src/event/mod.rs b/packages/app-lib/src/event/mod.rs index 3a1dabb954..af1d5fe69f 100644 --- a/packages/app-lib/src/event/mod.rs +++ b/packages/app-lib/src/event/mod.rs @@ -157,6 +157,9 @@ pub enum LoadingBarType { pack_id: Option, pack_version: Option, }, + PackImport { + pack_name: String, + }, MinecraftDownload { instance_id: String, instance_name: String, diff --git a/packages/app-lib/src/install/runner.rs b/packages/app-lib/src/install/runner.rs index 801d957a11..1c2446aa10 100644 --- a/packages/app-lib/src/install/runner.rs +++ b/packages/app-lib/src/install/runner.rs @@ -16,6 +16,7 @@ use crate::ErrorKind; use crate::api::pack::install_from::{ CreatePackLocation, generate_pack_from_file, generate_pack_from_version_id_with_reporter, get_instance_from_pack, + get_local_pack_instance, }; use crate::api::pack::install_mrpack::install_zipped_mrpack_files_with_reporter; use crate::event::InstancePayloadType; @@ -163,16 +164,6 @@ pub async fn retry_job(job_id: Uuid) -> crate::Result { job.state.progress.phase = InstallPhaseId::PreparingInstance; job.state.progress.progress = None; job.state.progress.details = InstallPhaseDetails::Empty; - if let Err(error) = prepare_initial_instance(&mut job.state, &state).await { - if let Err(cleanup_error) = - recovery::apply_cleanup(&job.state, &state).await - { - tracing::error!( - "Error cleaning up install job {job_id} retry preparation: {cleanup_error}" - ); - } - return Err(error); - } job.state.record_event(InstallJobEventKind::JobQueued { kind: job.state.request.kind(), }); @@ -197,6 +188,42 @@ pub async fn retry_job(job_id: Uuid) -> crate::Result { return Err(error); } }; + emit_install_job(&record.snapshot()).await?; + + if let Err(error) = prepare_initial_instance(&mut job.state, &state).await { + let error_view = install_error_view( + job.state.progress.phase, + &error, + job.state.context.clone(), + ); + if let Err(terminal_error) = + terminalize_failed_job(job_id, job.state, error_view, &state).await + { + tracing::error!( + "Failed to terminalize retried install job {job_id}: {terminal_error}" + ); + } + return Err(error); + } + let record = match store::update_state(job_id, &job.state, &state).await { + Ok(record) => record, + Err(error) => { + let error_view = install_error_view( + job.state.progress.phase, + &error, + job.state.context.clone(), + ); + if let Err(terminal_error) = + terminalize_failed_job(job_id, job.state, error_view, &state) + .await + { + tracing::error!( + "Failed to terminalize retried install job {job_id}: {terminal_error}" + ); + } + return Err(error); + } + }; if let Err(error) = lock_existing_instance_if_needed(&job.state, &state).await { @@ -337,31 +364,39 @@ async fn start(request: InstallRequest) -> crate::Result { let state = State::get().await?; let id = Uuid::new_v4(); let mut job_state = InstallJobState::new(request); + set_initial_display(&mut job_state); + let record = + store::insert(id, &job_state, InstallJobStatus::Queued, &state).await?; + emit_install_job(&record.snapshot()).await?; + if let Err(error) = prepare_initial_instance(&mut job_state, &state).await { - if let Err(cleanup_error) = - recovery::apply_cleanup(&job_state, &state).await + let error_view = install_error_view( + job_state.progress.phase, + &error, + job_state.context.clone(), + ); + if let Err(terminal_error) = + terminalize_failed_job(id, job_state, error_view, &state).await { tracing::error!( - "Error cleaning up install job preparation: {cleanup_error}" + "Failed to terminalize install job {id} after setup error: {terminal_error}" ); } return Err(error); } - let record = match store::insert( - id, - &job_state, - InstallJobStatus::Queued, - &state, - ) - .await - { + let record = match store::update_state(id, &job_state, &state).await { Ok(record) => record, Err(error) => { - if let Err(cleanup_error) = - recovery::apply_cleanup(&job_state, &state).await + let error_view = install_error_view( + job_state.progress.phase, + &error, + job_state.context.clone(), + ); + if let Err(terminal_error) = + terminalize_failed_job(id, job_state, error_view, &state).await { tracing::error!( - "Error cleaning up untracked install job {id}: {cleanup_error}" + "Failed to terminalize install job {id} after setup error: {terminal_error}" ); } return Err(error); @@ -422,7 +457,12 @@ async fn prepare_initial_instance( location, post_install_edit, } => { - let preview = get_instance_from_pack(location).await?; + let preview = match location { + CreatePackLocation::FromFile { path } => { + get_local_pack_instance(&path) + } + location => get_instance_from_pack(location).await?, + }; let name = post_install_edit .as_ref() .and_then(|edit| edit.name.clone()) @@ -1443,6 +1483,26 @@ fn set_display( job_state.display = Some(InstallJobDisplay { title, icon }); } +fn set_initial_display(job_state: &mut InstallJobState) { + let display = match &job_state.request { + InstallRequest::CreateModpackInstance { location, .. } => { + match location { + CreatePackLocation::FromVersionId { + title, icon_url, .. + } => Some((title.clone(), icon_url.clone())), + CreatePackLocation::FromFile { path } => { + Some((get_local_pack_instance(path).name, None)) + } + } + } + _ => None, + }; + + if let Some((title, icon)) = display { + set_display(job_state, title, icon); + } +} + fn install_error_view( phase: InstallPhaseId, error: &crate::Error, diff --git a/packages/app-lib/src/util/fetch.rs b/packages/app-lib/src/util/fetch.rs index 89a440dbf9..17c0ec002e 100644 --- a/packages/app-lib/src/util/fetch.rs +++ b/packages/app-lib/src/util/fetch.rs @@ -954,12 +954,24 @@ pub async fn sha1_async(bytes: Bytes) -> crate::Result { pub async fn sha1_file_async( path: impl AsRef, +) -> crate::Result<(u64, String)> { + sha1_file_async_with_progress(path, |_, _| Ok(())).await +} + +pub async fn sha1_file_async_with_progress( + path: impl AsRef, + mut progress: impl FnMut(u64, u64) -> crate::Result<()>, ) -> crate::Result<(u64, String)> { let path = path.as_ref(); // Local files can be multi-gigabyte .mrpacks, so hash them without materializing bytes. let mut file = File::open(path) .await .map_err(|e| IOError::with_path(e, path))?; + let total = file + .metadata() + .await + .map_err(|e| IOError::with_path(e, path))? + .len(); let mut hasher = sha1_smol::Sha1::new(); let mut size = 0; let mut buffer = vec![0; 262144]; @@ -975,6 +987,7 @@ pub async fn sha1_file_async( hasher.update(&buffer[..bytes_read]); size += bytes_read as u64; + progress(size, total)?; } Ok((size, hasher.digest().to_string()))