fix: modpack export (#7119)

* fix: stale function call set_webview_visible

* fix: modpack export progress bar and add notification when done

* remove: force_no_zip64

* feat: switch to zip crate

* add typo exception

* fix: remove unit test (dont do)

---------

Signed-off-by: Truman Gao <106889354+tdgao@users.noreply.github.com>
Co-authored-by: Calum H. (IMB11) <contact@cal.engineer>

_typos.toml

@@ -24,3 +24,7 @@ gam = "gam"
 consts = "consts"
 # short for "Copy"
 Cpy = "Cpy"
+
+[default.extend-identifiers]
+# Constant from the `zip` crate
+ZIP64_BYTES_THR = "ZIP64_BYTES_THR"

apps/app-frontend/src/components/ui/ExportModal.vue

@@ -1,11 +1,12 @@
 <script setup>
-import { XIcon } from '@modrinth/assets'
+import { FolderOpenIcon, XIcon } from '@modrinth/assets'
 import {
 	Button,
 	commonMessages,
 	defineMessages,
 	FileTreeSelect,
 	injectNotificationManager,
+	injectPopupNotificationManager,
 	NewModal,
 	StyledInput,
 	useVIntl,
@@ -15,8 +16,10 @@ import { ref, shallowRef } from 'vue'
 
 import { PackageIcon } from '@/assets/icons'
 import { export_instance_mrpack, get_pack_export_candidates } from '@/helpers/instance'
+import { highlightInFolder } from '@/helpers/utils'
 
 const { handleError } = injectNotificationManager()
+const popupNotificationManager = injectPopupNotificationManager()
 const { formatMessage } = useVIntl()
 
 const messages = defineMessages({
@@ -39,6 +42,14 @@ const messages = defineMessages({
 		defaultMessage: 'Enter modpack description...',
 	},
 	exportButton: { id: 'app.export-modal.export-button', defaultMessage: 'Export' },
+	exportComplete: {
+		id: 'app.export-modal.export-complete',
+		defaultMessage: 'Export complete',
+	},
+	exportCompleteDescription: {
+		id: 'app.export-modal.export-complete-description',
+		defaultMessage: '{name} was exported successfully.',
+	},
 })
 
 const props = defineProps({
@@ -93,7 +104,10 @@ const exportPack = async () => {
 	})
 
 	if (outputPath) {
-		export_instance_mrpack(
+		exportModal.value.hide()
+
+		try {
+			await export_instance_mrpack(
 				props.instance.id,
 				outputPath,
 				includedFilePaths.value,
@@ -101,8 +115,24 @@ const exportPack = async () => {
 				versionInput.value,
 				exportDescription.value,
 				nameInput.value,
-		).catch((err) => handleError(err))
+			)
-		exportModal.value.hide()
+
+			const fileName = outputPath.split(/[\\/]/).pop() ?? outputPath
+			popupNotificationManager.addPopupNotification({
+				title: formatMessage(messages.exportComplete),
+				text: formatMessage(messages.exportCompleteDescription, { name: fileName }),
+				type: 'success',
+				buttons: [
+					{
+						label: formatMessage(commonMessages.openInFolderButton),
+						icon: FolderOpenIcon,
+						action: () => highlightInFolder(outputPath).catch(handleError),
+					},
+				],
+			})
+		} catch (error) {
+			handleError(error)
+		}
 	}
 }
 

apps/app-frontend/src/locales/en-US/index.json

@@ -287,6 +287,12 @@
   "app.export-modal.export-button": {
     "message": "Export"
   },
+  "app.export-modal.export-complete": {
+    "message": "Export complete"
+  },
+  "app.export-modal.export-complete-description": {
+    "message": "{name} was exported successfully."
+  },
   "app.export-modal.header": {
     "message": "Export modpack"
   },

packages/app-lib/src/api/instance/export_mrpack.rs

@@ -11,16 +11,15 @@ use crate::state::{
     VersionEnvironment,
 };
 use crate::util::io::{self, IOError};
-use async_zip::tokio::write::ZipFileWriter;
-use async_zip::{Compression, ZipEntryBuilder};
 use futures::{StreamExt, stream};
 use path_util::SafeRelativeUtf8UnixPathBuf;
 use serde::Serialize;
 use std::collections::{HashMap, HashSet};
+use std::io::{Read, Seek, Write};
 use std::path::PathBuf;
 use std::time::UNIX_EPOCH;
-use tokio::fs::File;
+use zip::write::SimpleFileOptions;
-use tokio_util::compat::FuturesAsyncWriteCompatExt;
+use zip::{CompressionMethod, ZipWriter};
 
 const DEFAULT_SELECTED_EXPORT_PATH_PREFIXES: &[&str] = &[
     "mods",
@@ -30,6 +29,8 @@ const DEFAULT_SELECTED_EXPORT_PATH_PREFIXES: &[&str] = &[
     "config",
 ];
 const EXPORT_CANDIDATE_METADATA_CONCURRENCY: usize = 32;
+const EXPORT_COPY_BUFFER_SIZE: usize = 256 * 1024;
+const STANDARD_ZIP_FILE_SIZE_ERROR: &str = "Your modpack cannot be exported as it contains a file over the size limit of 4 GB";
 
 const NEVER_EXPORTABLE_PATH_PREFIXES: &[&str] = &[
     "profile.json",
@@ -182,10 +183,6 @@ pub async fn export_mrpack(
     );
 
     let instance_base_path = get_full_path(instance_id).await?;
-    let mut file = File::create(&export_path)
-        .await
-        .map_err(|e| IOError::with_path(e, &export_path))?;
-    let mut writer = ZipFileWriter::with_tokio(&mut file).force_no_zip64();
     let version_id = version_id.unwrap_or("1.0.0".to_string());
     let mut packfile =
         create_mrpack_json(&metadata, version_id, description).await?;
@@ -197,16 +194,8 @@ pub async fn export_mrpack(
         .iter()
         .map(|file| file.path.as_str().to_string())
         .collect::<HashSet<_>>();
-    let loading_bar = init_loading(
-        LoadingBarType::ZipExtract {
-            instance_id: metadata.instance.id.clone(),
-            instance_name: metadata.instance.name.clone(),
-        },
-        1.0,
-        "Exporting instance to .mrpack",
-    )
-    .await?;
 
+    let mut override_files = Vec::new();
     let mut directories = vec![instance_base_path.clone()];
     while let Some(directory) = directories.pop() {
         let mut read_dir = io::read_dir(&directory).await?;
@@ -239,34 +228,91 @@ pub async fn export_mrpack(
                 continue;
             }
 
-            let mut stream = writer
+            let size = entry
-                .write_entry_stream(
+                .metadata()
-                    ZipEntryBuilder::new(
-                        format!("overrides/{relative_path}").into(),
-                        Compression::Deflate,
-                    )
-                    .build(),
-                )
-                .await?
-                .compat_write();
-            let mut source = File::open(&path)
                 .await
-                .map_err(|e| IOError::with_path(e, &path))?;
+                .map_err(|e| IOError::with_path(e, &path))?
-            tokio::io::copy(&mut source, &mut stream)
+                .len();
-                .await
+            ensure_standard_zip_file_size(size)?;
-                .map_err(IOError::from)?;
+            override_files.push((path, relative_path, size));
-            stream.into_inner().close().await?;
         }
     }
 
+    let total_bytes = override_files
+        .iter()
+        .fold(1_u64, |total, (_, _, size)| total.saturating_add(*size));
+    let loading_bar = init_loading(
+        LoadingBarType::PackExport {
+            instance_id: metadata.instance.id.clone(),
+            instance_name: metadata.instance.name.clone(),
+        },
+        total_bytes as f64,
+        "Exporting instance to .mrpack",
+    )
+    .await?;
     let data = serde_json::to_vec_pretty(&packfile)?;
-    let builder = ZipEntryBuilder::new(
+    tokio::task::spawn_blocking(move || {
-        "modrinth.index.json".to_string().into(),
+        let file = std::fs::File::create(&export_path)
-        Compression::Deflate,
+            .map_err(|error| IOError::with_path(error, &export_path))?;
-    );
+        write_mrpack_archive(file, override_files, &data, |bytes_written| {
-    writer.write_entry_whole(builder, &data).await?;
+            emit_loading(&loading_bar, bytes_written as f64, None)
-    writer.close().await?;
+        })
-    emit_loading(&loading_bar, 1.0, None)?;
+    })
+    .await??;
+
+    Ok(())
+}
+
+fn ensure_standard_zip_file_size(size: u64) -> crate::Result<()> {
+    if size > zip::ZIP64_BYTES_THR {
+        return Err(crate::ErrorKind::OtherError(
+            STANDARD_ZIP_FILE_SIZE_ERROR.to_string(),
+        )
+        .into());
+    }
+
+    Ok(())
+}
+
+fn write_mrpack_archive<W, F>(
+    writer: W,
+    override_files: Vec<(PathBuf, SafeRelativeUtf8UnixPathBuf, u64)>,
+    packfile_data: &[u8],
+    mut emit_progress: F,
+) -> crate::Result<()>
+where
+    W: Write + Seek,
+    F: FnMut(u64) -> crate::Result<()>,
+{
+    let options = SimpleFileOptions::default()
+        .compression_method(CompressionMethod::Deflated);
+    let mut writer = ZipWriter::new(writer);
+    let mut buffer = vec![0_u8; EXPORT_COPY_BUFFER_SIZE];
+
+    for (path, relative_path, _) in override_files {
+        writer
+            .start_file(format!("overrides/{relative_path}"), options)
+            .map_err(std::io::Error::from)?;
+        let mut source = std::fs::File::open(&path)
+            .map_err(|error| IOError::with_path(error, &path))?;
+        loop {
+            let bytes_read = source
+                .read(&mut buffer)
+                .map_err(|error| IOError::with_path(error, &path))?;
+            if bytes_read == 0 {
+                break;
+            }
+            writer.write_all(&buffer[..bytes_read])?;
+            emit_progress(bytes_read as u64)?;
+        }
+    }
+
+    writer
+        .start_file("modrinth.index.json", options)
+        .map_err(std::io::Error::from)?;
+    writer.write_all(packfile_data)?;
+    writer.finish().map_err(std::io::Error::from)?;
+    emit_progress(1)?;
 
     Ok(())
 }

packages/app-lib/src/event/mod.rs

@@ -312,6 +312,10 @@ pub enum LoadingBarType {
         instance_id: String,
         instance_name: String,
     },
+    PackExport {
+        instance_id: String,
+        instance_name: String,
+    },
     ConfigChange {
         new_path: String,
     },