feat: compact logs if they have logspam to prevent app crashing (#6181)

* feat: compact logs if they have logspam to prevent app crashing

* fix: lint

apps/app-frontend/src/composables/useInstanceConsole.ts

@@ -8,7 +8,7 @@ interface LogEntry {
 	filename: string
 	name?: string
 	log_type: string
-	stdout?: string
+	output?: string | null
 	age?: number
 	live?: boolean
 }
@@ -50,12 +50,12 @@ async function getHistoricalLogs(profilePathId: string, instancePath: string): P
 	const entry = getOrCreate(profilePathId)
 	if (entry.logList) return entry.logList
 
-	const logs: LogEntry[] = await get_logs(instancePath, false)
+	const logs: LogEntry[] = await get_logs(instancePath, true)
 	entry.logList = logs
 
 	for (const log of logs) {
-		if (log.stdout && log.stdout !== '') {
+		if (log.output) {
-			entry.historicalCache.set(log.filename, log.stdout)
+			entry.historicalCache.set(log.filename, log.output)
 		}
 	}
 

apps/app-frontend/src/helpers/logs.js

@@ -6,12 +6,11 @@
 import { invoke } from '@tauri-apps/api/core'
 
 /*
-A log is a struct containing the filename string, stdout, and stderr, as follows:
+A log is a struct containing the filename string and optional output, as follows:
 
 pub struct Logs {
     pub filename:  String,
-    pub stdout: String,
+    pub output: Option<String>,
-    pub stderr: String,
 }
 */
 

apps/app-frontend/src/pages/instance/Logs.vue

@@ -77,7 +77,7 @@ function buildLogList(rawLogs) {
 					log.filename !== 'latest_stdout.log' &&
 					log.filename !== 'latest_stdout' &&
 					log.filename !== 'launcher_log.txt' &&
-					log.stdout !== '' &&
+					(log.output == null || log.output !== '') &&
 					(log.filename.includes('.log') || log.filename.endsWith('.txt')),
 			)
 			.map((log) => ({

packages/app-lib/src/api/logs.rs

@@ -1,4 +1,5 @@
-use std::io::{Read, SeekFrom};
+use std::fmt::Write as _;
+use std::io::{BufRead, SeekFrom};
 use std::time::SystemTime;
 
 use futures::TryFutureExt;
@@ -28,6 +29,8 @@ pub enum LogType {
     CrashReport,
 }
 
+const LOG_COMPACTION_THRESHOLD: usize = 20;
+
 #[derive(Serialize, Debug)]
 pub struct LatestLogCursor {
     pub cursor: u64,
@@ -68,6 +71,142 @@ impl CensoredString {
     }
 }
 
+#[derive(Clone, Copy, Debug, Default)]
+struct LogCompactionStats {
+    compacted_runs: usize,
+    compacted_lines: usize,
+}
+
+struct CompactedLog {
+    output: String,
+    stats: LogCompactionStats,
+}
+
+fn split_line_ending(line: &str) -> (&str, &str) {
+    if let Some(line) = line.strip_suffix("\r\n") {
+        (line, "\r\n")
+    } else if let Some(line) = line.strip_suffix('\n') {
+        (line, "\n")
+    } else if let Some(line) = line.strip_suffix('\r') {
+        (line, "\r")
+    } else {
+        (line, "")
+    }
+}
+
+fn push_compacted_log_run(
+    output: &mut String,
+    stats: &mut LogCompactionStats,
+    line: &str,
+    line_ending: &str,
+    count: usize,
+) {
+    if count >= LOG_COMPACTION_THRESHOLD {
+        output.push_str(line);
+        let _ = write!(output, " (x{count} times - compacted by Modrinth App)");
+        output.push_str(line_ending);
+        stats.compacted_runs += 1;
+        stats.compacted_lines += count;
+    } else {
+        for _ in 0..count {
+            output.push_str(line);
+            output.push_str(line_ending);
+        }
+    }
+}
+
+fn read_compacted_log<R: BufRead>(
+    reader: &mut R,
+) -> std::io::Result<CompactedLog> {
+    let mut output = String::new();
+    let mut stats = LogCompactionStats::default();
+    let mut buffer = Vec::new();
+    let mut current_line: Option<String> = None;
+    let mut current_line_ending = String::new();
+    let mut current_count = 0usize;
+
+    loop {
+        buffer.clear();
+        let bytes_read = reader.read_until(b'\n', &mut buffer)?;
+        if bytes_read == 0 {
+            break;
+        }
+
+        let line = String::from_utf8_lossy(&buffer);
+        let (line, line_ending) = split_line_ending(&line);
+
+        match current_line.as_deref() {
+            Some(current) if current == line => {
+                current_count += 1;
+                if current_line_ending.is_empty() && !line_ending.is_empty() {
+                    current_line_ending = line_ending.to_string();
+                }
+            }
+            _ => {
+                if let Some(current) = current_line.take() {
+                    push_compacted_log_run(
+                        &mut output,
+                        &mut stats,
+                        &current,
+                        &current_line_ending,
+                        current_count,
+                    );
+                }
+
+                current_line = Some(line.to_string());
+                current_line_ending = line_ending.to_string();
+                current_count = 1;
+            }
+        }
+    }
+
+    if let Some(current) = current_line {
+        push_compacted_log_run(
+            &mut output,
+            &mut stats,
+            &current,
+            &current_line_ending,
+            current_count,
+        );
+    }
+
+    Ok(CompactedLog { output, stats })
+}
+
+fn compact_duplicate_lines(input: &str) -> CompactedLog {
+    let mut reader = std::io::Cursor::new(input.as_bytes());
+    read_compacted_log(&mut reader)
+        .expect("compacting an in-memory log should not fail")
+}
+
+fn format_count(count: usize) -> String {
+    let raw = count.to_string();
+    let mut formatted = String::with_capacity(raw.len() + raw.len() / 3);
+    for (index, character) in raw.chars().enumerate() {
+        if index > 0 && (raw.len() - index).is_multiple_of(3) {
+            formatted.push(',');
+        }
+        formatted.push(character);
+    }
+    formatted
+}
+
+async fn maybe_emit_log_compaction_warning(
+    file_name: &str,
+    stats: LogCompactionStats,
+) {
+    if stats.compacted_runs == 0 {
+        return;
+    }
+
+    let _ = crate::event::emit::emit_warning(&format!(
+        "Modrinth App has compacted {} repeated log lines in {} before displaying it for performance reasons.",
+        format_count(stats.compacted_lines),
+        file_name,
+    ))
+    .await;
+}
+
 impl Logs {
     async fn build(
         log_type: LogType,
@@ -218,41 +357,25 @@ pub async fn get_output_by_filename(
         .map(|x| x.1)
         .collect::<Vec<_>>();
 
-    // Load .gz file into String
     if let Some(ext) = path.extension() {
         if ext == "gz" {
             let file = std::fs::File::open(&path)
                 .map_err(|e| IOError::with_path(e, &path))?;
-            let mut contents = [0; 1024];
+            let gz =
-            let mut result = String::new();
-            let mut gz =
                 flate2::read::GzDecoder::new(std::io::BufReader::new(file));
-
+            let mut reader = std::io::BufReader::new(gz);
-            while gz
+            let compacted = read_compacted_log(&mut reader)
-                .read(&mut contents)
-                .map_err(|e| IOError::with_path(e, &path))?
-                > 0
-            {
-                result.push_str(&String::from_utf8_lossy(&contents));
-                contents = [0; 1024];
-            }
-            return Ok(CensoredString::censor(result, &credentials));
-        } else if ext == "log" || ext == "txt" {
-            let mut result = String::new();
-            let mut contents = [0; 1024];
-            let mut file = std::fs::File::open(&path)
                 .map_err(|e| IOError::with_path(e, &path))?;
-            // iteratively read the file to a String
+            maybe_emit_log_compaction_warning(file_name, compacted.stats).await;
-            while file
+            return Ok(CensoredString::censor(compacted.output, &credentials));
-                .read(&mut contents)
+        } else if ext == "log" || ext == "txt" {
-                .map_err(|e| IOError::with_path(e, &path))?
+            let file = std::fs::File::open(&path)
-                > 0
+                .map_err(|e| IOError::with_path(e, &path))?;
-            {
+            let mut reader = std::io::BufReader::new(file);
-                result.push_str(&String::from_utf8_lossy(&contents));
+            let compacted = read_compacted_log(&mut reader)
-                contents = [0; 1024];
+                .map_err(|e| IOError::with_path(e, &path))?;
-            }
+            maybe_emit_log_compaction_warning(file_name, compacted.stats).await;
-            let result = CensoredString::censor(result, &credentials);
+            return Ok(CensoredString::censor(compacted.output, &credentials));
-            return Ok(result);
         }
     }
     Err(crate::ErrorKind::OtherError(format!(
@@ -306,13 +429,15 @@ pub async fn get_live_log_buffer(
     let state = State::get().await?;
     let lines = crate::state::get_log_buffer(profile_path);
     let joined = lines.join("\n");
+    let compacted = compact_duplicate_lines(&joined);
 
     let credentials = Credentials::get_all(&state.pool)
         .await?
         .into_iter()
         .map(|x| x.1)
         .collect::<Vec<_>>();
-    Ok(CensoredString::censor(joined, &credentials))
+    maybe_emit_log_compaction_warning("live log", compacted.stats).await;
+    Ok(CensoredString::censor(compacted.output, &credentials))
 }
 
 pub fn clear_live_log_buffer(profile_path: &str) {
@@ -369,7 +494,8 @@ pub async fn get_generic_live_log_cursor(
         .read_to_end(&mut buffer)
         .map_err(|e| IOError::with_path(e, &path))
         .await?; // Read to end of file
-    let output = String::from_utf8_lossy(&buffer).to_string(); // Convert to String
+    let output = String::from_utf8_lossy(&buffer); // Convert to String
+    let compacted = compact_duplicate_lines(&output);
     let cursor = cursor + bytes_read as u64; // Update cursor
 
     let credentials = Credentials::get_all(&state.pool)
@@ -377,7 +503,8 @@ pub async fn get_generic_live_log_cursor(
         .into_iter()
         .map(|x| x.1)
         .collect::<Vec<_>>();
-    let output = CensoredString::censor(output, &credentials);
+    maybe_emit_log_compaction_warning(log_file_name, compacted.stats).await;
+    let output = CensoredString::censor(compacted.output, &credentials);
     Ok(LatestLogCursor {
         cursor,
         new_file,