fix: try fix install jobs being bricked/improve tracing (#6890)

* fix: try fix install jobs being bricked/improve tracing

* fix: rev cmts

---------

Co-authored-by: Prospector <6166773+Prospector@users.noreply.github.com>

apps/app-frontend/src/components/ui/settings/ResourceManagementSettings.vue

@@ -1,6 +1,14 @@
 <script setup>
 import { BoxIcon, FolderOpenIcon, FolderSearchIcon, TrashIcon } from '@modrinth/assets'
-import { ButtonStyled, injectNotificationManager, Slider, StyledInput } from '@modrinth/ui'
+import {
+	ButtonStyled,
+	defineMessages,
+	injectNotificationManager,
+	Slider,
+	StyledInput,
+	Toggle,
+	useVIntl,
+} from '@modrinth/ui'
 import { open } from '@tauri-apps/plugin-dialog'
 import { ref, watch } from 'vue'
 
@@ -11,9 +19,23 @@ import { showAppDbBackupsFolder } from '@/helpers/utils.js'
 import { useTheming } from '@/store/state'
 
 const { handleError } = injectNotificationManager()
+const { formatMessage } = useVIntl()
 const themeStore = useTheming()
 const settings = ref(await get())
 const purgeCacheConfirmModal = ref(null)
+const alwaysShowCopyDetailsFlag = 'always_show_copy_details'
+
+const messages = defineMessages({
+	alwaysShowCopyDetailsTitle: {
+		id: 'app.resource-management-settings.always-show-copy-details.title',
+		defaultMessage: 'Always show copy details',
+	},
+	alwaysShowCopyDetailsDescription: {
+		id: 'app.resource-management-settings.always-show-copy-details.description',
+		defaultMessage:
+			'Show the Copy details action while an install is queued or running. It is always available for failed or interrupted installs.',
+	},
+})
 
 watch(
 	settings,
@@ -154,6 +176,28 @@ async function findLauncherDir() {
 			</p>
 		</div>
 
+		<div class="flex items-center justify-between gap-4">
+			<div>
+				<h2 class="m-0 text-lg font-semibold text-contrast">
+					{{ formatMessage(messages.alwaysShowCopyDetailsTitle) }}
+				</h2>
+				<p class="m-0 mt-1">
+					{{ formatMessage(messages.alwaysShowCopyDetailsDescription) }}
+				</p>
+			</div>
+			<Toggle
+				id="always-show-copy-details"
+				:model-value="themeStore.getFeatureFlag(alwaysShowCopyDetailsFlag)"
+				@update:model-value="
+					() => {
+						const newValue = !themeStore.getFeatureFlag(alwaysShowCopyDetailsFlag)
+						themeStore.featureFlags[alwaysShowCopyDetailsFlag] = newValue
+						settings.feature_flags[alwaysShowCopyDetailsFlag] = newValue
+					}
+				"
+			/>
+		</div>
+
 		<div class="flex flex-col gap-2.5">
 			<h2 class="mt-0 m-0 text-lg font-semibold text-contrast">App database backups</h2>
 			<button id="open-db-backups-folder" class="btn min-w-max" @click="openDbBackupsFolder">

apps/app-frontend/src/composables/browse/install-job-notifications.ts

@@ -23,6 +23,7 @@ import {
 	type InstallProgress,
 } from '@/helpers/install'
 import { get_many as getInstances } from '@/helpers/instance'
+import { useTheming } from '@/store/state'
 
 const messages = defineMessages({
 	installs: {
@@ -221,13 +222,6 @@ const failureSummaryMessages = defineMessages({
 })
 
 const visibleJobStatuses = new Set<InstallJobStatus>(['queued', 'running', 'failed', 'interrupted'])
-const copyDetailsStallMs = 30_000
-
-interface ProgressSnapshot {
-	signature: string
-	changedAt: number
-	timeout: number | null
-}
 
 function getDisplayIconUrl(icon: string | null | undefined): string | null {
 	if (!icon) return null
@@ -241,6 +235,7 @@ export async function useInstallJobNotifications(opts: {
 	onChange: () => void
 }) {
 	const { formatMessage } = useVIntl()
+	const themeStore = useTheming()
 	const jobs = ref<InstallJobSnapshot[]>([])
 	const iconUrls = ref<Record<string, string | null>>({})
 	const instanceNames = ref<Record<string, string>>({})
@@ -250,7 +245,6 @@ export async function useInstallJobNotifications(opts: {
 	let metadataRequest = 0
 	let nextJobOrder = 0
 	const copiedResetTimeouts = new Map<string, number>()
-	const progressSnapshots = new Map<string, ProgressSnapshot>()
 
 	function getTitle(job: InstallJobSnapshot): string {
 		if (job.display?.title) return job.display.title
@@ -421,116 +415,14 @@ export async function useInstallJobNotifications(opts: {
 		return job.status === 'failed' || job.status === 'interrupted'
 	}
 
-	function canShowStalledProgressDetails(job: InstallJobSnapshot): boolean {
-		return (
-			job.status === 'running' &&
-			job.phase !== 'preparing_instance' &&
-			job.phase !== 'finalizing' &&
-			job.phase !== 'rolling_back'
-		)
-	}
-
 	function getJobSortRank(job: InstallJobSnapshot): number {
 		if (isTerminalJob(job)) return 0
 		if (job.status === 'queued' || job.phase === 'preparing_instance') return 2
 		return 1
 	}
 
-	function progressSignature(job: InstallJobSnapshot): string {
-		const progress = job.progress
-		const secondary = progress?.secondary
-		return [
-			job.status,
-			job.phase,
-			JSON.stringify(job.details),
-			progress?.current ?? '',
-			progress?.total ?? '',
-			secondary?.current ?? '',
-			secondary?.total ?? '',
-		].join(':')
-	}
-
-	function clearCopied(jobId: string) {
-		if (!copiedJobIds.value.has(jobId)) {
-			return
-		}
-
-		const timeout = copiedResetTimeouts.get(jobId)
-		if (timeout != null) {
-			window.clearTimeout(timeout)
-			copiedResetTimeouts.delete(jobId)
-		}
-
-		const nextCopiedJobIds = new Set(copiedJobIds.value)
-		nextCopiedJobIds.delete(jobId)
-		copiedJobIds.value = nextCopiedJobIds
-	}
-
-	function clearProgressSnapshot(jobId: string) {
-		const snapshot = progressSnapshots.get(jobId)
-		if (snapshot?.timeout != null) {
-			window.clearTimeout(snapshot.timeout)
-		}
-		progressSnapshots.delete(jobId)
-	}
-
-	function scheduleStaleProgressRefresh(jobId: string) {
-		const snapshot = progressSnapshots.get(jobId)
-		if (!snapshot) {
-			return
-		}
-
-		snapshot.timeout = window.setTimeout(() => {
-			const snapshot = progressSnapshots.get(jobId)
-			if (!snapshot) {
-				return
-			}
-
-			snapshot.timeout = null
-			opts.onChange()
-		}, copyDetailsStallMs)
-	}
-
-	function syncProgressSnapshots(nextJobs: InstallJobSnapshot[]) {
-		const trackedJobIds = new Set<string>()
-		const now = Date.now()
-
-		for (const job of nextJobs) {
-			if (!canShowStalledProgressDetails(job)) {
-				continue
-			}
-
-			trackedJobIds.add(job.job_id)
-			const signature = progressSignature(job)
-			const snapshot = progressSnapshots.get(job.job_id)
-			if (snapshot?.signature === signature) {
-				continue
-			}
-
-			clearProgressSnapshot(job.job_id)
-			clearCopied(job.job_id)
-			progressSnapshots.set(job.job_id, {
-				signature,
-				changedAt: now,
-				timeout: null,
-			})
-			scheduleStaleProgressRefresh(job.job_id)
-		}
-
-		for (const jobId of progressSnapshots.keys()) {
-			if (!trackedJobIds.has(jobId)) {
-				clearProgressSnapshot(jobId)
-			}
-		}
-	}
-
-	function hasStalledProgress(job: InstallJobSnapshot): boolean {
-		const snapshot = progressSnapshots.get(job.job_id)
-		return !!snapshot && Date.now() - snapshot.changedAt >= copyDetailsStallMs
-	}
-
 	function shouldShowCopyDetails(job: InstallJobSnapshot): boolean {
-		return isTerminalJob(job) || (canShowStalledProgressDetails(job) && hasStalledProgress(job))
+		return isTerminalJob(job) || themeStore.getFeatureFlag('always_show_copy_details')
 	}
 
 	function isCopied(job: InstallJobSnapshot): boolean {
@@ -607,6 +499,16 @@ export async function useInstallJobNotifications(opts: {
 		return buttons
 	}
 
+	function getDismissHandler(job: InstallJobSnapshot): (() => Promise<void>) | undefined {
+		if (isTerminalJob(job)) {
+			return async () => {
+				await install_job_dismiss(job.job_id).catch(opts.handleError)
+				await refresh()
+			}
+		}
+		return undefined
+	}
+
 	function setJobs(nextJobs: InstallJobSnapshot[]) {
 		for (const job of nextJobs) {
 			if (!jobOrder.has(job.job_id)) {
@@ -615,7 +517,6 @@ export async function useInstallJobNotifications(opts: {
 		}
 
 		const visibleJobs = nextJobs.filter((job) => visibleJobStatuses.has(job.status))
-		syncProgressSnapshots(visibleJobs)
 
 		jobs.value = visibleJobs.sort(
 			(a, b) =>
@@ -642,12 +543,8 @@ export async function useInstallJobNotifications(opts: {
 				progressCurrent: isTerminalJob(job) ? undefined : progress?.current,
 				progressTotal: isTerminalJob(job) ? undefined : progress?.total,
 				buttons: getButtons(job),
-				onDismiss: isTerminalJob(job)
+				dismissible: isTerminalJob(job),
-					? async () => {
+				onDismiss: getDismissHandler(job),
-							await install_job_dismiss(job.job_id).catch(opts.handleError)
-							await refresh()
-						}
-					: undefined,
 			}
 		}),
 	)
@@ -730,8 +627,8 @@ export async function useInstallJobNotifications(opts: {
 		void refreshMetadata()
 	}
 
-	await refresh(false)
 	const unlisten = await install_job_listener((job: InstallJobSnapshot) => applyJobUpdate(job))
+	await refresh(false)
 
 	return {
 		active: computed(() => jobs.value.length > 0),
@@ -743,9 +640,6 @@ export async function useInstallJobNotifications(opts: {
 			for (const timeout of copiedResetTimeouts.values()) {
 				window.clearTimeout(timeout)
 			}
-			for (const jobId of progressSnapshots.keys()) {
-				clearProgressSnapshot(jobId)
-			}
 			unlisten()
 		},
 	}

apps/app-frontend/src/locales/en-US/index.json

@@ -704,6 +704,12 @@
   "app.project.versions.already-installed": {
     "message": "Already installed"
   },
+  "app.resource-management-settings.always-show-copy-details.description": {
+    "message": "Show the Copy details action while an install is queued or running. It is always available for failed or interrupted installs."
+  },
+  "app.resource-management-settings.always-show-copy-details.title": {
+    "message": "Always show copy details"
+  },
   "app.settings.developer-mode-enabled": {
     "message": "Developer mode enabled."
   },

apps/app-frontend/src/store/theme.ts

@@ -17,6 +17,7 @@ export const DEFAULT_FEATURE_FLAGS = {
 	i18n_debug: false,
 	show_instance_play_time: true,
 	advanced_filters_collapsed: true,
+	always_show_copy_details: false,
 }
 
 export const THEME_OPTIONS = ['dark', 'light', 'oled', 'system'] as const

packages/app-lib/src/install/events.rs

@@ -202,11 +202,20 @@ pub async fn emit_install_job(
     {
         use tauri::Emitter;
 
-        let event_state = crate::EventState::get()?;
+        let result: crate::Result<()> = (|| {
-        event_state
+            let event_state = crate::EventState::get()?;
-            .app
+            event_state
-            .emit("install_job", snapshot)
+                .app
-            .map_err(crate::event::EventError::from)?;
+                .emit("install_job", snapshot)
+                .map_err(crate::event::EventError::from)?;
+            Ok(())
+        })();
+        if let Err(error) = result {
+            tracing::warn!(
+                "Failed to emit install job {} update: {error}",
+                snapshot.job_id
+            );
+        }
     }
 
     Ok(())

packages/app-lib/src/install/recovery.rs

@@ -262,29 +262,51 @@ async fn copy_symlink(source: &Path, target: &Path) -> crate::Result<()> {
 pub async fn recover_interrupted_jobs(state: &State) -> crate::Result<()> {
     let jobs = store::list_interrupted_candidates(state).await?;
 
-    for mut job in jobs {
+    for job in jobs {
-        if job.state.display.is_none() {
+        let job_id = job.id;
-            job.state.display = display_from_request(&job.state);
+        if let Err(error) = recover_interrupted_job(job, state).await {
+            tracing::error!(
+                "Error recovering interrupted install job {job_id}: {error}"
+            );
         }
-        let interrupted_phase = job.state.progress.phase;
+    }
-        job.state.record_event(InstallJobEventKind::Interrupted {
-            reason: InstallInterruptReason::AppClosed,
-            phase: interrupted_phase,
-        });
-        job.state.progress.phase = InstallPhaseId::RollingBack;
-        job.state.progress.progress = None;
-        job.state.progress.details = InstallPhaseDetails::Empty;
-        job.state.error = Some(InstallErrorView::from_message(
-            "app_closed",
-            interrupted_phase,
-            "App closed while install was running",
-        ));
 
-        job.state
+    Ok(())
-            .record_event(InstallJobEventKind::RollbackStarted {
+}
-                cleanup: job.state.cleanup.clone(),
+
-            });
+async fn recover_interrupted_job(
-        if let Err(error) = apply_cleanup(&job.state, state).await {
+    mut job: store::InstallJobRecord,
+    state: &State,
+) -> crate::Result<()> {
+    if job.state.display.is_none() {
+        job.state.display = display_from_request(&job.state);
+    }
+    let interrupted_phase = job.state.progress.phase;
+    job.state.record_event(InstallJobEventKind::Interrupted {
+        reason: InstallInterruptReason::AppClosed,
+        phase: interrupted_phase,
+    });
+    job.state.progress.phase = InstallPhaseId::RollingBack;
+    job.state.progress.progress = None;
+    job.state.progress.details = InstallPhaseDetails::Empty;
+    job.state.error = Some(InstallErrorView::from_message(
+        "app_closed",
+        interrupted_phase,
+        "App closed while install was running",
+    ));
+
+    job.state
+        .record_event(InstallJobEventKind::RollbackStarted {
+            cleanup: job.state.cleanup.clone(),
+        });
+    let cleanup_succeeded = match apply_cleanup(&job.state, state).await {
+        Ok(()) => {
+            job.state
+                .record_event(InstallJobEventKind::RollbackCompleted);
+            clear_deleted_new_instance_id(&mut job.state);
+            true
+        }
+        Err(error) => {
             tracing::error!(
                 "Error cleaning up interrupted install job {}: {error}",
                 job.id
@@ -298,20 +320,19 @@ pub async fn recover_interrupted_jobs(state: &State) -> crate::Result<()> {
             job.state.record_event(InstallJobEventKind::RollbackFailed {
                 message: error.to_string(),
             });
-        } else {
+            false
-            job.state
-                .record_event(InstallJobEventKind::RollbackCompleted);
         }
-        clear_deleted_new_instance_id(&mut job.state);
+    };
 
-        let record = store::update_status(
+    if let Some(record) = store::finish_active(
-            job.id,
+        job.id,
-            InstallJobStatus::Interrupted,
+        InstallJobStatus::Interrupted,
-            &job.state,
+        &job.state,
-            state,
+        state,
-        )
+    )
-        .await?;
+    .await?
-        if job.state.rollback_error.is_none() {
+    {
+        if cleanup_succeeded {
             clear_staging_dir(&job.state).await;
         }
         emit_install_job(&record.snapshot()).await?;
@@ -383,10 +404,20 @@ pub async fn apply_cleanup(
     match &job_state.cleanup {
         InstallCleanup::DeleteNewInstance { instance_id } => {
             if let Some(instance_id) = instance_id {
-                let _ = crate::state::remove_instance(instance_id, state).await;
+                if crate::state::get_instance(instance_id, &state.pool)
-                let _ =
+                    .await?
+                    .is_some()
+                {
+                    crate::state::remove_instance(instance_id, state).await?;
+                }
+                if let Err(error) =
                     emit_instance(instance_id, InstancePayloadType::Removed)
-                        .await;
+                        .await
+                {
+                    tracing::warn!(
+                        "Failed to emit removed instance {instance_id}: {error}"
+                    );
+                }
             }
         }
         InstallCleanup::RestoreExistingInstance { instance_id } => {
@@ -410,7 +441,14 @@ pub async fn apply_cleanup(
                     )
                     .await?;
                 }
-                emit_instance(instance_id, InstancePayloadType::Edited).await?;
+                if let Err(error) =
+                    emit_instance(instance_id, InstancePayloadType::Edited)
+                        .await
+                {
+                    tracing::warn!(
+                        "Failed to emit restored instance {instance_id}: {error}"
+                    );
+                }
             }
         }
     }

packages/app-lib/src/install/runner.rs

@@ -163,19 +163,57 @@ pub async fn retry_job(job_id: Uuid) -> crate::Result<InstallJobSnapshot> {
     job.state.progress.phase = InstallPhaseId::PreparingInstance;
     job.state.progress.progress = None;
     job.state.progress.details = InstallPhaseDetails::Empty;
-    prepare_initial_instance(&mut job.state, &state).await?;
+    if let Err(error) = prepare_initial_instance(&mut job.state, &state).await {
+        if let Err(cleanup_error) =
+            recovery::apply_cleanup(&job.state, &state).await
+        {
+            tracing::error!(
+                "Error cleaning up install job {job_id} retry preparation: {cleanup_error}"
+            );
+        }
+        return Err(error);
+    }
     job.state.record_event(InstallJobEventKind::JobQueued {
         kind: job.state.request.kind(),
     });
 
-    let record = store::update_status(
+    let record = match store::update_status(
         job_id,
         InstallJobStatus::Queued,
         &job.state,
         &state,
     )
-    .await?;
+    .await
-    lock_existing_instance_if_needed(&job.state, &state).await?;
+    {
+        Ok(record) => record,
+        Err(error) => {
+            if let Err(cleanup_error) =
+                recovery::apply_cleanup(&job.state, &state).await
+            {
+                tracing::error!(
+                    "Error cleaning up unqueued install job {job_id}: {cleanup_error}"
+                );
+            }
+            return Err(error);
+        }
+    };
+    if let Err(error) =
+        lock_existing_instance_if_needed(&job.state, &state).await
+    {
+        let error_view = install_error_view(
+            job.state.progress.phase,
+            &error,
+            job.state.context.clone(),
+        );
+        if let Err(terminal_error) =
+            terminalize_failed_job(job_id, job.state, error_view, &state).await
+        {
+            tracing::error!(
+                "Failed to terminalize retried install job {job_id}: {terminal_error}"
+            );
+        }
+        return Err(error);
+    }
     emit_install_job(&record.snapshot()).await?;
     spawn_job(job_id);
 
@@ -206,31 +244,50 @@ pub async fn cancel_job(job_id: Uuid) -> crate::Result<InstallJobSnapshot> {
         .record_event(InstallJobEventKind::RollbackStarted {
             cleanup: job.state.cleanup.clone(),
         });
-    match recovery::apply_cleanup(&job.state, &state).await {
+    let status_updated = store::update_status_if(
-        Ok(()) => job
-            .state
-            .record_event(InstallJobEventKind::RollbackCompleted),
-        Err(error) => {
-            job.state.rollback_error = Some(InstallErrorView::from_error(
-                "rollback_error",
-                InstallPhaseId::RollingBack,
-                &error,
-                None,
-            ));
-            job.state.record_event(InstallJobEventKind::RollbackFailed {
-                message: error.to_string(),
-            });
-        }
-    }
-    clear_deleted_new_instance_id(&mut job.state);
-    let record = store::update_status(
         job_id,
-        InstallJobStatus::Canceled,
+        InstallJobStatus::Queued,
+        InstallJobStatus::Running,
         &job.state,
         &state,
     )
-    .await?;
+    .await?
-    if job.state.rollback_error.is_none() {
+    .is_some();
+    if !status_updated {
+        return Err(crate::ErrorKind::InputError(
+            "Install job is no longer queued".to_string(),
+        )
+        .into());
+    }
+    let cleanup_succeeded =
+        match recovery::apply_cleanup(&job.state, &state).await {
+            Ok(()) => {
+                job.state
+                    .record_event(InstallJobEventKind::RollbackCompleted);
+                clear_deleted_new_instance_id(&mut job.state);
+                true
+            }
+            Err(error) => {
+                job.state.rollback_error = Some(InstallErrorView::from_error(
+                    "rollback_error",
+                    InstallPhaseId::RollingBack,
+                    &error,
+                    None,
+                ));
+                job.state.record_event(InstallJobEventKind::RollbackFailed {
+                    message: error.to_string(),
+                });
+                false
+            }
+        };
+    let status = if cleanup_succeeded {
+        InstallJobStatus::Canceled
+    } else {
+        InstallJobStatus::Failed
+    };
+    let record =
+        store::update_status(job_id, status, &job.state, &state).await?;
+    if cleanup_succeeded {
         recovery::clear_staging_dir(&job.state).await;
     }
     emit_install_job(&record.snapshot()).await?;
@@ -247,10 +304,53 @@ async fn start(request: InstallRequest) -> crate::Result<InstallJobSnapshot> {
     let state = State::get().await?;
     let id = Uuid::new_v4();
     let mut job_state = InstallJobState::new(request);
-    prepare_initial_instance(&mut job_state, &state).await?;
+    if let Err(error) = prepare_initial_instance(&mut job_state, &state).await {
-    let record =
+        if let Err(cleanup_error) =
-        store::insert(id, &job_state, InstallJobStatus::Queued, &state).await?;
+            recovery::apply_cleanup(&job_state, &state).await
-    lock_existing_instance_if_needed(&job_state, &state).await?;
+        {
+            tracing::error!(
+                "Error cleaning up install job preparation: {cleanup_error}"
+            );
+        }
+        return Err(error);
+    }
+    let record = match store::insert(
+        id,
+        &job_state,
+        InstallJobStatus::Queued,
+        &state,
+    )
+    .await
+    {
+        Ok(record) => record,
+        Err(error) => {
+            if let Err(cleanup_error) =
+                recovery::apply_cleanup(&job_state, &state).await
+            {
+                tracing::error!(
+                    "Error cleaning up untracked install job {id}: {cleanup_error}"
+                );
+            }
+            return Err(error);
+        }
+    };
+    if let Err(error) =
+        lock_existing_instance_if_needed(&job_state, &state).await
+    {
+        let error_view = install_error_view(
+            job_state.progress.phase,
+            &error,
+            job_state.context.clone(),
+        );
+        if let Err(terminal_error) =
+            terminalize_failed_job(id, job_state, error_view, &state).await
+        {
+            tracing::error!(
+                "Failed to terminalize install job {id} after setup error: {terminal_error}"
+            );
+        }
+        return Err(error);
+    }
     emit_install_job(&record.snapshot()).await?;
     spawn_job(id);
     Ok(record.snapshot())
@@ -370,9 +470,9 @@ async fn prepare_initial_instance(
                 metadata.instance.icon_path,
             );
             let instance_id = metadata.instance.id;
+            set_instance_id(job_state, instance_id.clone());
             attach_pending_shared_instance(&instance_id, &data, state).await?;
             emit_instance(&instance_id, InstancePayloadType::Edited).await?;
-            set_instance_id(job_state, instance_id);
         }
         InstallRequest::ImportInstance {
             instance_folder, ..
@@ -433,9 +533,14 @@ async fn prepare_initial_instance(
 fn spawn_job(job_id: Uuid) {
     tokio::spawn(async move {
         if let Err(error) = Box::pin(run_job(job_id)).await {
-            tracing::error!(
+            let failure = error.to_string();
-                "Install job {job_id} failed to update state: {error}"
+            tracing::error!("Install job {job_id} terminated: {failure}");
-            );
+            if let Err(error) = terminalize_stranded_job(job_id, failure).await
+            {
+                tracing::error!(
+                    "Failed to terminalize stranded install job {job_id}: {error}"
+                );
+            }
         }
     });
 }
@@ -457,13 +562,17 @@ async fn run_job(job_id: Uuid) -> crate::Result<()> {
 
     let mut job_state = job.state.clone();
     job_state.record_event(InstallJobEventKind::JobStarted);
-    let record = store::update_status(
+    let Some(record) = store::update_status_if(
         job_id,
+        InstallJobStatus::Queued,
         InstallJobStatus::Running,
         &job_state,
         &state,
     )
-    .await?;
+    .await?
+    else {
+        return Ok(());
+    };
     emit_install_job(&record.snapshot()).await?;
 
     let result = Box::pin(run_request(job_id, &mut job_state, &state)).await;
@@ -472,19 +581,21 @@ async fn run_job(job_id: Uuid) -> crate::Result<()> {
     }
 
     let result = match result {
-        Ok(instance_id) => {
+        Ok(Some(instance_id)) => {
-            if let Some(instance_id) = instance_id {
+            set_instance_id(&mut job_state, instance_id.clone());
-                set_instance_id(&mut job_state, instance_id);
+            Ok(instance_id)
-            }
-            finalize_existing_instance_success(&job_state, &state).await
         }
+        Ok(None) => Err(crate::ErrorKind::InputError(
+            "Install job completed without an instance id".to_string(),
+        )
+        .into()),
         Err(error) => Err(error),
     };
 
     match result {
-        Ok(()) => {
+        Ok(instance_id) => {
             job_state.record_event(InstallJobEventKind::JobSucceeded {
-                instance_id: current_instance_id(&job_state),
+                instance_id: Some(instance_id.clone()),
             });
             job_state.progress.phase = InstallPhaseId::Finalizing;
             job_state.progress.progress = None;
@@ -492,71 +603,119 @@ async fn run_job(job_id: Uuid) -> crate::Result<()> {
             job_state.error = None;
             job_state.rollback_error = None;
             job_state.context = None;
-            let record = store::update_status(
+            if let Some(record) =
-                job_id,
+                store::complete_success(job_id, &job_state, &state).await?
-                InstallJobStatus::Succeeded,
+            {
-                &job_state,
+                recovery::clear_staging_dir(&job_state).await;
-                &state,
+                if let Err(error) =
-            )
+                    emit_instance(&instance_id, InstancePayloadType::Edited)
-            .await?;
+                        .await
-            recovery::clear_staging_dir(&job_state).await;
+                {
-            emit_install_job(&record.snapshot()).await?;
+                    tracing::warn!(
+                        "Failed to emit completed instance {instance_id}: {error}"
+                    );
+                }
+                emit_install_job(&record.snapshot()).await?;
+            }
         }
         Err(error) => {
-            let failed_phase = job_state.progress.phase;
+            tracing::error!("Install job {job_id} failed: {error}");
             let error_view = install_error_view(
-                failed_phase,
+                job_state.progress.phase,
                 &error,
                 job_state.context.clone(),
             );
-            job_state.record_event(InstallJobEventKind::Failed {
+            terminalize_failed_job(job_id, job_state, error_view, &state)
-                phase: failed_phase,
+                .await?;
-                code: error_view.code.clone(),
-                message: error_view.message.clone(),
-            });
-            job_state.error = Some(error_view);
-            job_state.progress.phase = InstallPhaseId::RollingBack;
-            job_state.progress.progress = None;
-            job_state.progress.details = InstallPhaseDetails::Empty;
-            job_state.record_event(InstallJobEventKind::RollbackStarted {
-                cleanup: job_state.cleanup.clone(),
-            });
-            if let Err(rollback_error) =
-                recovery::apply_cleanup(&job_state, &state).await
-            {
-                tracing::error!(
-                    "Error rolling back failed install job {job_id}: {rollback_error}"
-                );
-                job_state.rollback_error = Some(install_error_view(
-                    InstallPhaseId::RollingBack,
-                    &rollback_error,
-                    None,
-                ));
-                job_state.record_event(InstallJobEventKind::RollbackFailed {
-                    message: rollback_error.to_string(),
-                });
-            } else {
-                job_state.record_event(InstallJobEventKind::RollbackCompleted);
-            }
-            clear_deleted_new_instance_id(&mut job_state);
-            let record = store::update_status(
-                job_id,
-                InstallJobStatus::Failed,
-                &job_state,
-                &state,
-            )
-            .await?;
-            if job_state.rollback_error.is_none() {
-                recovery::clear_staging_dir(&job_state).await;
-            }
-            emit_install_job(&record.snapshot()).await?;
-            return Err(error);
         }
     }
 
     Ok(())
 }
 
+async fn terminalize_stranded_job(
+    job_id: Uuid,
+    message: String,
+) -> crate::Result<()> {
+    let state = State::get().await?;
+    let job = store::get_required(job_id, &state).await?;
+    if !matches!(
+        job.status,
+        InstallJobStatus::Queued | InstallJobStatus::Running
+    ) {
+        return Ok(());
+    }
+
+    let error_view = InstallErrorView::from_message(
+        "install_worker_terminated",
+        job.state.progress.phase,
+        message,
+    );
+    terminalize_failed_job(job_id, job.state, error_view, &state).await
+}
+
+async fn terminalize_failed_job(
+    job_id: Uuid,
+    mut job_state: InstallJobState,
+    error_view: InstallErrorView,
+    state: &State,
+) -> crate::Result<()> {
+    let failed_phase = job_state.progress.phase;
+    job_state.record_event(InstallJobEventKind::Failed {
+        phase: failed_phase,
+        code: error_view.code.clone(),
+        message: error_view.message.clone(),
+    });
+    job_state.error = Some(error_view);
+    job_state.rollback_error = None;
+    job_state.progress.phase = InstallPhaseId::RollingBack;
+    job_state.progress.progress = None;
+    job_state.progress.details = InstallPhaseDetails::Empty;
+    job_state.record_event(InstallJobEventKind::RollbackStarted {
+        cleanup: job_state.cleanup.clone(),
+    });
+
+    let cleanup_succeeded = match recovery::apply_cleanup(&job_state, state)
+        .await
+    {
+        Ok(()) => {
+            job_state.record_event(InstallJobEventKind::RollbackCompleted);
+            clear_deleted_new_instance_id(&mut job_state);
+            true
+        }
+        Err(rollback_error) => {
+            tracing::error!(
+                "Error rolling back failed install job {job_id}: {rollback_error}"
+            );
+            job_state.rollback_error = Some(install_error_view(
+                InstallPhaseId::RollingBack,
+                &rollback_error,
+                None,
+            ));
+            job_state.record_event(InstallJobEventKind::RollbackFailed {
+                message: rollback_error.to_string(),
+            });
+            false
+        }
+    };
+
+    if let Some(record) = store::finish_active(
+        job_id,
+        InstallJobStatus::Failed,
+        &job_state,
+        state,
+    )
+    .await?
+    {
+        if cleanup_succeeded {
+            recovery::clear_staging_dir(&job_state).await;
+        }
+        emit_install_job(&record.snapshot()).await?;
+    }
+
+    Ok(())
+}
+
 async fn run_request(
     job_id: Uuid,
     job_state: &mut InstallJobState,
@@ -1176,25 +1335,6 @@ async fn lock_existing_instance(
     Ok(())
 }
 
-async fn finalize_existing_instance_success(
-    job_state: &InstallJobState,
-    state: &State,
-) -> crate::Result<()> {
-    if let InstallCleanup::RestoreExistingInstance { instance_id } =
-        &job_state.cleanup
-    {
-        crate::state::instances::commands::set_instance_install_stage(
-            instance_id,
-            InstanceInstallStage::Installed,
-            &state.pool,
-        )
-        .await?;
-        emit_instance(instance_id, InstancePayloadType::Edited).await?;
-    }
-
-    Ok(())
-}
-
 pub(super) async fn update_progress(
     job_id: Uuid,
     job_state: &mut InstallJobState,

packages/app-lib/src/install/store.rs

@@ -171,7 +171,7 @@ pub async fn list(
 		.await?
     };
 
-    rows.into_iter().map(row_to_record).collect()
+    Ok(deserialize_rows(rows))
 }
 
 pub async fn list_interrupted_candidates(
@@ -198,7 +198,7 @@ pub async fn list_interrupted_candidates(
     .fetch_all(&app_state.pool)
     .await?;
 
-    rows.into_iter().map(row_to_record).collect()
+    Ok(deserialize_rows(rows))
 }
 
 pub async fn update_state(
@@ -262,6 +262,143 @@ pub async fn update_status(
     get_required(id, app_state).await
 }
 
+pub async fn update_status_if(
+    id: Uuid,
+    expected_status: InstallJobStatus,
+    status: InstallJobStatus,
+    state: &InstallJobState,
+    app_state: &State,
+) -> crate::Result<Option<InstallJobRecord>> {
+    let now = Utc::now();
+    let finished = status.is_finished().then_some(now.timestamp());
+    let json = serde_json::to_string(state)?;
+    let status_value = status.as_str();
+    let expected_status_value = expected_status.as_str();
+    let instance_id = instance_id(state);
+    let id_value = id.to_string();
+    let modified = now.timestamp();
+
+    let result = sqlx::query(
+        "
+		UPDATE install_jobs
+		SET instance_id = ?, status = ?, state = ?, modified = ?, finished = ?
+		WHERE id = ? AND status = ?
+		",
+    )
+    .bind(instance_id)
+    .bind(status_value)
+    .bind(json)
+    .bind(modified)
+    .bind(finished)
+    .bind(id_value)
+    .bind(expected_status_value)
+    .execute(&app_state.pool)
+    .await?;
+
+    if result.rows_affected() == 0 {
+        return Ok(None);
+    }
+
+    get_required(id, app_state).await.map(Some)
+}
+
+pub async fn finish_active(
+    id: Uuid,
+    status: InstallJobStatus,
+    state: &InstallJobState,
+    app_state: &State,
+) -> crate::Result<Option<InstallJobRecord>> {
+    let now = Utc::now();
+    let finished = now.timestamp();
+    let json = serde_json::to_string(state)?;
+    let status_value = status.as_str();
+    let instance_id = instance_id(state);
+    let id_value = id.to_string();
+    let modified = finished;
+
+    let result = sqlx::query(
+        "
+		UPDATE install_jobs
+		SET instance_id = ?, status = ?, state = ?, modified = ?, finished = ?
+		WHERE id = ? AND status IN ('queued', 'running')
+		",
+    )
+    .bind(instance_id)
+    .bind(status_value)
+    .bind(json)
+    .bind(modified)
+    .bind(finished)
+    .bind(id_value)
+    .execute(&app_state.pool)
+    .await?;
+
+    if result.rows_affected() == 0 {
+        return Ok(None);
+    }
+
+    get_required(id, app_state).await.map(Some)
+}
+
+pub async fn complete_success(
+    id: Uuid,
+    state: &InstallJobState,
+    app_state: &State,
+) -> crate::Result<Option<InstallJobRecord>> {
+    let Some(instance_id) = instance_id(state) else {
+        return Err(crate::ErrorKind::InputError(
+            "Install job is missing its instance id".to_string(),
+        )
+        .into());
+    };
+    let now = Utc::now().timestamp();
+    let json = serde_json::to_string(state)?;
+    let id_value = id.to_string();
+    let mut transaction = app_state.pool.begin().await?;
+
+    let job_result = sqlx::query(
+		"
+		UPDATE install_jobs
+		SET instance_id = ?, status = 'succeeded', state = ?, modified = ?, finished = ?
+		WHERE id = ? AND status = 'running'
+		",
+	)
+	.bind(&instance_id)
+	.bind(json)
+	.bind(now)
+	.bind(now)
+	.bind(id_value)
+	.execute(&mut *transaction)
+	.await?;
+
+    if job_result.rows_affected() == 0 {
+        transaction.rollback().await?;
+        return Ok(None);
+    }
+
+    let instance_result = sqlx::query(
+        "
+		UPDATE instances
+		SET install_stage = 'installed', modified = ?
+		WHERE id = ?
+		",
+    )
+    .bind(now)
+    .bind(&instance_id)
+    .execute(&mut *transaction)
+    .await?;
+
+    if instance_result.rows_affected() == 0 {
+        transaction.rollback().await?;
+        return Err(crate::ErrorKind::InputError(format!(
+            "Unknown instance {instance_id}"
+        ))
+        .into());
+    }
+
+    transaction.commit().await?;
+    get_required(id, app_state).await.map(Some)
+}
+
 pub async fn dismiss(id: Uuid, app_state: &State) -> crate::Result<()> {
     let id = id.to_string();
     let modified = Utc::now().timestamp();
@@ -308,6 +445,23 @@ fn row_to_record(row: InstallJobRow) -> crate::Result<InstallJobRecord> {
     })
 }
 
+fn deserialize_rows(rows: Vec<InstallJobRow>) -> Vec<InstallJobRecord> {
+    rows.into_iter()
+        .filter_map(|row| {
+            let id = row.id.clone();
+            match row_to_record(row) {
+                Ok(record) => Some(record),
+                Err(error) => {
+                    tracing::error!(
+                        "Failed to deserialize install job {id}: {error}"
+                    );
+                    None
+                }
+            }
+        })
+        .collect()
+}
+
 fn instance_id(state: &InstallJobState) -> Option<String> {
     match &state.target {
         super::model::InstallTarget::NewInstance { instance_id } => {

packages/app-lib/src/launcher/mod.rs

@@ -603,19 +603,21 @@ pub async fn install_minecraft_with_reporter(
 
     let protocol_version = read_protocol_version_from_jar(client_path).await?;
 
-    crate::state::instances::commands::set_instance_install_stage(
-        &instance.id,
-        InstanceInstallStage::Installed,
-        &state.pool,
-    )
-    .await?;
-    emit_instance(&instance.id, InstancePayloadType::Edited).await?;
     crate::state::instances::commands::set_applied_content_set_protocol_version(
         &instance.id,
         protocol_version,
         &state.pool,
     )
     .await?;
+	if reporter.is_none() {
+		crate::state::instances::commands::set_instance_install_stage(
+			&instance.id,
+			InstanceInstallStage::Installed,
+			&state.pool,
+		)
+		.await?;
+		emit_instance(&instance.id, InstancePayloadType::Edited).await?;
+	}
     if let Some(loading_bar) = &loading_bar {
         emit_loading(loading_bar, 1.0, Some("Finished installing"))?;
     }

packages/app-lib/src/state/settings.rs

@@ -65,6 +65,7 @@ pub enum FeatureFlag {
     ShowInstancePlayTime,
     SkipNonEssentialWarnings,
     AdvancedFiltersCollapsed,
+    AlwaysShowCopyDetails,
 }
 
 impl Settings {

packages/ui/src/components/nav/PopupNotificationPanel.vue

@@ -52,6 +52,7 @@
 							:progress-current="progressItem.progressCurrent"
 							:progress-total="progressItem.progressTotal"
 							:actions="progressItem.buttons"
+							:dismissible="progressItem.dismissible"
 							@dismiss="handleProgressItemDismiss(item, progressItem)"
 							@action="(index) => handleProgressItemAction(progressItem, index)"
 						/>

packages/ui/src/components/notifications/NotificationToast.vue

@@ -49,7 +49,7 @@
 							</template>
 						</template>
 					</p>
-					<ButtonStyled size="small" type="transparent" circular>
+					<ButtonStyled v-if="dismissible" size="small" type="transparent" circular>
 						<button
 							type="button"
 							class="notification-toast-dismiss"
@@ -96,7 +96,7 @@
 					{{ entityLabel }}
 				</p>
 				<div class="col-start-2 row-start-1 justify-self-end">
-					<ButtonStyled size="small" type="transparent" circular>
+					<ButtonStyled v-if="dismissible" size="small" type="transparent" circular>
 						<button
 							type="button"
 							class="notification-toast-dismiss"
@@ -213,6 +213,7 @@ const props = withDefaults(
 		progressCurrent?: number
 		progressTotal?: number
 		actions?: PopupNotificationButton[]
+		dismissible?: boolean
 	}>(),
 	{
 		actionLoading: null,
@@ -224,6 +225,7 @@ const props = withDefaults(
 		showProgress: true,
 		wrapText: false,
 		progressType: 'percentage',
+		dismissible: true,
 	},
 )
 

packages/ui/src/providers/popup-notifications.ts

@@ -24,6 +24,7 @@ export interface PopupNotificationProgressItem {
 	progressType?: PopupNotificationProgressType
 	progressCurrent?: number
 	progressTotal?: number
+	dismissible?: boolean
 	onDismiss?: () => void | Promise<void>
 	buttons?: PopupNotificationButton[]
 }