system: keep the process heap lock unowned across close-time guest thread suspension and make the forced-exit watchdog heap-free

src/system/kernel_state.cpp

@@ -12,6 +12,11 @@
 #include <cstring>
 #include <string>
 
+#include <rex/platform.h>
+#if REX_PLATFORM_WIN32
+#include <windows.h>
+#endif
+
 #include <fmt/format.h>
 
 #include <rex/assert.h>
@@ -863,6 +868,21 @@ void KernelState::TerminateTitle() {
   global_lock.lock();
 #else
   // Suspend all running guest threads so they stop touching shared state.
+  //
+  // Guest threads allocate on the process heap constantly (kernel calls,
+  // capture hooks), and SuspendThread stops a thread at an arbitrary
+  // instruction: without protection one can be caught inside the allocator
+  // OWNING the heap lock, and the thread-object frees below then deadlock
+  // the close forever (observed: UI thread parked in RtlpFreeHeap while the
+  // suspended guest render thread sat in RtlpAllocateHeap). Holding the heap
+  // lock across the suspension makes that state unreachable: a thread inside
+  // the allocator finishes before HeapLock returns, and later entrants block
+  // without owning the lock. The lock is reentrant for this thread, so the
+  // bookkeeping below may still allocate.
+#if REX_PLATFORM_WIN32
+  HANDLE process_heap = GetProcessHeap();
+  HeapLock(process_heap);
+#endif
   std::vector<XThread*> suspended_threads;
   for (auto it = threads_by_id_.begin(); it != threads_by_id_.end(); ++it) {
     if (!XThread::IsInThread(it->second) && it->second->is_guest_thread() &&
@@ -871,6 +891,9 @@ void KernelState::TerminateTitle() {
       suspended_threads.push_back(it->second);
     }
   }
+#if REX_PLATFORM_WIN32
+  HeapUnlock(process_heap);
+#endif
 
   // Terminate each suspended thread. Must drop the lock since Terminate waits.
   global_lock.unlock();

src/ui/rex_app.cpp

@@ -47,13 +47,21 @@
 #include <sys/utsname.h>
 #endif
 
+#if REX_PLATFORM_WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
 #include <fmt/format.h>
 #include <imgui.h>
 #include <toml++/toml.hpp>
 
 #include <algorithm>
 #include <chrono>
+#include <cstdio>
 #include <cstdlib>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
 #include <optional>
@@ -206,8 +214,30 @@ void LogLinuxRuntimeDiagnostics() {
 void StartForcedExitWatchdog(const char* reason) {
   std::thread([reason]() {
     std::this_thread::sleep_for(std::chrono::seconds(10));
-    REXLOG_WARN("{} watchdog exiting process after shutdown timeout", reason);
+    // The teardown this guards can wedge with the process heap lock orphaned
+    // (a guest thread suspended or terminated mid-allocation), so the kill
+    // path must not allocate or run lock-taking shutdown: logging and
+    // std::_Exit are both off-limits on Windows (ExitProcess acquires the
+    // heap lock inside RtlExitUserProcess, deadlocking the watchdog itself).
+    // TerminateProcess skips user-mode cleanup entirely; on POSIX std::_Exit
+    // is a plain exit_group with no lock use. The marker is emitted through
+    // heap-free primitives only.
+    char msg[160];
+    std::snprintf(msg, sizeof(msg), "%s watchdog terminating process after shutdown timeout\n",
+                  reason);
+#if REX_PLATFORM_WIN32
+    OutputDebugStringA(msg);
+    HANDLE log_handle = GetStdHandle(STD_ERROR_HANDLE);
+    if (log_handle != INVALID_HANDLE_VALUE && log_handle != nullptr) {
+      DWORD written = 0;
+      WriteFile(log_handle, msg, DWORD(std::strlen(msg)), &written, nullptr);
+    }
+    TerminateProcess(GetCurrentProcess(), EXIT_SUCCESS);
+#else
+    ssize_t written = write(STDERR_FILENO, msg, std::strlen(msg));
+    (void)written;
     std::_Exit(EXIT_SUCCESS);
+#endif
   }).detach();
 }