render: recover streaming-revoked guest read faults on linux and macos

src/native/skate3_native_guest_read.cpp

@@ -13,7 +13,11 @@
 #else
 #include <pthread.h>
 #include <signal.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <unistd.h>
 
+#include <atomic>
 #include <csetjmp>
 #include <mutex>
 
@@ -112,6 +116,108 @@ bool GuestTryCopy(void* dst, const void* src, size_t size) {
   tl_in_guest_copy = false;
   return true;
 }
+
+// ---- Raw-load read-fault recovery (see the header contract) ---------------
+// The handler must be async-signal-safe: TLS reads, atomics and the
+// mprotect/clock_gettime syscalls only. It resumes the interrupted load in
+// place (registers untouched, pc unchanged), so unlike the siglongjmp guard
+// above it never crosses C++ scopes and imposes no constraints on the armed
+// code. Recovery restores READ access only: a page the runtime keeps
+// write-protected (write watches) still traps guest writes exactly as
+// before.
+namespace {
+
+// Full span of the runtime's guest views measured from the virtual membase:
+// 4 GB of virtual address space plus the 512 MB physical-raw view mapped
+// directly above it (the SDK maps both from one backing object).
+constexpr uint64_t kGuestSpanBytes = 0x120000000ull;
+
+thread_local int tl_read_recovery_depth = 0;
+thread_local uint8_t* tl_read_recovery_base = nullptr;
+// Livelock breaker: a read fault that RECURS on the same page immediately
+// after a successful recovery means read access cannot actually be restored
+// there (e.g. SIGBUS from the backing object rather than page protection).
+// Distinct pages, and the same page faulting again later (streaming re-
+// revoked it), reset the streak.
+thread_local uintptr_t tl_read_recovery_page = 0;
+thread_local uint32_t tl_read_recovery_streak = 0;
+thread_local uint64_t tl_read_recovery_last_ns = 0;
+
+std::atomic<uint64_t> g_read_recoveries{0};
+size_t g_host_page_size = 4096;
+
+bool GuestReadFaultHandler(rex::arch::Exception* ex, void* /*data*/) {
+  if (tl_read_recovery_depth <= 0 || tl_in_guest_copy ||
+      ex->code() != rex::arch::Exception::Code::kAccessViolation ||
+      ex->access_violation_operation() !=
+          rex::arch::Exception::AccessViolationOperation::kRead) {
+    return false;  // not ours - keep dispatching down the chain
+  }
+  uint8_t* base = tl_read_recovery_base;
+  const uint64_t fault = ex->fault_address();
+  const uint64_t lo = uint64_t(reinterpret_cast<uintptr_t>(base));
+  if (base == nullptr || fault < lo || fault - lo >= kGuestSpanBytes) {
+    return false;
+  }
+  const uintptr_t page =
+      uintptr_t(fault) & ~(uintptr_t(g_host_page_size) - 1);
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  const uint64_t now_ns = uint64_t(ts.tv_sec) * 1000000000ull + uint64_t(ts.tv_nsec);
+  if (page == tl_read_recovery_page &&
+      now_ns - tl_read_recovery_last_ns < 10000000ull) {
+    if (++tl_read_recovery_streak > 16) {
+      return false;  // recovery is not sticking on this page: die loudly
+    }
+  } else {
+    tl_read_recovery_page = page;
+    tl_read_recovery_streak = 0;
+  }
+  tl_read_recovery_last_ns = now_ns;
+  if (mprotect(reinterpret_cast<void*>(page), g_host_page_size, PROT_READ) != 0) {
+    return false;
+  }
+  g_read_recoveries.fetch_add(1, std::memory_order_relaxed);
+  return true;  // resume: the faulted load retries against the readable page
+}
+
+void EnsureGuestReadRecoveryHandler() {
+  static std::once_flag once;
+  std::call_once(once, [] {
+    const long ps = sysconf(_SC_PAGESIZE);
+    if (ps > 0) {
+      g_host_page_size = size_t(ps);
+    }
+    rex::arch::ExceptionHandler::Install(GuestReadFaultHandler, nullptr);
+  });
+}
+
+}  // namespace
+
+GuestReadRecoveryScope::GuestReadRecoveryScope(uint8_t* base) {
+  EnsureGuestReadRecoveryHandler();
+  if (tl_read_recovery_depth++ == 0) {
+    tl_read_recovery_base = base;
+  }
+}
+
+GuestReadRecoveryScope::~GuestReadRecoveryScope() {
+  if (--tl_read_recovery_depth == 0) {
+    tl_read_recovery_base = nullptr;
+  }
+}
+
+void ArmGuestReadRecoveryForThread(uint8_t* base) {
+  if (tl_read_recovery_depth == 0) {
+    EnsureGuestReadRecoveryHandler();
+    tl_read_recovery_depth = 1;
+    tl_read_recovery_base = base;
+  }
+}
+
+uint64_t GuestReadRecoveryCount() {
+  return g_read_recoveries.load(std::memory_order_relaxed);
+}
 #endif
 
 float GuestHalfToFloat(uint16_t h) {

src/native/skate3_native_guest_read.h

@@ -35,6 +35,52 @@ struct SkinSampleVert {
 // open-coded __try{memcpy}__except compiles to an UNPROTECTED `jmp memcpy`.
 bool GuestTryCopy(void* dst, const void* src, size_t size);
 
+// Read-fault recovery for the renderer's RAW guest loads (the REX_LOAD-style
+// structure walks that do not route through GuestTryCopy). World streaming
+// can revoke a guest range between a pointer being captured and the renderer
+// dereferencing it; on POSIX that read is a fatal SIGSEGV/SIGBUS. While a
+// thread is armed, a handler on the runtime exception chain recovers a READ
+// fault inside the guest mapping by restoring read access to the faulted
+// host page and resuming the interrupted load, which then completes with
+// resident-but-meaningless bytes that the callers' existing plausibility
+// gates reject (the same "not ready, skip this item" outcome the guarded
+// copy produces). The handler resumes execution in place - no siglongjmp -
+// so arming carries NO destructor/unwind constraints and is safe at any
+// scope width, including around code that owns locks, vectors or strings.
+// Faults inside GuestTryCopy keep that function's fail-the-copy semantics
+// (the recovery handler declines them); write faults and faults outside the
+// guest mapping are never recovered. Windows builds compile these to no-ops
+// (raw reads there are served by the pagefile-backed guest mapping).
+//
+// GuestReadRecoveryScope: arm for one native-scene entry point on a GUEST
+// thread (capture hooks, frame build). Guest threads run recompiled game
+// code the rest of the time, and a genuine wild read there must still crash
+// loudly, hence the narrow scope. Nestable.
+//
+// ArmGuestReadRecoveryForThread: permanently arm a thread that only ever
+// reads guest memory on the renderer's behalf (the render thread, the
+// prewarm decode workers). Idempotent and cheap; call per frame/iteration.
+#if defined(_WIN32)
+class GuestReadRecoveryScope {
+ public:
+  explicit GuestReadRecoveryScope(uint8_t* /*base*/) {}
+};
+inline void ArmGuestReadRecoveryForThread(uint8_t* /*base*/) {}
+inline uint64_t GuestReadRecoveryCount() { return 0; }
+#else
+class GuestReadRecoveryScope {
+ public:
+  explicit GuestReadRecoveryScope(uint8_t* base);
+  ~GuestReadRecoveryScope();
+  GuestReadRecoveryScope(const GuestReadRecoveryScope&) = delete;
+  GuestReadRecoveryScope& operator=(const GuestReadRecoveryScope&) = delete;
+};
+void ArmGuestReadRecoveryForThread(uint8_t* base);
+// Total recovered reads since startup (telemetry; logged by the render
+// thread's frame stats when non-zero).
+uint64_t GuestReadRecoveryCount();
+#endif
+
 float GuestHalfToFloat(uint16_t h);
 
 // Bind-pose bbox diagonal of the item: the reference size every

src/skate3_native_render.cpp

@@ -2,6 +2,7 @@
 
 #include "native/skate3_native_diag.h"
 #include "native/skate3_native_entity.h"
+#include "native/skate3_native_guest_read.h"
 #include "native/skate3_native_lw.h"
 #include "native/skate3_native_palette.h"
 #include "skate3_native_scene.h"
@@ -104,6 +105,7 @@ void OnSceneDrawList(uint8_t* base, uint32_t view, uint32_t sort_vec, uint32_t f
   if (count == 0 || count > 100000) {
     return;
   }
+  skate3::native_scene::GuestReadRecoveryScope guest_read_recovery(base);
   const uint32_t entries = REX_LOAD_U32(sort_vec);
   if (entries == 0) {
     return;
@@ -169,6 +171,7 @@ uint32_t FilterSceneDrawList(uint8_t* base, uint32_t sort_vec, uint32_t first,
   if (f.ctxs.empty()) {
     return count;
   }
+  skate3::native_scene::GuestReadRecoveryScope guest_read_recovery(base);
   const uint32_t entries = REX_LOAD_U32(sort_vec);
   if (entries == 0) {
     return count;
@@ -742,6 +745,7 @@ extern "C" REX_FUNC(sub_827F0D00) {
   if (!(max_expo > 0.0f) || max_expo > 16.0f) {
     return;
   }
+  skate3::native_scene::GuestReadRecoveryScope guest_read_recovery(base);
   uint32_t prev_bits = REX_LOAD_U32(self + 3240);
   float prev;
   std::memcpy(&prev, &prev_bits, sizeof(prev));

src/skate3_native_scene.cpp

@@ -3356,6 +3356,7 @@ void OnSetViewport(uint8_t* base, uint32_t viewport_ptr) {
   if (viewport_ptr == 0) {
     return;
   }
+  GuestReadRecoveryScope guest_read_recovery(base);
   for (int i = 0; i < 6; ++i) {
     g_cur_viewport[i].store(REX_LOAD_U32(viewport_ptr + i * 4), std::memory_order_relaxed);
   }
@@ -3365,6 +3366,7 @@ void OnSetScissor(uint8_t* base, uint32_t rect_ptr) {
   if (rect_ptr == 0) {
     return;
   }
+  GuestReadRecoveryScope guest_read_recovery(base);
   for (int i = 0; i < 4; ++i) {
     g_cur_scissor[i].store(REX_LOAD_U32(rect_ptr + i * 4), std::memory_order_relaxed);
   }
@@ -4129,6 +4131,10 @@ uint32_t CaptureDynamicState(uint8_t* base, uint32_t ctx, bool world_path,
   if (bank == 0) {
     return 0;
   }
+  // Streaming can revoke any pointer in the ctx/record/mesh chain between
+  // the game queueing the draw and this walk; recover raw-load read faults
+  // for the whole capture (POSIX; no-op on Windows).
+  GuestReadRecoveryScope guest_read_recovery(base);
   // Items drawn inside an AUX perspective pass (skater-portrait RTTs) never
   // enter the frame at all: they share (ib,vb) buffers with the on-screen
   // player, so letting them sit PENDING lets them steal the player's own
@@ -4373,6 +4379,7 @@ uint32_t CaptureClothDraw(uint8_t* base, uint32_t r4, uint32_t r5, uint32_t r6,
   if (!GuestReadableApprox(base, vb_obj)) {
     return 0;
   }
+  GuestReadRecoveryScope guest_read_recovery(base);
   const uint32_t addr = REX_LOAD_U32(vb_obj + 0x18) & 0xFFFFFFFC;
   const uint32_t size = REX_LOAD_U32(vb_obj + 0x20);
   constexpr uint32_t kStride = 24;
@@ -4512,6 +4519,9 @@ void OnDrawDone(uint8_t* base, uint32_t func, uint32_t r4, uint32_t r5, uint32_t
   if (bank == 0) {
     return;
   }
+  // The bank / PS-bank / pending-fixup reads below are raw loads; recover
+  // read faults for the whole post-draw path (POSIX; no-op on Windows).
+  GuestReadRecoveryScope guest_read_recovery(base);
   // Fog rows: grab c5/c6 from the first 3D draw whose bank c4 row matches
   // the last built scene's camera (main-pass layout; tolerant of one frame
   // of camera motion). See g_fog_rows. The same camera-keyed draws also
@@ -8286,6 +8296,10 @@ void BuildFrameScene(uint8_t* base, const SubmitRecord* records, size_t count) {
   if (!SceneEnabled()) {
     return;
   }
+  // The frame-end walks below chase captured pointers whose ranges world
+  // streaming may have revoked during the frame; recover raw-load read
+  // faults for the whole build (POSIX; no-op on Windows).
+  GuestReadRecoveryScope guest_read_recovery(base);
   // Published every frame (not just on world submissions): boot/menu frames
   // carry only 2D, and the render thread's 2D texture decodes need the
   // guest base from the very first natively rendered boot frame.

src/skate3_native_scene_gpu.cpp

@@ -4706,6 +4706,10 @@ void PrewarmWorkerLoop() {
       std::this_thread::sleep_for(std::chrono::milliseconds(100));
       continue;
     }
+    // Decode workers only ever read guest memory on the renderer's behalf:
+    // keep them permanently armed for raw-load read-fault recovery (POSIX;
+    // no-op on Windows).
+    ArmGuestReadRecoveryForThread(base);
     if (have_dyn) {
       PrewarmResult res;
       res.mesh_valid =
@@ -7022,6 +7026,22 @@ void RenderOutlineComposite(const NativeGuestOutputRenderContext& context,
 void LogFrameStats(const FrameScene& scene, uint64_t frames, uint32_t drawn,
                    uint32_t drawn_2d, uint32_t drawn_spline, bool shadow_ready,
                    uint32_t shadow_draws) {
+  // Raw guest-load read-fault recoveries (POSIX only; always 0 on Windows).
+  // Each one is a streaming race that revoked a captured range mid-walk and
+  // would otherwise be a fatal SIGSEGV; a slowly growing count during heavy
+  // world streaming is expected and benign.
+  {
+    static uint64_t s_recov_reported = 0;
+    static uint64_t s_recov_frame = 0;
+    const uint64_t recov = GuestReadRecoveryCount();
+    if (recov != s_recov_reported &&
+        (s_recov_frame == 0 || frames - s_recov_frame >= 600)) {
+      REXLOG_INFO("native-scene: guest read faults recovered total={} (+{})",
+                  recov, recov - s_recov_reported);
+      s_recov_reported = recov;
+      s_recov_frame = frames;
+    }
+  }
   const uint64_t interval = uint64_t(
       std::max(60, REXCVAR_GET(skate3_native_render_scene_perf_interval)));
   if (frames % interval == 0 && REXCVAR_GET(skate3_native_render_scene_perf_log)) {
@@ -7532,6 +7552,11 @@ bool RenderScene(const NativeGuestOutputRenderContext& context, void* /*user_dat
   // flow renders yielded-emulated or native).
   uint8_t* base = g_guest_base.load(std::memory_order_relaxed);
   if (base != nullptr) {
+    // The render thread reads guest memory for the rest of the frame (mesh
+    // decode, texture fingerprints, dynamic-item interpolation) and runs no
+    // guest code of its own: keep it permanently armed for raw-load
+    // read-fault recovery (POSIX; no-op on Windows).
+    ArmGuestReadRecoveryForThread(base);
     UpdatePhotoGrabWindow(base);
   }
   if (YieldForMenus(context)) {