ci: capture native recovery startup context before platform probe

tools/ci/MacOsNativeDiagnostic.cs

@@ -117,6 +117,8 @@ static class NativeDiagnostic
             AssertContainer(File.ReadAllText(Path.Combine(output, "container-created.stdout.log")), token);
             await Command("docker", ["start", id], output, "docker-start", deadline.Token);
             Console.WriteLine("The owned unprivileged TCG guest is starting. Success requires native macOS 14+/x86_64 and a writable 64-GiB disk; no installer will run.");
+            var recoveryStarted = Stopwatch.StartNew();
+            var heartbeat = Stopwatch.StartNew();
             while (true)
             {
                 deadline.Token.ThrowIfCancellationRequested();
@@ -132,6 +134,11 @@ static class NativeDiagnostic
                 }
                 var running = await Command("docker", ["inspect", "--format", "{{.State.Running}}", id], output, "container-running", deadline.Token);
                 if (running.Output.Trim() != "true") throw new InvalidOperationException("Guest container exited before a native readiness result.");
+                if (heartbeat.Elapsed >= TimeSpan.FromSeconds(60))
+                {
+                    Console.WriteLine($"[native-diagnostic] phase=recovery; elapsed={recoveryStarted.Elapsed.TotalMinutes:F1} minutes; container=running; readiness=pending");
+                    heartbeat.Restart();
+                }
                 await Task.Delay(TimeSpan.FromSeconds(20), deadline.Token);
             }
         }
@@ -144,7 +151,7 @@ static class NativeDiagnostic
         {
             Console.CancelKeyPress -= cancelHandler;
             using var captureDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(45));
-            try { await CaptureGuest(state.ContainerName, output, captureDeadline.Token); } catch (Exception exception) { Console.Error.WriteLine("Final evidence capture: " + exception.Message); }
+            try { await CaptureGuest(state.ContainerId ?? state.ContainerName, output, captureDeadline.Token, true, state.Token); } catch (Exception exception) { Console.Error.WriteLine("Final evidence capture: " + exception.Message); }
             try { PrintGuestProof(output, state.Token); } catch (Exception exception) { Console.Error.WriteLine("Native proof output: " + exception.Message); }
             var clean = await Cleanup(output);
             if (!clean) { outcome = "failed"; error = (error ?? "") + " Owned-resource cleanup failed; inspect cleanup evidence."; }
@@ -229,8 +236,9 @@ static class NativeDiagnostic
             throw new InvalidOperationException("Created container exceeds the owned/unprivileged diagnostic boundary.");
     }
 
-    static async Task CaptureGuest(string id, string output, CancellationToken cancellation)
+    static async Task CaptureGuest(string id, string output, CancellationToken cancellation, bool final = false, string? token = null)
     {
+        if (final && token is not null) await CaptureMonitor(id, output, token, cancellation);
         var logs = await Command("docker", ["logs", "--tail", "3000", id], output, "container", cancellation, requireSuccess: false);
         foreach (var file in new[] { ("proof.log", "guest-proof.log"), ("result.json", "guest-result.json") })
         {
@@ -240,6 +248,41 @@ static class NativeDiagnostic
         await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test ! -f /storage/14/setup.dmg || sha256sum /storage/14/setup.dmg; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false);
     }
 
+    static async Task CaptureMonitor(string id, string output, string token, CancellationToken cancellation)
+    {
+        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellation);
+        deadline.CancelAfter(TimeSpan.FromSeconds(10));
+        int? monitorExit = null, copyExit = null;
+        string? error = null;
+        try
+        {
+            if (!System.Text.RegularExpressions.Regex.IsMatch(id, "^[0-9a-f]{64}$") || !System.Text.RegularExpressions.Regex.IsMatch(token, "^[0-9a-f]{32}$")) throw new InvalidOperationException("No saved owned container identity for the optional monitor capture.");
+            var inspection = await Command("docker", ["inspect", id], output, "capture-monitor-container", deadline.Token);
+            AssertContainer(inspection.Output, token);
+            using (var document = JsonDocument.Parse(inspection.Output))
+                if (document.RootElement[0].GetProperty("Id").GetString() != id || !document.RootElement[0].GetProperty("State").GetProperty("Running").GetBoolean()) throw new InvalidOperationException("Owned guest container is no longer running for the optional monitor capture.");
+            var screen = "/dev/shm/native-diagnostic-screen-" + token + ".ppm";
+            var monitor = await Command("docker", ["exec", id, "sh", "-c", """
+                test -S /run/shm/monitor.sock || exit 1
+                rm -f -- "$1" || exit 1
+                printf 'info status\nscreendump %s\n' "$1" | /usr/bin/timeout -s KILL 5 /usr/bin/nc.openbsd -q 1 -w 2 -U /run/shm/monitor.sock
+                monitor_exit=$?
+                printf '\n[monitor-exit] %s\n' "$monitor_exit"
+                [ "$monitor_exit" -eq 0 ] || exit "$monitor_exit"
+                bytes=$(stat -c%s "$1") || exit 1
+                [ "$bytes" -gt 0 ] && [ "$bytes" -le 8388608 ] || exit 1
+                printf '[screen-bytes] %s\n' "$bytes"
+                """, "native-monitor", screen], output, "capture-monitor", deadline.Token, requireSuccess: false);
+            monitorExit = monitor.ExitCode;
+            if (monitorExit != 0) throw new InvalidOperationException("Optional monitor status/screenshot command exited " + monitorExit + ".");
+            var copy = await Command("docker", ["cp", id + ":" + screen, Path.Combine(output, "guest-screen-" + token + ".ppm")], output, "capture-monitor-screen", deadline.Token, requireSuccess: false);
+            copyExit = copy.ExitCode;
+            if (copyExit != 0) throw new InvalidOperationException("Optional monitor screenshot copy exited " + copyExit + ".");
+        }
+        catch (Exception exception) { error = exception.Message; Console.Error.WriteLine("Optional final monitor capture: " + error); }
+        finally { Save(Path.Combine(output, "monitor-capture.json"), new { token, monitorExit, copyExit, success = error is null, error, capturedUtc = DateTimeOffset.UtcNow }); }
+    }
+
     static async Task<bool> Cleanup(string output)
     {
         var path = Path.Combine(output, "owned-resources.json");

tools/ci/macos-native-readiness.sh

@@ -45,28 +45,59 @@ finish() {
 run_command() {
   local name="$1"
   shift
-  local process timer sleeper exit_code
+  local process timer sleeper exit_code started observer=""
   LAST_OUTPUT="/tmp/native-diagnostic-$name.out"
   printf '\n[proof-command] %s:' "$name" >> "$PROOF_LOG"
   printf ' %s' "$@" >> "$PROOF_LOG"
   printf '\n' >> "$PROOF_LOG"
+  started=$SECONDS
   "$@" > "$LAST_OUTPUT" 2>&1 &
   process=$!
+  printf '[proof-start] %s child=%s shell=%s parent=%s seconds=%s\n' "$name" "$process" "$$" "$PPID" "$started" >> "$PROOF_LOG"
   (
     trap 'kill "$sleeper" 2>/dev/null || :; exit 0' TERM INT
     sleep 45 &
     sleeper=$!
     wait "$sleeper"
+    printf '[proof-timeout] %s child=%s elapsed=%ss signal=TERM\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG"
     kill -TERM "$process" 2>/dev/null || :
-    sleep 2
+    sleep 2 & sleeper=$!; wait "$sleeper"
     kill -KILL "$process" 2>/dev/null || :
   ) &
   timer=$!
+  if [[ "$name" = platform || "$name" = platform-warm ]]; then
+    # Observers never extend the independent 45-second command deadline.
+    (
+      local sample_pid="" sample_timer="" pause_pid="" pause sample_exit
+      trap 'kill -KILL "$sample_pid" 2>/dev/null || :; kill -TERM "$sample_timer" "$pause_pid" 2>/dev/null || :; exit 0' TERM INT
+      for pause in 10 15; do
+        sleep "$pause" & pause_pid=$!; wait "$pause_pid"
+        printf '[proof-process] %s child=%s elapsed=%ss fields=pid,ppid,stat,cpu-time,elapsed,cpu-percent,wchan,comm\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG"
+        /bin/ps -p "$process" -o pid=,ppid=,stat=,time=,etime=,pcpu=,wchan=,comm= >> "$PROOF_LOG" 2>&1 &
+        sample_pid=$!
+        (
+          local sample_sleeper=""
+          trap 'kill "$sample_sleeper" 2>/dev/null || :; exit 0' TERM INT
+          sleep 5 & sample_sleeper=$!; wait "$sample_sleeper"
+          kill -KILL "$sample_pid" 2>/dev/null || :
+        ) & sample_timer=$!
+        wait "$sample_pid"; sample_exit=$?
+        kill -TERM "$sample_timer" 2>/dev/null || :; wait "$sample_timer" 2>/dev/null || :
+        printf '[proof-process-exit] %s %s\n' "$name" "$sample_exit" >> "$PROOF_LOG"
+        sample_pid=""; sample_timer=""; pause_pid=""
+      done
+    ) & observer=$!
+  fi
   wait "$process"
   exit_code=$?
   kill -TERM "$timer" 2>/dev/null || :
   wait "$timer" 2>/dev/null || :
+  if [ -n "$observer" ]; then
+    kill -TERM "$observer" 2>/dev/null || :
+    wait "$observer" 2>/dev/null || :
+  fi
   /usr/bin/tail -c 524288 "$LAST_OUTPUT" >> "$PROOF_LOG"
+  printf '\n[proof-duration] %s child=%s elapsed=%ss\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG"
   printf '\n[proof-exit] %s\n' "$exit_code" >> "$PROOF_LOG"
   LAST_EXIT="$exit_code"
   local size
@@ -75,13 +106,7 @@ run_command() {
   return 0
 }
 
-run_command platform /usr/bin/sw_vers
-(( LAST_EXIT == 0 )) || finish false sw_vers_failed
-run_command version /usr/bin/sw_vers -productVersion
-(( LAST_EXIT == 0 )) || finish false product_version_failed
-os_version=$(cat "$LAST_OUTPUT")
-[[ "$os_version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || finish false product_version_invalid
-(( ${os_version%%.*} >= 14 )) || finish false unsupported_macos_version
+# Collect cheap native identity/context before the first framework-dependent probe.
 run_command kernel /usr/bin/uname -a
 (( LAST_EXIT == 0 )) || finish false uname_failed
 run_command architecture /usr/bin/uname -m
@@ -94,14 +119,28 @@ run_command uid /usr/bin/id -u
 uid=$(cat "$LAST_OUTPUT")
 [ "$uid" = 0 ] || finish false recovery_account_not_root
 run_command bootargs /usr/sbin/sysctl kern.bootargs
+run_command cpu /usr/sbin/sysctl machdep.cpu.brand_string machdep.cpu.features machdep.cpu.leaf7_features
 run_command parent /bin/ps -p "$$" -p "$PPID" -o pid=,ppid=,comm=
 run_command processes /bin/ps -axo pid,ppid,comm
+run_command platform /usr/bin/sw_vers
+platform_exit="$LAST_EXIT"
 run_command system /bin/launchctl print system
 system_exit="$LAST_EXIT"
 run_command arbitration /bin/launchctl print system/com.apple.diskarbitrationd
 arbitration_exit="$LAST_EXIT"
 run_command recovery /bin/launchctl print system/com.apple.recoveryosd
 recovery_exit="$LAST_EXIT"
+if (( platform_exit != 0 )); then
+  printf '[proof-retry] sw_vers once after native service context; same 45-second deadline\n' >> "$PROOF_LOG"
+  run_command platform-warm /usr/bin/sw_vers
+  platform_exit="$LAST_EXIT"
+fi
+(( platform_exit == 0 )) || finish false sw_vers_failed
+run_command version /usr/bin/sw_vers -productVersion
+(( LAST_EXIT == 0 )) || finish false product_version_failed
+os_version=$(cat "$LAST_OUTPUT")
+[[ "$os_version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || finish false product_version_invalid
+(( ${os_version%%.*} >= 14 )) || finish false unsupported_macos_version
 
 # Bound readiness independently of the host's 40-minute overall deadline.
 readiness_start=$SECONDS