diagnose native Recovery disk IPC with bounded observation

.gitea/workflows/pr-push-build-and-test.yaml

@@ -3,6 +3,10 @@ name: PR and Push Build/Test
 on:
   pull_request:
   push:
+    # This temporary branch changes only the native prerequisite diagnostic.
+    # Its manual workflow provides that evidence; the PR branch still runs all jobs.
+    branches-ignore:
+      - codex/macos-ci-kvm-compatibility
   workflow_dispatch:
 
 jobs:

docs/macos-native-diagnostic.md

@@ -57,7 +57,11 @@ No new force/beta argument is needed for actual no-AVX2 CPUs. Baseline arguments
 
 The Apple wrapper is byte-identical to baseline: background `/Volumes/installstate/readiness.sh` then `exec /usr/libexec/recoveryosd` under the same launchd job/PID. Source evidence does not prove Apple's executable ran.
 
-Readiness changes only minimum macOS 14 to 13. Validation normalizes that gate to 14 and requires baseline SHA256 `4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3`. Architecture, UID, services, disk size/writability/uniqueness, retries, proof bounds, timers and native-wait/cleanup/flush metrics remain identical. Limits stay 45 seconds per native command, 180 seconds for UID, ten minutes disk readiness, 40 minutes host and 45 minutes workflow.
+The disk IPC continuation adds six explicitly marked observation blocks and limits disk enumeration to two attempts. Validation removes only those marked blocks, restores the former attempt condition and normalizes macOS 13 to 14 before requiring baseline SHA256 `4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3`. The receipt explicitly records these exclusions and the two-attempt limit. Architecture, UID, native service exits, disk size/writability/uniqueness, proof bounds, existing command watchdogs and native-wait/cleanup/flush metrics remain identical. Limits stay 45 seconds per required native command, 180 seconds for UID, ten minutes maximum disk readiness, 40 minutes host and 45 minutes workflow.
+
+Run 4175 at `94a70b200508d3ba295124896d923fbb785d1658` reached macOS 13.6, x86_64 and UID 0 with KVM enabled; its nine `diskutil list physical` attempts timed out. This identifies a disk-readiness failure without proving whether SATA/IOMedia, service IPC or the probe context is responsible. Before the first attempt the continuation records bounded `launchctl print` output for `com.apple.diskarbitrationd` and `com.apple.diskmanagementd`, plus `ioreg -r -c IOMedia -l -w 0`. During that first owned diskutil process it captures process state, optionally runs `/usr/bin/sample <owned-child-pid> 3 10 -file <owned-output>`, then observes both service jobs again. The separate sample report is flushed into the proof alongside command output. Each optional observer command has an eight-second watchdog plus the existing two-second TERM/KILL grace. Missing sample tooling or an already completed diskutil is reported explicitly; nonzero observation exits are logged and cannot satisfy any native gate.
+
+The observer owns its command/timer PIDs and is stopped when the disk query completes or the probe is canceled. Its output enters the existing 512-KiB per-output and 4-MiB proof budgets. The hook only reads media/service/process state and writes its existing diagnostic files: it does not load, restart, erase or modify any service or disk. Bash remains necessary because Apple Recovery runs this hook before a .NET SDK is installed. The CPU, Recovery, QEMU, Apple wrapper and container profile are unchanged. These observations are prepared diagnostics, not a new successful guest or full native CI receipt.
 
 Container profile: `KVM=Y`, `CPU_MODEL=host`, `VERSION=13`, 4-GiB guest, two guest/host CPUs, 6-GiB memory/swap and 512-MiB shared memory. Fresh anonymous `/storage` holds the 64-GiB disk; evidence reads `/storage/13/setup.dmg`. Existing resource budget checks remain.
 

tools/ci/MacOsNativeDiagnostic.cs

@@ -69,7 +69,7 @@ static class NativeDiagnostic
                 await PrepareSource(Path.GetFullPath(source), output, "validation", false, Option(args, "--cryptex-archive"), CancellationToken.None);
                 await ValidateResourceRetention(output);
                 await ValidateRecoveryPatch(output);
-                Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
+                Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, readinessDiagnosticBlocksExcluded = 6, diskReadinessAttemptLimit = 2, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
             }
             Console.WriteLine("Source patch contracts and diagnostic result validation passed; no Docker or guest execution occurred.");
             return 0;
@@ -184,9 +184,11 @@ static class NativeDiagnostic
     static void ValidateContracts()
     {
         var readiness = File.ReadAllText(Path.Combine("tools", "ci", "macos-native-readiness.sh"));
-        var baseline = ReplaceOnce(readiness, "(( ${os_version%%.*} >= 13 ))", "(( ${os_version%%.*} >= 14 ))");
+        var baseline = NormalizeReadinessDiagnostics(readiness);
+        baseline = ReplaceOnce(baseline, "(( ${os_version%%.*} >= 13 ))", "(( ${os_version%%.*} >= 14 ))");
+        baseline = ReplaceOnce(baseline, "while (( attempt < 2 && SECONDS - readiness_start < 600 )); do", "while (( SECONDS - readiness_start < 600 )); do");
         if (Hash(Encoding.UTF8.GetBytes(baseline)) != "4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3")
-            throw new InvalidOperationException("Compatibility readiness may change only the baseline's macOS minimum to 13; identity, services, disk and limits must remain identical.");
+            throw new InvalidOperationException("Outside six explicit diagnostic blocks, the macOS minimum and two-attempt limit, baseline identity/service/disk gates and watchdogs must remain identical.");
         if (Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-bootstrap.sh"))) != "94f069e116fdc7685a4d233cab6fa50df9f39274386bb82157674061e74fadb5")
             throw new InvalidOperationException("Compatibility profile must preserve the baseline Apple recoveryosd wrapper.");
         if (Hash(Encoding.UTF8.GetBytes(OriginalDaemon13)) != "af9d7f6c1948079bd4384d27b6882678d6fb4e338fcf6a8be8f84fceef174ad6") throw new InvalidOperationException("macOS 13 allowlist bytes differ from the independently read comparison plist.");
@@ -214,6 +216,24 @@ static class NativeDiagnostic
         }
     }
 
+    static string NormalizeReadinessDiagnostics(string source)
+    {
+        const string start = "# BEGIN disk IPC diagnostic\n";
+        const string end = "# END disk IPC diagnostic\n";
+        var blocks = 0;
+        while (source.IndexOf(start, StringComparison.Ordinal) is var from && from >= 0)
+        {
+            var to = source.IndexOf(end, from + start.Length, StringComparison.Ordinal);
+            if (to < 0 || source.IndexOf(start, from + start.Length, to - from - start.Length, StringComparison.Ordinal) >= 0)
+                throw new InvalidOperationException("Readiness diagnostic blocks are unbalanced or nested.");
+            source = source.Remove(from, to + end.Length - from);
+            blocks++;
+        }
+        if (blocks != 6 || source.Contains(end, StringComparison.Ordinal))
+            throw new InvalidOperationException("Readiness must contain exactly six explicit disk IPC diagnostic blocks.");
+        return source;
+    }
+
     static async Task PrepareSource(string source, string output, string token, bool writeSource, string? cryptexArchive, CancellationToken cancellation)
     {
         Directory.CreateDirectory(output);

tools/ci/macos-native-readiness.sh

@@ -14,6 +14,9 @@ TIMER_FIFO="/tmp/native-diagnostic-$PROOF_TOKEN-$$.fifo"
 PENDING_OUTPUTS=()
 ACTIVE_COMMAND=""
 ACTIVE_TIMER=""
+# BEGIN disk IPC diagnostic
+ACTIVE_OBSERVER=""
+# END disk IPC diagnostic
 os_version=""
 architecture=""
 uid=-1
@@ -96,8 +99,75 @@ read_scalar() {
   SCALAR="$value"
 }
 
+# BEGIN disk IPC diagnostic
+# Optional observations have their own child/timer ownership. No service is
+# loaded, restarted or changed, and samples target only this probe's diskutil.
+observe_disk_query() {
+  local disk_process="$1" output="$2" sample_output="$3" observation_child="" observation_timer=""
+  cancel_observation() {
+    trap '' TERM INT
+    if [ -n "$observation_child" ]; then
+      kill -TERM "$observation_child" 2>/dev/null || :
+      IFS= read -r -t 2 -u 9 unused || :
+      kill -KILL "$observation_child" 2>/dev/null || :
+      wait "$observation_child" 2>/dev/null || :
+    fi
+    [ -z "$observation_timer" ] || { kill -TERM "$observation_timer" 2>/dev/null || :; wait "$observation_timer" 2>/dev/null || :; }
+    printf '[disk-observation] stopped after the owned disk query\n' >> "$output"
+    exit 143
+  }
+  observe_command() {
+    local name="$1" status started=$SECONDS
+    shift
+    printf '\n[disk-observation-command] %s:' "$name" >> "$output"
+    printf ' %s' "$@" >> "$output"
+    printf '\n' >> "$output"
+    "$@" >> "$output" 2>&1 &
+    observation_child=$!
+    (
+      trap 'exit 0' TERM INT
+      IFS= read -r -t 8 -u 9 unused || :
+      printf '[disk-observation-timeout] %s child=%s limit=8s\n' "$name" "$observation_child" >> "$output"
+      kill -TERM "$observation_child" 2>/dev/null || :
+      IFS= read -r -t 2 -u 9 unused || :
+      kill -KILL "$observation_child" 2>/dev/null || :
+    ) &
+    observation_timer=$!
+    wait "$observation_child"; status=$?
+    kill -TERM "$observation_timer" 2>/dev/null || :
+    wait "$observation_timer" 2>/dev/null || :
+    printf '[disk-observation-exit] %s status=%s elapsed=%ss\n' "$name" "$status" "$((SECONDS - started))" >> "$output"
+    observation_child=""; observation_timer=""
+  }
+  trap cancel_observation TERM INT
+  printf '[disk-observation] owned-diskutil-child=%s parent-shell=%s\n' "$disk_process" "$$" >> "$output"
+  observe_command processes /bin/ps -axo pid,ppid,state,comm
+  if [ -x /usr/bin/sample ]; then
+    if kill -0 "$disk_process" 2>/dev/null; then
+      observe_command diskutil-sample /usr/bin/sample "$disk_process" 3 10 -file "$sample_output"
+    else
+      printf '[disk-observation-unavailable] diskutil already exited before sample\n' >> "$output"
+    fi
+  else
+    printf '[disk-observation-unavailable] /usr/bin/sample is unavailable\n' >> "$output"
+  fi
+  observe_command arbitration /bin/launchctl print system/com.apple.diskarbitrationd
+  observe_command management /bin/launchctl print system/com.apple.diskmanagementd
+}
+
+stop_disk_observation() {
+  [ -n "$ACTIVE_OBSERVER" ] || return 0
+  kill -TERM "$ACTIVE_OBSERVER" 2>/dev/null || :
+  wait "$ACTIVE_OBSERVER" 2>/dev/null || :
+  ACTIVE_OBSERVER=""
+}
+
+# END disk IPC diagnostic
 cancel_probe() {
   trap '' TERM INT
+# BEGIN disk IPC diagnostic
+  stop_disk_observation
+# END disk IPC diagnostic
   if [ -n "$ACTIVE_COMMAND" ]; then
     kill -TERM "$ACTIVE_COMMAND" 2>/dev/null || :
     IFS= read -r -t 2 -u 9 unused || :
@@ -136,6 +206,18 @@ run_command() {
   ) &
   timer=$!
   ACTIVE_TIMER="$timer"
+# BEGIN disk IPC diagnostic
+  if [[ "$name" == disks && "$attempt" == 1 ]]; then
+    local observation_output="/tmp/native-diagnostic-disk-observation.out"
+    local sample_output="/tmp/native-diagnostic-disk-sample.out"
+    : > "$observation_output"
+    : > "$sample_output"
+    observe_disk_query "$process" "$observation_output" "$sample_output" &
+    ACTIVE_OBSERVER=$!
+    printf '[disk-observation-start] observer=%s owned-diskutil-child=%s\n' "$ACTIVE_OBSERVER" "$process" >&3
+    PENDING_OUTPUTS+=("$observation_output" "$sample_output")
+  fi
+# END disk IPC diagnostic
   wait "$process"
   exit_code=$?
   waited=$SECONDS
@@ -143,6 +225,9 @@ run_command() {
   printf '[proof-native-wait] %s child=%s elapsed=%ss exit=%s\n' "$name" "$process" "$((waited - started))" "$exit_code" >&3
   kill -TERM "$timer" 2>/dev/null || :
   wait "$timer" 2>/dev/null || :
+# BEGIN disk IPC diagnostic
+  stop_disk_observation
+# END disk IPC diagnostic
   ACTIVE_COMMAND=""; ACTIVE_TIMER=""
   printf '[proof-cleanup] %s child=%s elapsed=%ss total=%ss\n' "$name" "$process" "$((SECONDS - waited))" "$((SECONDS - started))" >&3
   printf '[proof-exit] %s\n' "$exit_code" >&3
@@ -203,10 +288,17 @@ os_version="$SCALAR"
 (( ${os_version%%.*} >= 13 )) || fail_probe unsupported_macos_version
 flush_outputs || finish false diagnostic_log_budget_exceeded
 
+# BEGIN disk IPC diagnostic
+run_command arbitration_before /bin/launchctl print system/com.apple.diskarbitrationd
+run_command management_before /bin/launchctl print system/com.apple.diskmanagementd
+run_command media_before /usr/sbin/ioreg -r -c IOMedia -l -w 0
+flush_outputs || finish false diagnostic_log_budget_exceeded
+
+# END disk IPC diagnostic
 # Bound readiness independently of the host's 40-minute overall deadline.
 readiness_start=$SECONDS
 attempt=0
-while (( SECONDS - readiness_start < 600 )); do
+while (( attempt < 2 && SECONDS - readiness_start < 600 )); do
   attempt=$((attempt + 1))
   printf '\n[readiness-attempt] %s\n' "$attempt" >&3
   run_command disks /usr/sbin/diskutil list physical