From 25989cf0eb7205f30ac0bb44eb279aa3b463f12c Mon Sep 17 00:00:00 2001 From: dh Date: Sun, 4 Oct 2026 08:41:06 +0200 Subject: [PATCH] diagnose native Recovery disk IPC with bounded observation --- .gitea/workflows/pr-push-build-and-test.yaml | 4 + docs/macos-native-diagnostic.md | 6 +- tools/ci/MacOsNativeDiagnostic.cs | 26 +++++- tools/ci/macos-native-readiness.sh | 94 +++++++++++++++++++- 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/pr-push-build-and-test.yaml b/.gitea/workflows/pr-push-build-and-test.yaml index 789a0c0..5e93b7d 100644 --- a/.gitea/workflows/pr-push-build-and-test.yaml +++ b/.gitea/workflows/pr-push-build-and-test.yaml @@ -3,6 +3,10 @@ name: PR and Push Build/Test on: pull_request: push: + # This temporary branch changes only the native prerequisite diagnostic. + # Its manual workflow provides that evidence; the PR branch still runs all jobs. + branches-ignore: + - codex/macos-ci-kvm-compatibility workflow_dispatch: jobs: diff --git a/docs/macos-native-diagnostic.md b/docs/macos-native-diagnostic.md index 51273bc..7069d25 100644 --- a/docs/macos-native-diagnostic.md +++ b/docs/macos-native-diagnostic.md @@ -57,7 +57,11 @@ No new force/beta argument is needed for actual no-AVX2 CPUs. Baseline arguments The Apple wrapper is byte-identical to baseline: background `/Volumes/installstate/readiness.sh` then `exec /usr/libexec/recoveryosd` under the same launchd job/PID. Source evidence does not prove Apple's executable ran. -Readiness changes only minimum macOS 14 to 13. Validation normalizes that gate to 14 and requires baseline SHA256 `4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3`. Architecture, UID, services, disk size/writability/uniqueness, retries, proof bounds, timers and native-wait/cleanup/flush metrics remain identical. Limits stay 45 seconds per native command, 180 seconds for UID, ten minutes disk readiness, 40 minutes host and 45 minutes workflow. +The disk IPC continuation adds six explicitly marked observation blocks and limits disk enumeration to two attempts. Validation removes only those marked blocks, restores the former attempt condition and normalizes macOS 13 to 14 before requiring baseline SHA256 `4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3`. The receipt explicitly records these exclusions and the two-attempt limit. Architecture, UID, native service exits, disk size/writability/uniqueness, proof bounds, existing command watchdogs and native-wait/cleanup/flush metrics remain identical. Limits stay 45 seconds per required native command, 180 seconds for UID, ten minutes maximum disk readiness, 40 minutes host and 45 minutes workflow. + +Run 4175 at `94a70b200508d3ba295124896d923fbb785d1658` reached macOS 13.6, x86_64 and UID 0 with KVM enabled; its nine `diskutil list physical` attempts timed out. This identifies a disk-readiness failure without proving whether SATA/IOMedia, service IPC or the probe context is responsible. Before the first attempt the continuation records bounded `launchctl print` output for `com.apple.diskarbitrationd` and `com.apple.diskmanagementd`, plus `ioreg -r -c IOMedia -l -w 0`. During that first owned diskutil process it captures process state, optionally runs `/usr/bin/sample 3 10 -file `, then observes both service jobs again. The separate sample report is flushed into the proof alongside command output. Each optional observer command has an eight-second watchdog plus the existing two-second TERM/KILL grace. Missing sample tooling or an already completed diskutil is reported explicitly; nonzero observation exits are logged and cannot satisfy any native gate. + +The observer owns its command/timer PIDs and is stopped when the disk query completes or the probe is canceled. Its output enters the existing 512-KiB per-output and 4-MiB proof budgets. The hook only reads media/service/process state and writes its existing diagnostic files: it does not load, restart, erase or modify any service or disk. Bash remains necessary because Apple Recovery runs this hook before a .NET SDK is installed. The CPU, Recovery, QEMU, Apple wrapper and container profile are unchanged. These observations are prepared diagnostics, not a new successful guest or full native CI receipt. Container profile: `KVM=Y`, `CPU_MODEL=host`, `VERSION=13`, 4-GiB guest, two guest/host CPUs, 6-GiB memory/swap and 512-MiB shared memory. Fresh anonymous `/storage` holds the 64-GiB disk; evidence reads `/storage/13/setup.dmg`. Existing resource budget checks remain. diff --git a/tools/ci/MacOsNativeDiagnostic.cs b/tools/ci/MacOsNativeDiagnostic.cs index 67d2a83..eed82fa 100644 --- a/tools/ci/MacOsNativeDiagnostic.cs +++ b/tools/ci/MacOsNativeDiagnostic.cs @@ -69,7 +69,7 @@ static class NativeDiagnostic await PrepareSource(Path.GetFullPath(source), output, "validation", false, Option(args, "--cryptex-archive"), CancellationToken.None); await ValidateResourceRetention(output); await ValidateRecoveryPatch(output); - Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow }); + Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, readinessDiagnosticBlocksExcluded = 6, diskReadinessAttemptLimit = 2, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow }); } Console.WriteLine("Source patch contracts and diagnostic result validation passed; no Docker or guest execution occurred."); return 0; @@ -184,9 +184,11 @@ static class NativeDiagnostic static void ValidateContracts() { var readiness = File.ReadAllText(Path.Combine("tools", "ci", "macos-native-readiness.sh")); - var baseline = ReplaceOnce(readiness, "(( ${os_version%%.*} >= 13 ))", "(( ${os_version%%.*} >= 14 ))"); + var baseline = NormalizeReadinessDiagnostics(readiness); + baseline = ReplaceOnce(baseline, "(( ${os_version%%.*} >= 13 ))", "(( ${os_version%%.*} >= 14 ))"); + baseline = ReplaceOnce(baseline, "while (( attempt < 2 && SECONDS - readiness_start < 600 )); do", "while (( SECONDS - readiness_start < 600 )); do"); if (Hash(Encoding.UTF8.GetBytes(baseline)) != "4d428f594dac14eff64ed87b172c81ecf85ac91da8c5460cd6ec4b1d310800c3") - throw new InvalidOperationException("Compatibility readiness may change only the baseline's macOS minimum to 13; identity, services, disk and limits must remain identical."); + throw new InvalidOperationException("Outside six explicit diagnostic blocks, the macOS minimum and two-attempt limit, baseline identity/service/disk gates and watchdogs must remain identical."); if (Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-bootstrap.sh"))) != "94f069e116fdc7685a4d233cab6fa50df9f39274386bb82157674061e74fadb5") throw new InvalidOperationException("Compatibility profile must preserve the baseline Apple recoveryosd wrapper."); if (Hash(Encoding.UTF8.GetBytes(OriginalDaemon13)) != "af9d7f6c1948079bd4384d27b6882678d6fb4e338fcf6a8be8f84fceef174ad6") throw new InvalidOperationException("macOS 13 allowlist bytes differ from the independently read comparison plist."); @@ -214,6 +216,24 @@ static class NativeDiagnostic } } + static string NormalizeReadinessDiagnostics(string source) + { + const string start = "# BEGIN disk IPC diagnostic\n"; + const string end = "# END disk IPC diagnostic\n"; + var blocks = 0; + while (source.IndexOf(start, StringComparison.Ordinal) is var from && from >= 0) + { + var to = source.IndexOf(end, from + start.Length, StringComparison.Ordinal); + if (to < 0 || source.IndexOf(start, from + start.Length, to - from - start.Length, StringComparison.Ordinal) >= 0) + throw new InvalidOperationException("Readiness diagnostic blocks are unbalanced or nested."); + source = source.Remove(from, to + end.Length - from); + blocks++; + } + if (blocks != 6 || source.Contains(end, StringComparison.Ordinal)) + throw new InvalidOperationException("Readiness must contain exactly six explicit disk IPC diagnostic blocks."); + return source; + } + static async Task PrepareSource(string source, string output, string token, bool writeSource, string? cryptexArchive, CancellationToken cancellation) { Directory.CreateDirectory(output); diff --git a/tools/ci/macos-native-readiness.sh b/tools/ci/macos-native-readiness.sh index 5c68ae2..38b7c8e 100644 --- a/tools/ci/macos-native-readiness.sh +++ b/tools/ci/macos-native-readiness.sh @@ -14,6 +14,9 @@ TIMER_FIFO="/tmp/native-diagnostic-$PROOF_TOKEN-$$.fifo" PENDING_OUTPUTS=() ACTIVE_COMMAND="" ACTIVE_TIMER="" +# BEGIN disk IPC diagnostic +ACTIVE_OBSERVER="" +# END disk IPC diagnostic os_version="" architecture="" uid=-1 @@ -96,8 +99,75 @@ read_scalar() { SCALAR="$value" } +# BEGIN disk IPC diagnostic +# Optional observations have their own child/timer ownership. No service is +# loaded, restarted or changed, and samples target only this probe's diskutil. +observe_disk_query() { + local disk_process="$1" output="$2" sample_output="$3" observation_child="" observation_timer="" + cancel_observation() { + trap '' TERM INT + if [ -n "$observation_child" ]; then + kill -TERM "$observation_child" 2>/dev/null || : + IFS= read -r -t 2 -u 9 unused || : + kill -KILL "$observation_child" 2>/dev/null || : + wait "$observation_child" 2>/dev/null || : + fi + [ -z "$observation_timer" ] || { kill -TERM "$observation_timer" 2>/dev/null || :; wait "$observation_timer" 2>/dev/null || :; } + printf '[disk-observation] stopped after the owned disk query\n' >> "$output" + exit 143 + } + observe_command() { + local name="$1" status started=$SECONDS + shift + printf '\n[disk-observation-command] %s:' "$name" >> "$output" + printf ' %s' "$@" >> "$output" + printf '\n' >> "$output" + "$@" >> "$output" 2>&1 & + observation_child=$! + ( + trap 'exit 0' TERM INT + IFS= read -r -t 8 -u 9 unused || : + printf '[disk-observation-timeout] %s child=%s limit=8s\n' "$name" "$observation_child" >> "$output" + kill -TERM "$observation_child" 2>/dev/null || : + IFS= read -r -t 2 -u 9 unused || : + kill -KILL "$observation_child" 2>/dev/null || : + ) & + observation_timer=$! + wait "$observation_child"; status=$? + kill -TERM "$observation_timer" 2>/dev/null || : + wait "$observation_timer" 2>/dev/null || : + printf '[disk-observation-exit] %s status=%s elapsed=%ss\n' "$name" "$status" "$((SECONDS - started))" >> "$output" + observation_child=""; observation_timer="" + } + trap cancel_observation TERM INT + printf '[disk-observation] owned-diskutil-child=%s parent-shell=%s\n' "$disk_process" "$$" >> "$output" + observe_command processes /bin/ps -axo pid,ppid,state,comm + if [ -x /usr/bin/sample ]; then + if kill -0 "$disk_process" 2>/dev/null; then + observe_command diskutil-sample /usr/bin/sample "$disk_process" 3 10 -file "$sample_output" + else + printf '[disk-observation-unavailable] diskutil already exited before sample\n' >> "$output" + fi + else + printf '[disk-observation-unavailable] /usr/bin/sample is unavailable\n' >> "$output" + fi + observe_command arbitration /bin/launchctl print system/com.apple.diskarbitrationd + observe_command management /bin/launchctl print system/com.apple.diskmanagementd +} + +stop_disk_observation() { + [ -n "$ACTIVE_OBSERVER" ] || return 0 + kill -TERM "$ACTIVE_OBSERVER" 2>/dev/null || : + wait "$ACTIVE_OBSERVER" 2>/dev/null || : + ACTIVE_OBSERVER="" +} + +# END disk IPC diagnostic cancel_probe() { trap '' TERM INT +# BEGIN disk IPC diagnostic + stop_disk_observation +# END disk IPC diagnostic if [ -n "$ACTIVE_COMMAND" ]; then kill -TERM "$ACTIVE_COMMAND" 2>/dev/null || : IFS= read -r -t 2 -u 9 unused || : @@ -136,6 +206,18 @@ run_command() { ) & timer=$! ACTIVE_TIMER="$timer" +# BEGIN disk IPC diagnostic + if [[ "$name" == disks && "$attempt" == 1 ]]; then + local observation_output="/tmp/native-diagnostic-disk-observation.out" + local sample_output="/tmp/native-diagnostic-disk-sample.out" + : > "$observation_output" + : > "$sample_output" + observe_disk_query "$process" "$observation_output" "$sample_output" & + ACTIVE_OBSERVER=$! + printf '[disk-observation-start] observer=%s owned-diskutil-child=%s\n' "$ACTIVE_OBSERVER" "$process" >&3 + PENDING_OUTPUTS+=("$observation_output" "$sample_output") + fi +# END disk IPC diagnostic wait "$process" exit_code=$? waited=$SECONDS @@ -143,6 +225,9 @@ run_command() { printf '[proof-native-wait] %s child=%s elapsed=%ss exit=%s\n' "$name" "$process" "$((waited - started))" "$exit_code" >&3 kill -TERM "$timer" 2>/dev/null || : wait "$timer" 2>/dev/null || : +# BEGIN disk IPC diagnostic + stop_disk_observation +# END disk IPC diagnostic ACTIVE_COMMAND=""; ACTIVE_TIMER="" printf '[proof-cleanup] %s child=%s elapsed=%ss total=%ss\n' "$name" "$process" "$((SECONDS - waited))" "$((SECONDS - started))" >&3 printf '[proof-exit] %s\n' "$exit_code" >&3 @@ -203,10 +288,17 @@ os_version="$SCALAR" (( ${os_version%%.*} >= 13 )) || fail_probe unsupported_macos_version flush_outputs || finish false diagnostic_log_budget_exceeded +# BEGIN disk IPC diagnostic +run_command arbitration_before /bin/launchctl print system/com.apple.diskarbitrationd +run_command management_before /bin/launchctl print system/com.apple.diskmanagementd +run_command media_before /usr/sbin/ioreg -r -c IOMedia -l -w 0 +flush_outputs || finish false diagnostic_log_budget_exceeded + +# END disk IPC diagnostic # Bound readiness independently of the host's 40-minute overall deadline. readiness_start=$SECONDS attempt=0 -while (( SECONDS - readiness_start < 600 )); do +while (( attempt < 2 && SECONDS - readiness_start < 600 )); do attempt=$((attempt + 1)) printf '\n[readiness-attempt] %s\n' "$attempt" >&3 run_command disks /usr/sbin/diskutil list physical