From b19dbd21a76cc99378964e8f6343f9f8d88051d7 Mon Sep 17 00:00:00 2001 From: dh Date: Sat, 3 Oct 2026 16:47:45 +0200 Subject: [PATCH] ci: retain bounded native platform process diagnostics in full candidate --- docs/macos-native-diagnostic.md | 4 ++- tools/ci/macos-native-readiness.sh | 57 +++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/docs/macos-native-diagnostic.md b/docs/macos-native-diagnostic.md index 4c3e0fc..7bab260 100644 --- a/docs/macos-native-diagnostic.md +++ b/docs/macos-native-diagnostic.md @@ -20,7 +20,9 @@ Dependencies are the existing Linux/x64 runner, .NET 10 SDK, Git, Bash and Docke The helper clones Dockur commit `16a5b470cdd601bae8b05b02d748d7edfb36c12e`, verifies its exact Recovery patcher hash, and makes three narrowly verified source edits. The early `rc.cdrom.sh` hook only mounts the existing state share and returns. A same-length XML replacement makes the existing `com.apple.recoveryosd` LaunchDaemon execute `/bin/bash /Volumes/installstate/launch.sh` after boot tasks. The staged `launch.sh` is replaced entirely by the checked-in read-only readiness probe. All replacement counts are exact; an upstream mismatch fails. The two imported QEMU image digests are pinned and the final image/source/Recovery hashes are retained. Other upstream Dockerfile downloads are observed through the resulting image identity rather than asserted to be immutable. -The VM uses TCG (`KVM=N`), slirp networking, a 4-GiB guest, two virtual CPUs and a sparse 64-GiB data disk. Its container has a 6-GiB memory/swap ceiling and a two-CPU limit. The existing Docker daemon must report at least two CPUs and 6 GiB total memory, the runner must have at least 5 GiB available memory, and the Docker filesystem must have at least 8 GiB free before Recovery downloads or boot. Its own native commands have per-command watchdogs and a ten-minute readiness phase; the host orchestrator has a 40-minute deadline and the workflow a 45-minute limit. +The VM uses TCG (`KVM=N`), slirp networking, a 4-GiB guest, two virtual CPUs and a sparse 64-GiB data disk. Its container has a 6-GiB memory/swap ceiling and a two-CPU limit. The existing Docker daemon must report at least two CPUs and 6 GiB total memory, the runner must have at least 5 GiB available memory, and the Docker filesystem must have at least 8 GiB free before Recovery downloads or boot. Its own native commands retain 45-second watchdogs and a ten-minute readiness phase; the host orchestrator has a 40-minute deadline and the workflow a 45-minute limit. + +Actual remote run 4155 stopped at the first `sw_vers` with exit 143 before kernel, process or service probes ran. The updated hook collects native `uname`, root identity, bootargs, guest CPU features and process context first. It logs each child PID and builtin elapsed time, explicitly tags watchdog TERM, and takes two independently five-second-bounded CPU/state/command snapshots during each `sw_vers` attempt. After an initial platform failure it still collects native launchd context and repeats the identical `sw_vers` command once, with the same 45-second limit. A successful native `sw_vers`, native product version and all original identity/service/disk gates remain required. Process state or a retry alone does not establish whether initialization was slow or a service blocked. The upstream AVX2 warning reads host flags; the pinned TCG CPU path configures an Intel guest with AVX/AVX2, so the hook observes actual guest CPU flags without changing host or guest CPU settings. ## Evidence and cleanup diff --git a/tools/ci/macos-native-readiness.sh b/tools/ci/macos-native-readiness.sh index d451f88..8382e7c 100644 --- a/tools/ci/macos-native-readiness.sh +++ b/tools/ci/macos-native-readiness.sh @@ -45,28 +45,59 @@ finish() { run_command() { local name="$1" shift - local process timer sleeper exit_code + local process timer sleeper exit_code started observer="" LAST_OUTPUT="/tmp/native-diagnostic-$name.out" printf '\n[proof-command] %s:' "$name" >> "$PROOF_LOG" printf ' %s' "$@" >> "$PROOF_LOG" printf '\n' >> "$PROOF_LOG" + started=$SECONDS "$@" > "$LAST_OUTPUT" 2>&1 & process=$! + printf '[proof-start] %s child=%s shell=%s parent=%s seconds=%s\n' "$name" "$process" "$$" "$PPID" "$started" >> "$PROOF_LOG" ( trap 'kill "$sleeper" 2>/dev/null || :; exit 0' TERM INT sleep 45 & sleeper=$! wait "$sleeper" + printf '[proof-timeout] %s child=%s elapsed=%ss signal=TERM\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG" kill -TERM "$process" 2>/dev/null || : - sleep 2 + sleep 2 & sleeper=$!; wait "$sleeper" kill -KILL "$process" 2>/dev/null || : ) & timer=$! + if [[ "$name" = platform || "$name" = platform-warm ]]; then + # Observers never extend the independent 45-second command deadline. + ( + local sample_pid="" sample_timer="" pause_pid="" pause sample_exit + trap 'kill -KILL "$sample_pid" 2>/dev/null || :; kill -TERM "$sample_timer" "$pause_pid" 2>/dev/null || :; exit 0' TERM INT + for pause in 10 15; do + sleep "$pause" & pause_pid=$!; wait "$pause_pid" + printf '[proof-process] %s child=%s elapsed=%ss fields=pid,ppid,stat,cpu-time,elapsed,cpu-percent,wchan,comm\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG" + /bin/ps -p "$process" -o pid=,ppid=,stat=,time=,etime=,pcpu=,wchan=,comm= >> "$PROOF_LOG" 2>&1 & + sample_pid=$! + ( + local sample_sleeper="" + trap 'kill "$sample_sleeper" 2>/dev/null || :; exit 0' TERM INT + sleep 5 & sample_sleeper=$!; wait "$sample_sleeper" + kill -KILL "$sample_pid" 2>/dev/null || : + ) & sample_timer=$! + wait "$sample_pid"; sample_exit=$? + kill -TERM "$sample_timer" 2>/dev/null || :; wait "$sample_timer" 2>/dev/null || : + printf '[proof-process-exit] %s %s\n' "$name" "$sample_exit" >> "$PROOF_LOG" + sample_pid=""; sample_timer=""; pause_pid="" + done + ) & observer=$! + fi wait "$process" exit_code=$? kill -TERM "$timer" 2>/dev/null || : wait "$timer" 2>/dev/null || : + if [ -n "$observer" ]; then + kill -TERM "$observer" 2>/dev/null || : + wait "$observer" 2>/dev/null || : + fi /usr/bin/tail -c 524288 "$LAST_OUTPUT" >> "$PROOF_LOG" + printf '\n[proof-duration] %s child=%s elapsed=%ss\n' "$name" "$process" "$((SECONDS - started))" >> "$PROOF_LOG" printf '\n[proof-exit] %s\n' "$exit_code" >> "$PROOF_LOG" LAST_EXIT="$exit_code" local size @@ -75,13 +106,7 @@ run_command() { return 0 } -run_command platform /usr/bin/sw_vers -(( LAST_EXIT == 0 )) || finish false sw_vers_failed -run_command version /usr/bin/sw_vers -productVersion -(( LAST_EXIT == 0 )) || finish false product_version_failed -os_version=$(cat "$LAST_OUTPUT") -[[ "$os_version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || finish false product_version_invalid -(( ${os_version%%.*} >= 14 )) || finish false unsupported_macos_version +# Collect cheap native identity/context before the first framework-dependent probe. run_command kernel /usr/bin/uname -a (( LAST_EXIT == 0 )) || finish false uname_failed run_command architecture /usr/bin/uname -m @@ -94,14 +119,28 @@ run_command uid /usr/bin/id -u uid=$(cat "$LAST_OUTPUT") [ "$uid" = 0 ] || finish false recovery_account_not_root run_command bootargs /usr/sbin/sysctl kern.bootargs +run_command cpu /usr/sbin/sysctl machdep.cpu.brand_string machdep.cpu.features machdep.cpu.leaf7_features run_command parent /bin/ps -p "$$" -p "$PPID" -o pid=,ppid=,comm= run_command processes /bin/ps -axo pid,ppid,comm +run_command platform /usr/bin/sw_vers +platform_exit="$LAST_EXIT" run_command system /bin/launchctl print system system_exit="$LAST_EXIT" run_command arbitration /bin/launchctl print system/com.apple.diskarbitrationd arbitration_exit="$LAST_EXIT" run_command recovery /bin/launchctl print system/com.apple.recoveryosd recovery_exit="$LAST_EXIT" +if (( platform_exit != 0 )); then + printf '[proof-retry] sw_vers once after native service context; same 45-second deadline\n' >> "$PROOF_LOG" + run_command platform-warm /usr/bin/sw_vers + platform_exit="$LAST_EXIT" +fi +(( platform_exit == 0 )) || finish false sw_vers_failed +run_command version /usr/bin/sw_vers -productVersion +(( LAST_EXIT == 0 )) || finish false product_version_failed +os_version=$(cat "$LAST_OUTPUT") +[[ "$os_version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || finish false product_version_invalid +(( ${os_version%%.*} >= 14 )) || finish false unsupported_macos_version # Bound readiness independently of the host's 40-minute overall deadline. readiness_start=$SECONDS