From 90d86af887149bd5eac978516be6660016a4e240 Mon Sep 17 00:00:00 2001 From: dh Date: Sat, 3 Oct 2026 15:54:58 +0200 Subject: [PATCH] ci: report native progress and require complete unique test evidence --- docs/macos-native-diagnostic.md | 4 +-- tools/ci/MacOsNativeDiagnostic.cs | 47 +++++++++++++++++++++++++++++-- tools/ci/MacOsNativeGuest.cs | 22 +++++++++++++-- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/docs/macos-native-diagnostic.md b/docs/macos-native-diagnostic.md index a5db199..8d2f554 100644 --- a/docs/macos-native-diagnostic.md +++ b/docs/macos-native-diagnostic.md @@ -43,7 +43,7 @@ dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --cleanup --output artifa The host requires a clean exact Git HEAD, creates its Git/PAX source archive and SHA-256, and downloads macOS/x64 SDK `10.0.401` from Microsoft's release URL with the fixed official SHA-512 recorded in both helpers. Source archive, SDK and helper files are copied inside the image into the newly owned anonymous `/storage` volume; no workstation bind mount is introduced. The full flow uses persistent `/storage/14/ci-state` as its existing 9p share, with a run-owner marker and an erase guard that is never removed on installer failure. It never automatically restarts a container or retries erasure. -Before the only guest `eraseDisk`, C# revalidates the owned Docker boundary, sole anonymous storage mount, exact 64-GiB raw image, live QEMU attachment and per-run emulated disk serial. Only after a valid native Recovery receipt does it atomically provide the run/commit/disk permit. The guarded Apple installer rechecks `diskutil` and the corresponding IORegistry serial; missing or ambiguous identity fails. Upstream `startosinstall`, USR1 bootstrap staging, Setup Assistant/admin packages and byte-for-byte staging checks remain in use. Installer reboots preserve the same QEMU process, disk, NVRAM and share. The readonly Recovery media stays attached. +Before the only guest `eraseDisk`, C# revalidates the owned Docker boundary, sole anonymous storage mount, exact 64-GiB raw image, live QEMU attachment and per-run emulated disk serial. Only after a valid native Recovery receipt does it atomically provide the run/commit/disk permit. The guarded Apple installer rechecks `diskutil` and the corresponding IORegistry serial; missing or ambiguous identity fails. With mounted run-owned state, `fail()`, nonzero `startosinstall` and TERM/INT atomically publish a token-bound `installation-failed` phase for the next host poll, preserving the erase guard. Upstream `startosinstall`, USR1 bootstrap staging, Setup Assistant/admin packages and byte-for-byte staging checks remain in use. Installer reboots preserve the same QEMU process, disk, NVRAM and share. The readonly Recovery media stays attached. The existing firstboot LaunchDaemon invokes `tools/ci/macos-native-firstboot.sh` before its staging cleanup. This Bash seam is required because the guest has Apple boot tools but no .NET SDK yet. It proves installed APFS `/` belongs to the same owned 64-GiB physical disk, mounts the state share, installs a compatible Apple CLT catalog label through headless `softwareupdate`, verifies the CLT package/compiler and builds a framework smoke program. There is no GUI fallback, Apple account or new secret. `macos-native-disk-guard.sh` holds the shared pre-.NET Apple disk/IORegistry check. The existing upstream Python UDIF patcher remains the image-format runtime binding; both its exact patch matches and compressed-slot checks remain enforced. @@ -51,6 +51,6 @@ After verifying and extracting the SDK on the guest's own APFS work directory, ` The full helper's outer deadline is 172 minutes; the job declares 180 minutes within the existing three-hour server limit, leaving time for evidence and owned-resource cleanup. Independent budgets are Recovery 40 minutes, installer 80, firstboot/CLT 30 and guest checks/restore/build/tests 25; the outer deadline also bounds their combined runtime and preparation. The same 4-GiB/two-CPU guest and 6-GiB container remain, with explicit sparse allocation. Full execution checks 32 GiB of existing Docker free space before Recovery downloads/boot and 8 GiB of guest free space before toolchain work. Insufficient resources, networking, Apple catalog availability, disk ownership, installer progress or test proof fail clearly without changing infrastructure. -Artifacts add source/SDK hashes, generated pinned boot-source patches, installer/Apple/firstboot logs, installed-root/disk identity, native tool logs, guest phase and full-result receipts, native helper hashes and binary-preserved TRX. Bounded final build/test output and the full-result receipt are also printed in CI. Cleanup uses the same exact saved resource ID/ownership label through `finally` and workflow `always()`; it removes only this run's container/image/anonymous volume. Installed guest files disappear with that volume and retained CI evidence stays outside it. Shared Docker build cache is not pruned. +Artifacts add source/SDK hashes, generated pinned boot-source patches, installer/Apple/firstboot logs, installed-root/disk identity, native tool logs, guest phase and full-result receipts, native helper hashes and binary-preserved TRX. The polling loop prints a bounded heartbeat after each elapsed minute with the current phase, its elapsed/budget time, container liveness and readiness state, without dumping environment variables or download URLs. Bounded final build/test output and the full-result receipt are also printed in CI. Cleanup uses the same exact saved resource ID/ownership label through `finally` and workflow `always()`; it removes only this run's container/image/anonymous volume. Installed guest files disappear with that volume and retained CI evidence stays outside it. Shared Docker build cache is not pruned. Local `--validate` creates only patch/fixture evidence; it never starts Docker, installs an OS/toolchain, erases a disk, builds the application or runs native tests. With `--compression-chunk` it can read the retained qualified Recovery raw chunk and validate the unchanged LaunchDaemon/mount patch against Python zlib's real compressed slot, without changing the DMG. The IORegistry parser's root association and installed-APFS mapping still require the actual emulated guest's output; synthetic fixtures do not qualify that disk identity. This is a candidate until an actual remote installed guest produces every required native receipt. diff --git a/tools/ci/MacOsNativeDiagnostic.cs b/tools/ci/MacOsNativeDiagnostic.cs index 621043a..831c753 100644 --- a/tools/ci/MacOsNativeDiagnostic.cs +++ b/tools/ci/MacOsNativeDiagnostic.cs @@ -134,6 +134,7 @@ static class NativeDiagnostic var phaseStarted = Stopwatch.StartNew(); var phase = "recovery"; var phaseBudget = TimeSpan.FromMinutes(40); + var heartbeat = Stopwatch.StartNew(); var permitted = false; while (true) { @@ -185,6 +186,11 @@ static class NativeDiagnostic } var running = await Command("docker", ["inspect", "--format", "{{.State.Running}}", id], output, "container-running", deadline.Token); if (running.Output.Trim() != "true") throw new InvalidOperationException("Guest container exited before a native readiness result."); + if (heartbeat.Elapsed >= TimeSpan.FromSeconds(60)) + { + Console.WriteLine($"[native-diagnostic] phase={phase}; elapsed={phaseStarted.Elapsed.TotalMinutes:F1}/{phaseBudget.TotalMinutes:F0} minutes; container=running; readiness={(permitted ? "passed" : "pending")}"); + heartbeat.Restart(); + } await Task.Delay(TimeSpan.FromSeconds(20), deadline.Token); } } @@ -346,9 +352,18 @@ static class NativeDiagnostic installer = ReplaceOnce(installer, installer[selectorStart..selectorEnd], selector); installer = ReplaceOnce(installer, "MIN_TARGET_SIZE=$((16 * 1024 * 1024 * 1024))", "# Target policy is exclusively the own writable 64-GiB emulated disk."); installer = ReplaceOnce(installer, "no writable installation disk of at least 16 GiB was found", "the run-owned writable 64-GiB installation disk was not proved"); + installer = ReplaceOnce(installer, " local message=\"$1\"\n\n echo \"[log] ERROR: $message\"", " local message=\"$1\"\n\n mark_installation_failed || :\n echo \"[log] ERROR: $message\""); + installer = ReplaceOnce(installer, "if (( rc != 0 )); then\n", "if (( rc != 0 )); then\n mark_installation_failed || :\n"); installer = ReplaceAllExact(installer, "rm -f \"$STARTED\"", ": # Keep the owned erase guard on failure; never erase again.", 2); installer = ReplaceOnce(installer, ": > \"$STARTED\" || fail \"failed to create installation guard\"", "( set -o noclobber; printf '%s:%s:%s\\n' \"$PROOF_TOKEN\" \"$(cat \"$STATE_DIR/source.commit\")\" \"$TARGET_DISK\" > \"$STARTED\" ) || fail \"failed to create the exclusive owned installation guard\""); installer = ReplaceOnce(installer, "set -u\n", "set -u\nPROOF_TOKEN=\"" + token + "\"\n" + """ + mark_installation_failed() { + local state="${STATE_DIR:-/Volumes/installstate}" temporary + [ "$(cat "$state/run.owner" 2>/dev/null)" = "$PROOF_TOKEN" ] || return 1 + temporary="$state/guest-phase.install.$$.tmp" + printf '{"token":"%s","phase":"installation-failed"}\n' "$PROOF_TOKEN" > "$temporary" && + /bin/mv -f "$temporary" "$state/guest-phase.json" + } installer_parent=$$ # Installer watchdog: 80 minutes, also bounded by the host's 172-minute total. ( @@ -356,7 +371,7 @@ static class NativeDiagnostic sleep 4800 & sleeper=$!; wait "$sleeper"; kill -TERM "$installer_parent" 2>/dev/null || : ) & install_watchdog=$! trap 'kill -TERM "$install_watchdog" 2>/dev/null || :; wait "$install_watchdog" 2>/dev/null || :' EXIT - trap 'kill "${STARTOSINSTALL_PID:-}" "${BOOTSTRAPPER_PID:-}" 2>/dev/null || :; printf "{\"token\":\"%s\",\"phase\":\"installation-failed\"}\n" "$PROOF_TOKEN" > /Volumes/installstate/guest-phase.json; exit 1' TERM INT + trap 'kill "${STARTOSINSTALL_PID:-}" "${BOOTSTRAPPER_PID:-}" 2>/dev/null || :; mark_installation_failed || :; exit 1' TERM INT """ + "\n"); var firstboot = ReadPinned(source, "src/install/firstboot/launch.sh", "d6b29bb42ffe99edda6b3be3faf6009c4e0b34e5b8bba6eb0855cf24a0c24307"); firstboot = ReplaceOnce(firstboot, "log \"prebuilt account package installed successfully\"\n", "log \"prebuilt account package installed successfully\"\n" + """ @@ -392,12 +407,40 @@ static class NativeDiagnostic foreach (var name in new[] { "macos-native-firstboot.sh", "macos-native-disk-guard.sh" }) await Command("bash", ["-n", Path.Combine("tools", "ci", name)], output, name + "-syntax", cancellation); Save(Path.Combine(output, "full-source-hashes.json"), new Dictionary { ["full-install.sh"] = Hash(Encoding.UTF8.GetBytes(installer)), ["full-firstboot.sh"] = Hash(Encoding.UTF8.GetBytes(firstboot)), ["full-state-source.sh"] = Hash(Encoding.UTF8.GetBytes(initialize)), ["MacOsNativeGuest.cs"] = Hash(File.ReadAllBytes("tools/ci/MacOsNativeGuest.cs")), ["macos-native-firstboot.sh"] = Hash(File.ReadAllBytes("tools/ci/macos-native-firstboot.sh")), ["macos-native-disk-guard.sh"] = Hash(File.ReadAllBytes("tools/ci/macos-native-disk-guard.sh")) }); - if (!writeSource) return; + if (!writeSource) + { + await ValidateInstallerFailureReceipt(installer, output, token, cancellation); + return; + } File.WriteAllText(Path.Combine(source, "src/install/recovery/full-install.sh"), installer, new UTF8Encoding(false)); File.WriteAllText(Path.Combine(source, "src/install/firstboot/launch.sh"), firstboot, new UTF8Encoding(false)); File.WriteAllText(Path.Combine(source, "src/install.sh"), initialize, new UTF8Encoding(false)); } + static async Task ValidateInstallerFailureReceipt(string installer, string output, string token, CancellationToken cancellation) + { + const string start = "mark_installation_failed() {"; + const string end = "\n}\ninstaller_parent=$$"; + var begin = installer.IndexOf(start, StringComparison.Ordinal); + var finish = begin < 0 ? -1 : installer.IndexOf(end, begin, StringComparison.Ordinal); + if (begin < 0 || finish < begin || installer.Split("mark_installation_failed || :", StringSplitOptions.None).Length != 4) + throw new InvalidOperationException("Pinned installer must publish its terminal failure phase from fail(), nonzero startosinstall and TERM/INT."); + var function = installer[begin..(finish + 2)]; + var state = Path.Combine(output, "installer-failure-fixture"); + Directory.CreateDirectory(state); + File.WriteAllText(Path.Combine(state, "run.owner"), token); + var command = "set -u\n" + function + "\nPROOF_TOKEN=\"$1\"; STATE_DIR=\"$2\"; mark_installation_failed"; + await Command("bash", ["-c", command, "installer-failure-validation", token, state], output, "installer-terminal-failure", cancellation); + var receipt = File.ReadAllText(Path.Combine(state, "guest-phase.json")); + using var json = JsonDocument.Parse(receipt); + if (json.RootElement.GetProperty("token").GetString() != token || json.RootElement.GetProperty("phase").GetString() != "installation-failed" || Directory.GetFiles(state, "*.tmp").Length != 0) + throw new InvalidOperationException("Installer failed to publish a complete atomic terminal phase."); + File.WriteAllText(Path.Combine(state, "run.owner"), "foreign"); + var foreign = await Command("bash", ["-c", command, "installer-failure-validation", token, state], output, "installer-foreign-failure", cancellation, requireSuccess: false); + if (foreign.ExitCode == 0 || File.ReadAllText(Path.Combine(state, "guest-phase.json")) != receipt) + throw new InvalidOperationException("Installer terminal phase overwrote foreign run-owned state."); + } + static async Task PermitInstallation(string id, string output, string token, string commit, string readiness, CancellationToken cancellation) { await Command("docker", ["inspect", id], output, "full-container-boundary", cancellation); diff --git a/tools/ci/MacOsNativeGuest.cs b/tools/ci/MacOsNativeGuest.cs index 7a10183..2564937 100644 --- a/tools/ci/MacOsNativeGuest.cs +++ b/tools/ci/MacOsNativeGuest.cs @@ -277,17 +277,18 @@ static class NativeGuest } var results = root.Element(ns + "Results")?.Elements(ns + "UnitTestResult").ToArray() ?? []; var executionIds = new HashSet(StringComparer.Ordinal); + var testIds = new HashSet(StringComparer.Ordinal); var native = new List(); foreach (var result in results) { var id = (string?)result.Attribute("testId") ?? ""; var executionId = (string?)result.Attribute("executionId") ?? ""; - if ((string?)result.Attribute("outcome") != "Passed" || executionId.Length == 0 || !executionIds.Add(executionId) || !definitions.TryGetValue(id, out var identity)) + if ((string?)result.Attribute("outcome") != "Passed" || executionId.Length == 0 || !executionIds.Add(executionId) || !testIds.Add(id) || !definitions.TryGetValue(id, out var identity)) throw new InvalidOperationException("TRX has a missing, duplicate or non-passed execution."); if (RequiredNativeTests.Contains(identity, StringComparer.Ordinal)) native.Add(identity); } - if (results.Length != expected || native.Count != RequiredNativeTests.Length || !native.Order().SequenceEqual(RequiredNativeTests.Order())) - throw new InvalidOperationException("TRX does not prove every expected execution and the five explicit native macOS tests."); + if (definitions.Count != expected || results.Length != expected || native.Count != RequiredNativeTests.Length || !native.Order().SequenceEqual(RequiredNativeTests.Order())) + throw new InvalidOperationException("TRX does not prove every expected test definition exactly once and the five explicit native macOS tests."); return new(total, executed, passed, failed, notExecuted, native.ToArray()); } @@ -461,6 +462,21 @@ static class NativeGuest var duplicate = new XDocument(fixture); duplicate.Descendants(ns + "UnitTestResult").Last().SetAttributeValue("executionId", "execution-0"); Reject(() => ValidateTrx(duplicate.ToString(), ExpectedTests, started)); + var repeatedManaged = new XDocument(fixture); + repeatedManaged.Descendants(ns + "UnitTestResult").Last().SetAttributeValue("testId", "test-5"); + Reject(() => ValidateTrx(repeatedManaged.ToString(), ExpectedTests, started)); + var unexecutedDefinition = new XDocument(fixture); + var extraDefinition = new XElement(unexecutedDefinition.Descendants(ns + "UnitTest").Last()); + extraDefinition.SetAttributeValue("id", "unexecuted-test"); + unexecutedDefinition.Root!.Element(ns + "TestDefinitions")!.Add(extraDefinition); + Reject(() => ValidateTrx(unexecutedDefinition.ToString(), ExpectedTests, started)); + var missingDefinition = new XDocument(fixture); + missingDefinition.Descendants(ns + "UnitTest").Last().Remove(); + Reject(() => ValidateTrx(missingDefinition.ToString(), ExpectedTests, started)); + var theoryRows = new XDocument(fixture); + foreach (var method in theoryRows.Descendants(ns + "TestMethod").Skip(RequiredNativeTests.Length).Take(2)) + method.SetAttributeValue("name", "TheoryWithDistinctRowIds"); + ValidateTrx(theoryRows.ToString(), ExpectedTests, started); var counters = new XDocument(fixture); counters.Descendants(ns + "Counters").Single().SetAttributeValue("passed", "572"); Reject(() => ValidateTrx(counters.ToString(), ExpectedTests, started));