From bcb24e82eb81e16d1313f5f1f7c9755a794e1aee Mon Sep 17 00:00:00 2001 From: dh Date: Mon, 5 Oct 2026 08:48:26 +0200 Subject: [PATCH] Compare sector-equivalent RAW Recovery on the existing Ubuntu runner --- .gitea/workflows/macos-native-full.yaml | 2 +- .gitea/workflows/pr-push-build-and-test.yaml | 1 + docs/macos-native-diagnostic.md | 4 + tools/ci/MacOsNativeDiagnostic.cs | 129 ++++++++++++++++++- 4 files changed, 128 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/macos-native-full.yaml b/.gitea/workflows/macos-native-full.yaml index 5764c18..85b6f70 100644 --- a/.gitea/workflows/macos-native-full.yaml +++ b/.gitea/workflows/macos-native-full.yaml @@ -18,7 +18,7 @@ jobs: with: dotnet-version: "10.0.x" - name: Run owned macOS 14 TCG guest and all native tests - run: dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run --full --output artifacts/native-macos-full + run: dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run --full --recovery-format raw --output artifacts/native-macos-full - name: Always remove only this run's owned resources if: always() run: dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --cleanup --output artifacts/native-macos-full diff --git a/.gitea/workflows/pr-push-build-and-test.yaml b/.gitea/workflows/pr-push-build-and-test.yaml index a688bd3..043e0cf 100644 --- a/.gitea/workflows/pr-push-build-and-test.yaml +++ b/.gitea/workflows/pr-push-build-and-test.yaml @@ -9,6 +9,7 @@ on: - codex/macos-ci-kvm-compatibility - codex/macos-ci-tcg-supported - codex/macos-native-full-tcg + - codex/macos-native-raw-recovery workflow_dispatch: jobs: diff --git a/docs/macos-native-diagnostic.md b/docs/macos-native-diagnostic.md index 9493c4a..4e0ace4 100644 --- a/docs/macos-native-diagnostic.md +++ b/docs/macos-native-diagnostic.md @@ -1,5 +1,9 @@ # macOS 14 TCG prerequisite diagnostic +The isolated RAW Recovery comparison starts from Full candidate `2e2d702e294c39f24c6a3e44e5e94ff793d8774b`. Use `dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run --full --recovery-format raw --output artifacts/native-macos-full`; the manual Full workflow includes this option. Its only runtime variable is the Recovery disk's DMG versus RAW backend. The default remains DMG. CPU, macOS version, permissions, native process probe and all time limits are unchanged. + +The existing pinned container's `qemu-img` converts the already-patched DMG, requires sector equality with `qemu-img compare`, and checks that conversion preserved the DMG's SHA256. Both images stay in this run's owned storage. A retained JSON receipt binds both hashes and the successful comparison. The original readonly virtio attachment and I/O thread are preserved; cleanup removes both images with that owned volume. Local `--validate --full --recovery-format raw --source --output ` exercises conversion failures, source mutation and strict backend selection with a mocked QEMU boundary. The generated `raw-recovery-real-qemu-fixture.sh` accepts an already-patched **disposable writable DMG copy** plus destination for an actual container QEMU comparison; it does not patch, mount or boot a guest. This comparison does not yet prove faster guest operation or native application tests. + This manual candidate uses the existing Ubuntu/x64 Docker runner. Before downloading Apple Recovery it tests actual AVX/AVX2 instruction execution in the pinned QEMU binary, then probes a fresh macOS 14+ Recovery guest. It does not install macOS, erase a disk, provision .NET/CLT or run Meeting Assistant tests. Readiness is only a prerequisite for full native CI. ## Profile and evidence diff --git a/tools/ci/MacOsNativeDiagnostic.cs b/tools/ci/MacOsNativeDiagnostic.cs index 1a912b1..1bdb8e6 100644 --- a/tools/ci/MacOsNativeDiagnostic.cs +++ b/tools/ci/MacOsNativeDiagnostic.cs @@ -69,11 +69,13 @@ static class NativeDiagnostic { if (args.Length == 0 || args.Contains("--help")) { - Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--output artifacts/native-macos] [--source existing-dockur-clone] [--compression-chunk readonly-qualified-chunk]"); + Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--recovery-format dmg|raw] [--output artifacts/native-macos] [--source existing-dockur-clone] [--compression-chunk readonly-qualified-chunk]"); return 0; } var output = Path.GetFullPath(Option(args, "--output") ?? "artifacts/native-macos"); var full = args.Contains("--full"); + var recoveryFormat = Option(args, "--recovery-format") ?? "dmg"; + if (recoveryFormat is not ("dmg" or "raw")) throw new ArgumentException("Recovery format must be dmg or raw."); if (args.Contains("--validate")) { ValidateContracts(); @@ -84,14 +86,16 @@ static class NativeDiagnostic if (full) ValidateFullContracts(); if (Option(args, "--source") is { } source) { - await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full); + await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full, recoveryFormat); + if (recoveryFormat == "raw") await ValidateRawRecoveryFixture(output, CancellationToken.None); await ValidateResourceRetention(output); await ValidateRecoveryPatch(output); await ValidateTcgPreflight(output, CancellationToken.None); if (full) await ValidateDiskSerialParser(output); Save(Path.Combine(output, "validation.json"), new { - success = true, profile = Profile, mode = full ? "full" : "readiness", + success = true, profile = Profile, mode = full ? "full" : "readiness", recoveryFormat, + causalSingleVariableTest = recoveryFormat == "raw", comparisonBaseCommit = "2e2d702e294c39f24c6a3e44e5e94ff793d8774b", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), readinessSourceSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-readiness.sh"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), @@ -153,7 +157,7 @@ static class NativeDiagnostic throw new InvalidOperationException("This diagnostic runs on the existing Linux/x64 runner only."); ValidateContracts(); var sourceCommit = (await Command("git", ["rev-parse", "HEAD"], output, "candidate-commit", deadline.Token)).Output.Trim(); - Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = Profile, causalSingleVariableTest = false, kvm = false, cpuModel = CpuModel, recoveryMajor = 14, cpuFlags = CpuFlags, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" }); + Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = Profile, recoveryFormat, causalSingleVariableTest = recoveryFormat == "raw", comparisonBaseCommit = "2e2d702e294c39f24c6a3e44e5e94ff793d8774b", kvm = false, cpuModel = CpuModel, recoveryMajor = 14, cpuFlags = CpuFlags, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" }); var info = await Command("docker", ["info", "--format", "{{json .}}"], output, "docker-info", deadline.Token); using (var document = JsonDocument.Parse(info.Output)) { @@ -173,7 +177,7 @@ static class NativeDiagnostic var actualCommit = (await Command("git", ["-C", source, "rev-parse", "HEAD"], output, "dockur-commit", deadline.Token)).Output.Trim(); if (actualCommit != DockurCommit) throw new InvalidOperationException("Dockur source pin mismatch."); if (full) await PreparePayload(source, output, token, sourceCommit, deadline.Token); - await PrepareSource(source, output, token, true, deadline.Token, full); + await PrepareSource(source, output, token, true, deadline.Token, full, recoveryFormat); await Command("docker", ["build", "--platform", "linux/amd64", "--label", OwnerLabel + "=" + token, "--tag", state.ImageTag, source], output, "docker-build", deadline.Token, echo: true); var imageInspect = await Command("docker", ["image", "inspect", state.ImageTag], output, "image-inspect", deadline.Token); using (var image = JsonDocument.Parse(imageInspect.Output)) @@ -385,7 +389,7 @@ static class NativeDiagnostic return source.Remove(from, to + end.Length - from).Insert(from, originalSequence); } - static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false) + static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false, string recoveryFormat = "dmg") { Directory.CreateDirectory(output); var probe = ReadProbeAssets(); @@ -436,6 +440,11 @@ static class NativeDiagnostic image = ReplaceOnce(image, " chmod 0755 \"$script\"\n", " chmod 0755 \"$script\" \"${script%/*}/native-process-probe-x86_64\"\n printf '%s\\n' '" + token + "' > \"${script%/*}/run.owner\" || return 1\n"); image = ReplaceOnce(image, " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n", " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cmp -s \"$IMAGE_TOOLS/recovery/readiness.sh\" \"$state/readiness.sh\" ||\n ! cmp -s \"$IMAGE_ASSETS/native-process-probe/native-process-probe-x86_64\" \"$state/native-process-probe-x86_64\" ||\n ! cmp -s \"$IMAGE_ASSETS/native-process-probe/build-manifest.json\" \"$state/native-process-probe-manifest.json\" ||\n"); image = ReplaceOnce(image, " if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n", " info \"[recovery-original] bytes=$(stat -c%s -- \"$image\") sha256=$(sha256sum \"$image\" | awk '{print $1}')\"\n if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n"); + if (recoveryFormat == "raw") + { + image = ReplaceOnce(image, " if ! mv -f \"$source\" \"$dest\"; then\n error \"Failed to save automated recovery image to $dest.\"\n return 1\n fi\n", RawRecoveryPreparation); + entry = ReplaceOnce(entry, ". disk.sh # Initialize disks\n", ". disk.sh # Initialize disks\n" + RawRecoveryDriveSelection + "\n"); + } if (full) { await PrepareFullSource(source, output, token, writeSource, cancellation); @@ -472,6 +481,112 @@ static class NativeDiagnostic File.WriteAllBytes(Path.Combine(probeAssets, "build-manifest.json"), probe.Manifest); } + static readonly string RawRecoveryPreparation = """ + local raw="$dest.raw" pending="$dest.raw.tmp" source_hash after_hash raw_hash + [ ! -e "$raw" ] && [ ! -e "$pending" ] && [ ! -e "$raw.json" ] || { + error 'RAW Recovery output already exists.'; return 1; + } + source_hash=$(sha256sum "$source" | awk '{print $1}') || return 1 + [[ "$source_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + if ! qemu-img convert -f dmg -O raw "$source" "$pending" || + ! qemu-img compare -f dmg -F raw "$source" "$pending"; then + rm -f "$pending" + error 'RAW Recovery conversion or sector equality failed.' + return 1 + fi + after_hash=$(sha256sum "$source" | awk '{print $1}') || { rm -f "$pending"; return 1; } + raw_hash=$(sha256sum "$pending" | awk '{print $1}') || { rm -f "$pending"; return 1; } + if [ "$source_hash" != "$after_hash" ] || [[ ! "$raw_hash" =~ ^[0-9a-f]{64}$ ]]; then + rm -f "$pending" + error 'Patched DMG changed during RAW conversion.' + return 1 + fi + if ! mv -f "$source" "$dest" || ! mv -f "$pending" "$raw"; then + rm -f "$pending" + error 'Failed to retain the patched DMG and equivalent RAW Recovery.' + return 1 + fi + printf '{"sourceFormat":"dmg","targetFormat":"raw","sectorEqualityVerified":true,"sourceUnchanged":true,"sourceSha256":"%s","rawSha256":"%s"}\n' "$source_hash" "$raw_hash" > "$raw.json" || return 1 + info "[recovery-raw] sector-equality=true source-unchanged=true source-sha256=$source_hash raw-sha256=$raw_hash" + """ + "\n"; + + static readonly string RawRecoveryDriveSelection = """ + # BEGIN raw Recovery backend + raw_recovery_from="file=$STORAGE/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none" + raw_recovery_to="file=$STORAGE/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none" + [[ -s "$STORAGE/setup.dmg.raw" && -s "$STORAGE/setup.dmg.raw.json" && "$DISK_OPTS" == *"$raw_recovery_from"* ]] || { + error 'Verified RAW Recovery or expected DMG backend is missing.'; exit 34; + } + raw_recovery_tail=${DISK_OPTS#*"$raw_recovery_from"} + [[ "$raw_recovery_tail" != *"$raw_recovery_from"* ]] || { error 'Recovery backend is duplicated.'; exit 34; } + DISK_OPTS=${DISK_OPTS/"$raw_recovery_from"/"$raw_recovery_to"} + unset raw_recovery_from raw_recovery_to raw_recovery_tail + # END raw Recovery backend + """; + + static async Task ValidateRawRecoveryFixture(string output, CancellationToken cancellation) + { + var image = File.ReadAllText(Path.Combine(output, "image.sh.patched")); + var begin = image.IndexOf("prepareAutomatedRecovery() {", StringComparison.Ordinal); + var end = image.IndexOf("\nreturn 0\n", begin, StringComparison.Ordinal); + if (begin < 0 || end < 0) throw new InvalidOperationException("Missing Recovery preparation boundary."); + var preparation = image[begin..end]; + var realScript = "#!/bin/bash\nset -Eeuo pipefail\ninfo() { printf '%s\\n' \"$*\"; }\nhtml() { :; }\nerror() { printf '%s\\n' \"$*\" >&2; }\npatchRecoveryBootstrap() { :; }\n" + preparation + "\nprepareAutomatedRecovery \"$1\" \"$2\"\n"; + File.WriteAllText(Path.Combine(output, "raw-recovery-real-qemu-fixture.sh"), realScript); + var mock = """ + qemu-img() { + case "$1" in + info) printf '%s\n' '{"format":"dmg"}' ;; + convert) + [ "$2 $3 $4 $5" = '-f dmg -O raw' ] || return 65 + [ "$TEST_CASE" != convert-failed ] || return 1 + cp "$6" "$7" || return 1 + [ "$TEST_CASE" != source-mutated ] || printf 'mutation' >> "$6" + ;; + compare) + [ "$2 $3 $4 $5" = '-f dmg -F raw' ] || return 65 + [ "$TEST_CASE" != compare-failed ] || return 1 + [ "$TEST_CASE" = source-mutated ] || cmp -s "$6" "$7" + ;; + *) return 65 ;; + esac + } + """; + var script = realScript.Replace(preparation, mock + "\n" + preparation, StringComparison.Ordinal); + File.WriteAllText(Path.Combine(output, "raw-recovery-mock-fixture.sh"), script); + var cases = new[] { "equal", "convert-failed", "compare-failed", "source-mutated" }; + foreach (var name in cases) + { + var folder = Path.Combine(output, "raw-recovery-" + name); + Directory.CreateDirectory(folder); + var input = Path.Combine(folder, "source.dmg"); var destination = Path.Combine(folder, "setup.dmg"); + var bytes = Encoding.UTF8.GetBytes("known already-patched Recovery content\n"); + File.WriteAllBytes(input, bytes); + var result = await Command("bash", ["-c", "TEST_CASE=\"$3\"\n" + script, "raw-recovery-fixture", input, destination, name], output, "raw-recovery-" + name, cancellation, requireSuccess: false); + var passed = name == "equal"; + if ((result.ExitCode == 0) != passed || passed && (!File.Exists(destination + ".raw") || !File.ReadAllBytes(destination + ".raw").SequenceEqual(bytes) || !File.ReadAllBytes(destination).SequenceEqual(bytes)) + || !passed && File.Exists(destination + ".raw")) + throw new InvalidOperationException("RAW Recovery preparation behavior failed: " + name); + } + var entry = File.ReadAllText(Path.Combine(output, "container-entry.sh")); + var selectionStart = entry.IndexOf("# BEGIN raw Recovery backend", StringComparison.Ordinal); + var selectionEnd = entry.IndexOf("# END raw Recovery backend", selectionStart, StringComparison.Ordinal); + if (selectionStart < 0 || selectionEnd < 0) throw new InvalidOperationException("Missing RAW drive selection boundary."); + var selection = entry[selectionStart..selectionEnd]; + var storage = Path.Combine(output, "raw-recovery-equal"); + var device = " -device virtio-blk-pci,drive=install,bus=pcie.0,iothread=io2 -drive file=/data.img,id=data3,format=raw"; + var originalDrive = " -drive file=" + storage + "/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none"; + var expectedDrive = " -drive file=" + storage + "/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none"; + var selectionCases = new[] { ("one", originalDrive + device, true), ("missing", device, false), ("duplicate", originalDrive + originalDrive + device, false) }; + foreach (var test in selectionCases) + { + var result = await Command("bash", ["-c", "set -Eeuo pipefail\nSTORAGE=\"$1\"\nDISK_OPTS=\"$2\"\nerror() { printf '%s\\n' \"$*\" >&2; }\n" + selection + "\nprintf '%s' \"$DISK_OPTS\"\n", "raw-recovery-selection", storage, test.Item2], output, "raw-recovery-selection-" + test.Item1, cancellation, requireSuccess: false); + if ((result.ExitCode == 0) != test.Item3 || test.Item3 && result.Output != expectedDrive + device) + throw new InvalidOperationException("RAW Recovery backend selection failed: " + test.Item1); + } + Save(Path.Combine(output, "raw-recovery-fixtures.json"), new { success = true, cases, selectionCases = selectionCases.Select(test => test.Item1), qemuBoundaryMocked = true, realQemuExecuted = false, guestExecuted = false }); + } + static (byte[] Binary, byte[] Manifest) ReadProbeAssets() { var folder = Path.Combine("tools", "ci", "native-process-probe"); @@ -1275,7 +1390,7 @@ static class NativeDiagnostic // Hash it once instead of rereading the image on every twenty-second poll. if ((logs.Output + stage.Output).Contains("[supported-profile] accelerator=tcg", StringComparison.Ordinal) && !File.Exists(Path.Combine(output, "guest-container-resources.last-success.json"))) - await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test -f /storage/14/setup.dmg && sha256sum /storage/14/setup.dmg || exit 1; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true); + await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test -f /storage/14/setup.dmg && sha256sum /storage/14/setup.dmg || exit 1; if test -f /storage/14/setup.dmg.raw.json; then printf '[RAW Recovery equality receipt]\n'; cat /storage/14/setup.dmg.raw.json; fi; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true); } static async Task CaptureDiskStack(string id, string output, string token, bool full, bool final, CancellationToken cancellation)