forked from Manuel/meeting-assistant
prepare optional RAW Recovery backend for full KVM NoAVX CI
This commit is contained in:
dh committed
1 parent
5f686c5c80
commit
d1594e90b3
4 files changed
+152
-9
No files matched your search
@@ -62,11 +62,13 @@ static class NativeDiagnostic
|
||||
{
|
||||
if (args.Length == 0 || args.Contains("--help"))
|
||||
{
|
||||
Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--output artifacts/native-macos] [--source existing-dockur-clone] [--cryptex-archive verified-release.zip] [--noavx-archive verified-upstream.zip] [--compression-chunk readonly-qualified-chunk]");
|
||||
Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--recovery-format dmg|raw] [--output artifacts/native-macos] [--source existing-dockur-clone] [--cryptex-archive verified-release.zip] [--noavx-archive verified-upstream.zip] [--compression-chunk readonly-qualified-chunk]");
|
||||
return 0;
|
||||
}
|
||||
var output = Path.GetFullPath(Option(args, "--output") ?? "artifacts/native-macos");
|
||||
var full = args.Contains("--full");
|
||||
var recoveryFormat = Option(args, "--recovery-format") ?? "dmg";
|
||||
if (recoveryFormat is not ("dmg" or "raw")) throw new ArgumentException("Recovery format must be dmg or raw.");
|
||||
if (args.Contains("--validate"))
|
||||
{
|
||||
ValidateRunnerMemoryGate();
|
||||
@@ -75,12 +77,13 @@ static class NativeDiagnostic
|
||||
if (full) ValidateFullContracts();
|
||||
if (Option(args, "--source") is { } source)
|
||||
{
|
||||
await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full, Option(args, "--cryptex-archive"), Option(args, "--noavx-archive"));
|
||||
await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full, Option(args, "--cryptex-archive"), Option(args, "--noavx-archive"), recoveryFormat);
|
||||
if (recoveryFormat == "raw") await ValidateRawRecoveryFixture(output, CancellationToken.None);
|
||||
ValidateNoAvxStaging(output);
|
||||
ValidateNoAvxRejections(output);
|
||||
await ValidateResourceRetention(output);
|
||||
await ValidateRecoveryPatch(output);
|
||||
Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex-noavx", mode = full ? "full" : "readiness", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, noAvxArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
|
||||
Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex-noavx", mode = full ? "full" : "readiness", recoveryFormat, causalSingleVariableTest = recoveryFormat == "raw", comparisonBaselineCommit = "5f686c5c80b6bbb525745de173b57c6393f58bf0", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, noAvxArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
|
||||
}
|
||||
if (full) await ValidateDiskSerialParser(output);
|
||||
if (Option(args, "--compression-chunk") is { } chunk)
|
||||
@@ -114,7 +117,7 @@ static class NativeDiagnostic
|
||||
throw new InvalidOperationException("This diagnostic runs on the existing Linux/x64 runner only.");
|
||||
ValidateContracts();
|
||||
var sourceCommit = (await Command("git", ["rev-parse", "HEAD"], output, "candidate-commit", deadline.Token)).Output.Trim();
|
||||
Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = "kvm-host-ventura-cryptex-noavx", causalSingleVariableTest = true, comparisonBaselineCommit = "a356b88ae4a0a26d68098460df86038f9831c080", kvm = true, cpuModel = "host", recoveryMajor = 13, cryptexVersion = "1.0.5", liluVersion = "1.7.1", noAvxBaseVersion = "12.6", noAvxSha256 = NoAvxHash, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" });
|
||||
Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = "kvm-host-ventura-cryptex-noavx", recoveryFormat, causalSingleVariableTest = recoveryFormat == "raw", comparisonBaselineCommit = "5f686c5c80b6bbb525745de173b57c6393f58bf0", changedGuestVariable = recoveryFormat == "raw" ? "recovery-disk-backend" : "none", kvm = true, cpuModel = "host", recoveryMajor = 13, cryptexVersion = "1.0.5", liluVersion = "1.7.1", noAvxBaseVersion = "12.6", noAvxSha256 = NoAvxHash, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" });
|
||||
var info = await Command("docker", ["info", "--format", "{{json .}}"], output, "docker-info", deadline.Token);
|
||||
using (var document = JsonDocument.Parse(info.Output))
|
||||
{
|
||||
@@ -133,7 +136,7 @@ static class NativeDiagnostic
|
||||
var actualCommit = (await Command("git", ["-C", source, "rev-parse", "HEAD"], output, "dockur-commit", deadline.Token)).Output.Trim();
|
||||
if (actualCommit != DockurCommit) throw new InvalidOperationException("Dockur source pin mismatch.");
|
||||
if (full) await PreparePayload(source, output, token, sourceCommit, deadline.Token);
|
||||
await PrepareSource(source, output, token, true, deadline.Token, full, Option(args, "--cryptex-archive"), Option(args, "--noavx-archive"));
|
||||
await PrepareSource(source, output, token, true, deadline.Token, full, Option(args, "--cryptex-archive"), Option(args, "--noavx-archive"), recoveryFormat);
|
||||
await Command("docker", ["build", "--platform", "linux/amd64", "--label", OwnerLabel + "=" + token, "--tag", state.ImageTag, source], output, "docker-build", deadline.Token, echo: true);
|
||||
var imageInspect = await Command("docker", ["image", "inspect", state.ImageTag], output, "image-inspect", deadline.Token);
|
||||
using (var image = JsonDocument.Parse(imageInspect.Output))
|
||||
@@ -273,7 +276,7 @@ static class NativeDiagnostic
|
||||
}
|
||||
}
|
||||
|
||||
static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false, string? cryptexArchive = null, string? noAvxArchive = null)
|
||||
static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false, string? cryptexArchive = null, string? noAvxArchive = null, string recoveryFormat = "dmg")
|
||||
{
|
||||
Directory.CreateDirectory(output);
|
||||
var patchPath = Path.Combine(source, "src/install/recovery/patch.py");
|
||||
@@ -306,6 +309,11 @@ static class NativeDiagnostic
|
||||
var image = ReplaceOnce(originalImage, " if ! cp -f \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\"; then\n", " if ! cp -f \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cp -f \"$IMAGE_TOOLS/recovery/readiness.sh\" \"${script%/*}/readiness.sh\"; then\n");
|
||||
image = ReplaceOnce(image, " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n", " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cmp -s \"$IMAGE_TOOLS/recovery/readiness.sh\" \"$state/readiness.sh\" ||\n");
|
||||
image = ReplaceOnce(image, " if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n", " info \"[recovery-original] bytes=$(stat -c%s -- \"$image\") sha256=$(sha256sum \"$image\" | awk '{print $1}')\"\n if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n");
|
||||
if (recoveryFormat == "raw")
|
||||
{
|
||||
image = ReplaceOnce(image, " if ! mv -f \"$source\" \"$dest\"; then\n error \"Failed to save automated recovery image to $dest.\"\n return 1\n fi\n", RawRecoveryPreparation);
|
||||
entry = ReplaceOnce(entry, ". disk.sh # Initialize disks\n", ". disk.sh # Initialize disks\n" + RawRecoveryDriveSelection + "\n");
|
||||
}
|
||||
if (full)
|
||||
{
|
||||
await PrepareFullSource(source, output, token, writeSource, cancellation);
|
||||
@@ -385,6 +393,134 @@ static class NativeDiagnostic
|
||||
return string.Join("\n", lines.Select(line => new string(' ', spaces) + (line.Length > 0 ? line[common..] : "")));
|
||||
}
|
||||
|
||||
static readonly string RawRecoveryPreparation = """
|
||||
local raw="$dest.raw" pending="$dest.raw.tmp" source_hash after_hash raw_hash cache_before cache_after
|
||||
[ ! -e "$raw" ] && [ ! -e "$pending" ] && [ ! -e "$raw.json" ] || {
|
||||
error 'RAW Recovery output already exists.'; return 1;
|
||||
}
|
||||
source_hash=$(sha256sum "$source" | awk '{print $1}') || return 1
|
||||
[[ "$source_hash" =~ ^[0-9a-f]{64}$ ]] || return 1
|
||||
if ! qemu-img convert -f dmg -O raw "$source" "$pending" ||
|
||||
! qemu-img compare -f dmg -F raw "$source" "$pending"; then
|
||||
rm -f "$pending"
|
||||
error 'RAW Recovery conversion or sector equality failed.'
|
||||
return 1
|
||||
fi
|
||||
after_hash=$(sha256sum "$source" | awk '{print $1}') || { rm -f "$pending"; return 1; }
|
||||
raw_hash=$(sha256sum "$pending" | awk '{print $1}') || { rm -f "$pending"; return 1; }
|
||||
if [ "$source_hash" != "$after_hash" ] || [[ ! "$raw_hash" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
rm -f "$pending"
|
||||
error 'Patched DMG changed during RAW conversion.'
|
||||
return 1
|
||||
fi
|
||||
# These verified owned files are not guest RAM. Release only their
|
||||
# persisted data cache before Dockur computes its cgroup RAM allowance.
|
||||
cache_before=$(cat /sys/fs/cgroup/memory.current 2>/dev/null || printf unavailable)
|
||||
if ! dd of="$pending" oflag=nocache conv=nocreat,notrunc,fdatasync count=0 status=none ||
|
||||
! dd of="$source" oflag=nocache conv=nocreat,notrunc,fdatasync count=0 status=none; then
|
||||
rm -f "$pending"
|
||||
error 'Failed to release the verified Recovery file caches before RAM admission.'
|
||||
return 1
|
||||
fi
|
||||
cache_after=$(cat /sys/fs/cgroup/memory.current 2>/dev/null || printf unavailable)
|
||||
info "[recovery-raw-cache] files=two-owned memory-current-before=$cache_before memory-current-after=$cache_after"
|
||||
if ! mv -f "$source" "$dest" || ! mv -f "$pending" "$raw"; then
|
||||
rm -f "$pending"
|
||||
error 'Failed to retain the patched DMG and equivalent RAW Recovery.'
|
||||
return 1
|
||||
fi
|
||||
printf '{"sourceFormat":"dmg","targetFormat":"raw","sectorEqualityVerified":true,"sourceUnchanged":true,"sourceSha256":"%s","rawSha256":"%s"}\n' "$source_hash" "$raw_hash" > "$raw.json" || return 1
|
||||
info "[recovery-raw] sector-equality=true source-unchanged=true source-sha256=$source_hash raw-sha256=$raw_hash"
|
||||
""" + "\n";
|
||||
|
||||
static readonly string RawRecoveryDriveSelection = """
|
||||
# BEGIN raw Recovery backend
|
||||
raw_recovery_from="file=$STORAGE/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none"
|
||||
raw_recovery_to="file=$STORAGE/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none"
|
||||
[[ -s "$STORAGE/setup.dmg.raw" && -s "$STORAGE/setup.dmg.raw.json" && "$DISK_OPTS" == *"$raw_recovery_from"* ]] || {
|
||||
error 'Verified RAW Recovery or expected DMG backend is missing.'; exit 34;
|
||||
}
|
||||
raw_recovery_tail=${DISK_OPTS#*"$raw_recovery_from"}
|
||||
[[ "$raw_recovery_tail" != *"$raw_recovery_from"* ]] || { error 'Recovery backend is duplicated.'; exit 34; }
|
||||
DISK_OPTS=${DISK_OPTS/"$raw_recovery_from"/"$raw_recovery_to"}
|
||||
unset raw_recovery_from raw_recovery_to raw_recovery_tail
|
||||
# END raw Recovery backend
|
||||
""";
|
||||
|
||||
static async Task ValidateRawRecoveryFixture(string output, CancellationToken cancellation)
|
||||
{
|
||||
var image = File.ReadAllText(Path.Combine(output, "image.sh.patched"));
|
||||
var begin = image.IndexOf("prepareAutomatedRecovery() {", StringComparison.Ordinal);
|
||||
var end = image.IndexOf("\nreturn 0\n", begin, StringComparison.Ordinal);
|
||||
if (begin < 0 || end < 0) throw new InvalidOperationException("Missing Recovery preparation boundary.");
|
||||
var preparation = image[begin..end];
|
||||
var realScript = "#!/bin/bash\nset -Eeuo pipefail\ninfo() { printf '%s\\n' \"$*\"; }\nhtml() { :; }\nerror() { printf '%s\\n' \"$*\" >&2; }\npatchRecoveryBootstrap() { :; }\n" + preparation + "\nprepareAutomatedRecovery \"$1\" \"$2\"\n";
|
||||
File.WriteAllText(Path.Combine(output, "raw-recovery-real-qemu-fixture.sh"), realScript);
|
||||
var mock = """
|
||||
qemu-img() {
|
||||
case "$1" in
|
||||
info) printf '%s\n' '{"format":"dmg"}' ;;
|
||||
convert)
|
||||
[ "$2 $3 $4 $5" = '-f dmg -O raw' ] || return 65
|
||||
[ "$TEST_CASE" != convert-failed ] || return 1
|
||||
cp "$6" "$7" || return 1
|
||||
[ "$TEST_CASE" != source-mutated ] || printf 'mutation' >> "$6"
|
||||
;;
|
||||
compare)
|
||||
[ "$2 $3 $4 $5" = '-f dmg -F raw' ] || return 65
|
||||
[ "$TEST_CASE" != compare-failed ] || return 1
|
||||
[ "$TEST_CASE" = source-mutated ] || cmp -s "$6" "$7"
|
||||
;;
|
||||
*) return 65 ;;
|
||||
esac
|
||||
}
|
||||
dd() {
|
||||
[ "$2 $3 $4 $5" = 'oflag=nocache conv=nocreat,notrunc,fdatasync count=0 status=none' ] || return 65
|
||||
case "$1" in
|
||||
"of=$pending") [ "$TEST_CASE" != raw-cache-failed ] || return 1 ;;
|
||||
"of=$source") [ "$TEST_CASE" != source-cache-failed ] || return 1 ;;
|
||||
*) return 65 ;;
|
||||
esac
|
||||
printf '%s\n' "$1" >> "$dest.cache-eviction"
|
||||
}
|
||||
""";
|
||||
var script = realScript.Replace(preparation, mock + "\n" + preparation, StringComparison.Ordinal);
|
||||
File.WriteAllText(Path.Combine(output, "raw-recovery-mock-fixture.sh"), script);
|
||||
var cases = new[] { "equal", "convert-failed", "compare-failed", "source-mutated", "raw-cache-failed", "source-cache-failed" };
|
||||
foreach (var name in cases)
|
||||
{
|
||||
var folder = Path.Combine(output, "raw-recovery-" + name);
|
||||
Directory.CreateDirectory(folder);
|
||||
var input = Path.Combine(folder, "source.dmg"); var destination = Path.Combine(folder, "setup.dmg");
|
||||
var bytes = Encoding.UTF8.GetBytes("known already-patched Recovery content\n");
|
||||
File.WriteAllBytes(input, bytes);
|
||||
var result = await Command("bash", ["-c", "TEST_CASE=\"$3\"\n" + script, "raw-recovery-fixture", input, destination, name], output, "raw-recovery-" + name, cancellation, requireSuccess: false);
|
||||
var passed = name == "equal";
|
||||
if ((result.ExitCode == 0) != passed || passed && (!File.Exists(destination + ".raw") || !File.ReadAllBytes(destination + ".raw").SequenceEqual(bytes) || !File.ReadAllBytes(destination).SequenceEqual(bytes))
|
||||
|| !passed && File.Exists(destination + ".raw"))
|
||||
throw new InvalidOperationException("RAW Recovery preparation behavior failed: " + name);
|
||||
if (passed && (!File.Exists(destination + ".cache-eviction") || !File.ReadAllLines(destination + ".cache-eviction").SequenceEqual(new[] { "of=" + destination + ".raw.tmp", "of=" + input })))
|
||||
throw new InvalidOperationException("RAW Recovery must release only the two verified files' caches before retaining them.");
|
||||
}
|
||||
var entry = File.ReadAllText(Path.Combine(output, "container-entry.sh"));
|
||||
var selectionStart = entry.IndexOf("# BEGIN raw Recovery backend", StringComparison.Ordinal);
|
||||
var selectionEnd = entry.IndexOf("# END raw Recovery backend", selectionStart, StringComparison.Ordinal);
|
||||
if (selectionStart < 0 || selectionEnd < 0) throw new InvalidOperationException("Missing RAW drive selection boundary.");
|
||||
var selection = entry[selectionStart..selectionEnd];
|
||||
var storage = Path.Combine(output, "raw-recovery-equal");
|
||||
var device = " -device virtio-blk-pci,drive=install,bus=pcie.0,iothread=io2 -drive file=/data.img,id=data3,format=raw";
|
||||
var originalDrive = " -drive file=" + storage + "/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none";
|
||||
var expectedDrive = " -drive file=" + storage + "/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none";
|
||||
var selectionCases = new[] { ("one", originalDrive + device, true), ("missing", device, false), ("duplicate", originalDrive + originalDrive + device, false) };
|
||||
foreach (var test in selectionCases)
|
||||
{
|
||||
var result = await Command("bash", ["-c", "set -Eeuo pipefail\nSTORAGE=\"$1\"\nDISK_OPTS=\"$2\"\nerror() { printf '%s\\n' \"$*\" >&2; }\n" + selection + "\nprintf '%s' \"$DISK_OPTS\"\n", "raw-recovery-selection", storage, test.Item2], output, "raw-recovery-selection-" + test.Item1, cancellation, requireSuccess: false);
|
||||
if ((result.ExitCode == 0) != test.Item3 || test.Item3 && result.Output != expectedDrive + device)
|
||||
throw new InvalidOperationException("RAW Recovery backend selection failed: " + test.Item1);
|
||||
}
|
||||
Save(Path.Combine(output, "raw-recovery-fixtures.json"), new { success = true, cases, selectionCases = selectionCases.Select(test => test.Item1), qemuBoundaryMocked = true, realQemuExecuted = false, guestExecuted = false });
|
||||
}
|
||||
|
||||
static async Task<(string Boot, string Config, string Assets)> PrepareCompatibility(string source, string output, string? archivePath, string? noAvxArchivePath, CancellationToken cancellation)
|
||||
{
|
||||
var boot = File.ReadAllText(Path.Combine(source, "src", "boot.sh"));
|
||||
@@ -1138,7 +1274,7 @@ static class NativeDiagnostic
|
||||
await Command("docker", ["cp", id + ":" + FullState + "/guest-logs/.", Path.Combine(output, "guest-logs")], output, "capture-guest-logs", cancellation, requireSuccess: false);
|
||||
}
|
||||
}
|
||||
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test ! -f /storage/13/setup.dmg || sha256sum /storage/13/setup.dmg; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true);
|
||||
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test ! -f /storage/13/setup.dmg || sha256sum /storage/13/setup.dmg; if test -f /storage/13/setup.dmg.raw.json; then printf '[RAW Recovery equality receipt]\n'; cat /storage/13/setup.dmg.raw.json; fi; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true);
|
||||
}
|
||||
|
||||
static async Task CaptureMonitor(string id, string output, string token, CancellationToken cancellation)
|
||||
|
||||
Reference in new issue
Block a user