|
|
|
@@ -69,11 +69,13 @@ static class NativeDiagnostic
|
|
|
|
|
{
|
|
|
|
|
if (args.Length == 0 || args.Contains("--help"))
|
|
|
|
|
{
|
|
|
|
|
Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--output artifacts/native-macos] [--source existing-dockur-clone] [--compression-chunk readonly-qualified-chunk]");
|
|
|
|
|
Console.WriteLine("dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run|--cleanup|--validate [--full] [--recovery-format dmg|raw] [--output artifacts/native-macos] [--source existing-dockur-clone] [--compression-chunk readonly-qualified-chunk]");
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
var output = Path.GetFullPath(Option(args, "--output") ?? "artifacts/native-macos");
|
|
|
|
|
var full = args.Contains("--full");
|
|
|
|
|
var recoveryFormat = Option(args, "--recovery-format") ?? "dmg";
|
|
|
|
|
if (recoveryFormat is not ("dmg" or "raw")) throw new ArgumentException("Recovery format must be dmg or raw.");
|
|
|
|
|
if (args.Contains("--validate"))
|
|
|
|
|
{
|
|
|
|
|
ValidateContracts();
|
|
|
|
@@ -84,14 +86,16 @@ static class NativeDiagnostic
|
|
|
|
|
if (full) ValidateFullContracts();
|
|
|
|
|
if (Option(args, "--source") is { } source)
|
|
|
|
|
{
|
|
|
|
|
await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full);
|
|
|
|
|
await PrepareSource(Path.GetFullPath(source), output, full ? new string('0', 32) : "validation", false, CancellationToken.None, full, recoveryFormat);
|
|
|
|
|
if (recoveryFormat == "raw") await ValidateRawRecoveryFixture(output, CancellationToken.None);
|
|
|
|
|
await ValidateResourceRetention(output);
|
|
|
|
|
await ValidateRecoveryPatch(output);
|
|
|
|
|
await ValidateTcgPreflight(output, CancellationToken.None);
|
|
|
|
|
if (full) await ValidateDiskSerialParser(output);
|
|
|
|
|
Save(Path.Combine(output, "validation.json"), new
|
|
|
|
|
{
|
|
|
|
|
success = true, profile = Profile, mode = full ? "full" : "readiness",
|
|
|
|
|
success = true, profile = Profile, mode = full ? "full" : "readiness", recoveryFormat,
|
|
|
|
|
causalSingleVariableTest = recoveryFormat == "raw", comparisonBaseCommit = "2e2d702e294c39f24c6a3e44e5e94ff793d8774b",
|
|
|
|
|
helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))),
|
|
|
|
|
readinessSourceSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-readiness.sh"))),
|
|
|
|
|
udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))),
|
|
|
|
@@ -153,7 +157,7 @@ static class NativeDiagnostic
|
|
|
|
|
throw new InvalidOperationException("This diagnostic runs on the existing Linux/x64 runner only.");
|
|
|
|
|
ValidateContracts();
|
|
|
|
|
var sourceCommit = (await Command("git", ["rev-parse", "HEAD"], output, "candidate-commit", deadline.Token)).Output.Trim();
|
|
|
|
|
Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = Profile, causalSingleVariableTest = false, kvm = false, cpuModel = CpuModel, recoveryMajor = 14, cpuFlags = CpuFlags, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" });
|
|
|
|
|
Save(Path.Combine(output, "run-metadata.json"), new { token, startedUtc = DateTimeOffset.UtcNow, sourceCommit, dockurCommit = DockurCommit, profile = Profile, recoveryFormat, causalSingleVariableTest = recoveryFormat == "raw", comparisonBaseCommit = "2e2d702e294c39f24c6a3e44e5e94ff793d8774b", kvm = false, cpuModel = CpuModel, recoveryMajor = 14, cpuFlags = CpuFlags, runId = Environment.GetEnvironmentVariable("GITHUB_RUN_ID"), server = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"), architecture = RuntimeInformation.ProcessArchitecture.ToString(), deadlineMinutes, mode = full ? "full" : "readiness" });
|
|
|
|
|
var info = await Command("docker", ["info", "--format", "{{json .}}"], output, "docker-info", deadline.Token);
|
|
|
|
|
using (var document = JsonDocument.Parse(info.Output))
|
|
|
|
|
{
|
|
|
|
@@ -173,7 +177,7 @@ static class NativeDiagnostic
|
|
|
|
|
var actualCommit = (await Command("git", ["-C", source, "rev-parse", "HEAD"], output, "dockur-commit", deadline.Token)).Output.Trim();
|
|
|
|
|
if (actualCommit != DockurCommit) throw new InvalidOperationException("Dockur source pin mismatch.");
|
|
|
|
|
if (full) await PreparePayload(source, output, token, sourceCommit, deadline.Token);
|
|
|
|
|
await PrepareSource(source, output, token, true, deadline.Token, full);
|
|
|
|
|
await PrepareSource(source, output, token, true, deadline.Token, full, recoveryFormat);
|
|
|
|
|
await Command("docker", ["build", "--platform", "linux/amd64", "--label", OwnerLabel + "=" + token, "--tag", state.ImageTag, source], output, "docker-build", deadline.Token, echo: true);
|
|
|
|
|
var imageInspect = await Command("docker", ["image", "inspect", state.ImageTag], output, "image-inspect", deadline.Token);
|
|
|
|
|
using (var image = JsonDocument.Parse(imageInspect.Output))
|
|
|
|
@@ -385,7 +389,7 @@ static class NativeDiagnostic
|
|
|
|
|
return source.Remove(from, to + end.Length - from).Insert(from, originalSequence);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false)
|
|
|
|
|
static async Task PrepareSource(string source, string output, string token, bool writeSource, CancellationToken cancellation, bool full = false, string recoveryFormat = "dmg")
|
|
|
|
|
{
|
|
|
|
|
Directory.CreateDirectory(output);
|
|
|
|
|
var probe = ReadProbeAssets();
|
|
|
|
@@ -436,6 +440,11 @@ static class NativeDiagnostic
|
|
|
|
|
image = ReplaceOnce(image, " chmod 0755 \"$script\"\n", " chmod 0755 \"$script\" \"${script%/*}/native-process-probe-x86_64\"\n printf '%s\\n' '" + token + "' > \"${script%/*}/run.owner\" || return 1\n");
|
|
|
|
|
image = ReplaceOnce(image, " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n", " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cmp -s \"$IMAGE_TOOLS/recovery/readiness.sh\" \"$state/readiness.sh\" ||\n ! cmp -s \"$IMAGE_ASSETS/native-process-probe/native-process-probe-x86_64\" \"$state/native-process-probe-x86_64\" ||\n ! cmp -s \"$IMAGE_ASSETS/native-process-probe/build-manifest.json\" \"$state/native-process-probe-manifest.json\" ||\n");
|
|
|
|
|
image = ReplaceOnce(image, " if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n", " info \"[recovery-original] bytes=$(stat -c%s -- \"$image\") sha256=$(sha256sum \"$image\" | awk '{print $1}')\"\n if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n");
|
|
|
|
|
if (recoveryFormat == "raw")
|
|
|
|
|
{
|
|
|
|
|
image = ReplaceOnce(image, " if ! mv -f \"$source\" \"$dest\"; then\n error \"Failed to save automated recovery image to $dest.\"\n return 1\n fi\n", RawRecoveryPreparation);
|
|
|
|
|
entry = ReplaceOnce(entry, ". disk.sh # Initialize disks\n", ". disk.sh # Initialize disks\n" + RawRecoveryDriveSelection + "\n");
|
|
|
|
|
}
|
|
|
|
|
if (full)
|
|
|
|
|
{
|
|
|
|
|
await PrepareFullSource(source, output, token, writeSource, cancellation);
|
|
|
|
@@ -472,6 +481,112 @@ static class NativeDiagnostic
|
|
|
|
|
File.WriteAllBytes(Path.Combine(probeAssets, "build-manifest.json"), probe.Manifest);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static readonly string RawRecoveryPreparation = """
|
|
|
|
|
local raw="$dest.raw" pending="$dest.raw.tmp" source_hash after_hash raw_hash
|
|
|
|
|
[ ! -e "$raw" ] && [ ! -e "$pending" ] && [ ! -e "$raw.json" ] || {
|
|
|
|
|
error 'RAW Recovery output already exists.'; return 1;
|
|
|
|
|
}
|
|
|
|
|
source_hash=$(sha256sum "$source" | awk '{print $1}') || return 1
|
|
|
|
|
[[ "$source_hash" =~ ^[0-9a-f]{64}$ ]] || return 1
|
|
|
|
|
if ! qemu-img convert -f dmg -O raw "$source" "$pending" ||
|
|
|
|
|
! qemu-img compare -f dmg -F raw "$source" "$pending"; then
|
|
|
|
|
rm -f "$pending"
|
|
|
|
|
error 'RAW Recovery conversion or sector equality failed.'
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
after_hash=$(sha256sum "$source" | awk '{print $1}') || { rm -f "$pending"; return 1; }
|
|
|
|
|
raw_hash=$(sha256sum "$pending" | awk '{print $1}') || { rm -f "$pending"; return 1; }
|
|
|
|
|
if [ "$source_hash" != "$after_hash" ] || [[ ! "$raw_hash" =~ ^[0-9a-f]{64}$ ]]; then
|
|
|
|
|
rm -f "$pending"
|
|
|
|
|
error 'Patched DMG changed during RAW conversion.'
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
if ! mv -f "$source" "$dest" || ! mv -f "$pending" "$raw"; then
|
|
|
|
|
rm -f "$pending"
|
|
|
|
|
error 'Failed to retain the patched DMG and equivalent RAW Recovery.'
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
printf '{"sourceFormat":"dmg","targetFormat":"raw","sectorEqualityVerified":true,"sourceUnchanged":true,"sourceSha256":"%s","rawSha256":"%s"}\n' "$source_hash" "$raw_hash" > "$raw.json" || return 1
|
|
|
|
|
info "[recovery-raw] sector-equality=true source-unchanged=true source-sha256=$source_hash raw-sha256=$raw_hash"
|
|
|
|
|
""" + "\n";
|
|
|
|
|
|
|
|
|
|
static readonly string RawRecoveryDriveSelection = """
|
|
|
|
|
# BEGIN raw Recovery backend
|
|
|
|
|
raw_recovery_from="file=$STORAGE/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none"
|
|
|
|
|
raw_recovery_to="file=$STORAGE/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none"
|
|
|
|
|
[[ -s "$STORAGE/setup.dmg.raw" && -s "$STORAGE/setup.dmg.raw.json" && "$DISK_OPTS" == *"$raw_recovery_from"* ]] || {
|
|
|
|
|
error 'Verified RAW Recovery or expected DMG backend is missing.'; exit 34;
|
|
|
|
|
}
|
|
|
|
|
raw_recovery_tail=${DISK_OPTS#*"$raw_recovery_from"}
|
|
|
|
|
[[ "$raw_recovery_tail" != *"$raw_recovery_from"* ]] || { error 'Recovery backend is duplicated.'; exit 34; }
|
|
|
|
|
DISK_OPTS=${DISK_OPTS/"$raw_recovery_from"/"$raw_recovery_to"}
|
|
|
|
|
unset raw_recovery_from raw_recovery_to raw_recovery_tail
|
|
|
|
|
# END raw Recovery backend
|
|
|
|
|
""";
|
|
|
|
|
|
|
|
|
|
static async Task ValidateRawRecoveryFixture(string output, CancellationToken cancellation)
|
|
|
|
|
{
|
|
|
|
|
var image = File.ReadAllText(Path.Combine(output, "image.sh.patched"));
|
|
|
|
|
var begin = image.IndexOf("prepareAutomatedRecovery() {", StringComparison.Ordinal);
|
|
|
|
|
var end = image.IndexOf("\nreturn 0\n", begin, StringComparison.Ordinal);
|
|
|
|
|
if (begin < 0 || end < 0) throw new InvalidOperationException("Missing Recovery preparation boundary.");
|
|
|
|
|
var preparation = image[begin..end];
|
|
|
|
|
var realScript = "#!/bin/bash\nset -Eeuo pipefail\ninfo() { printf '%s\\n' \"$*\"; }\nhtml() { :; }\nerror() { printf '%s\\n' \"$*\" >&2; }\npatchRecoveryBootstrap() { :; }\n" + preparation + "\nprepareAutomatedRecovery \"$1\" \"$2\"\n";
|
|
|
|
|
File.WriteAllText(Path.Combine(output, "raw-recovery-real-qemu-fixture.sh"), realScript);
|
|
|
|
|
var mock = """
|
|
|
|
|
qemu-img() {
|
|
|
|
|
case "$1" in
|
|
|
|
|
info) printf '%s\n' '{"format":"dmg"}' ;;
|
|
|
|
|
convert)
|
|
|
|
|
[ "$2 $3 $4 $5" = '-f dmg -O raw' ] || return 65
|
|
|
|
|
[ "$TEST_CASE" != convert-failed ] || return 1
|
|
|
|
|
cp "$6" "$7" || return 1
|
|
|
|
|
[ "$TEST_CASE" != source-mutated ] || printf 'mutation' >> "$6"
|
|
|
|
|
;;
|
|
|
|
|
compare)
|
|
|
|
|
[ "$2 $3 $4 $5" = '-f dmg -F raw' ] || return 65
|
|
|
|
|
[ "$TEST_CASE" != compare-failed ] || return 1
|
|
|
|
|
[ "$TEST_CASE" = source-mutated ] || cmp -s "$6" "$7"
|
|
|
|
|
;;
|
|
|
|
|
*) return 65 ;;
|
|
|
|
|
esac
|
|
|
|
|
}
|
|
|
|
|
""";
|
|
|
|
|
var script = realScript.Replace(preparation, mock + "\n" + preparation, StringComparison.Ordinal);
|
|
|
|
|
File.WriteAllText(Path.Combine(output, "raw-recovery-mock-fixture.sh"), script);
|
|
|
|
|
var cases = new[] { "equal", "convert-failed", "compare-failed", "source-mutated" };
|
|
|
|
|
foreach (var name in cases)
|
|
|
|
|
{
|
|
|
|
|
var folder = Path.Combine(output, "raw-recovery-" + name);
|
|
|
|
|
Directory.CreateDirectory(folder);
|
|
|
|
|
var input = Path.Combine(folder, "source.dmg"); var destination = Path.Combine(folder, "setup.dmg");
|
|
|
|
|
var bytes = Encoding.UTF8.GetBytes("known already-patched Recovery content\n");
|
|
|
|
|
File.WriteAllBytes(input, bytes);
|
|
|
|
|
var result = await Command("bash", ["-c", "TEST_CASE=\"$3\"\n" + script, "raw-recovery-fixture", input, destination, name], output, "raw-recovery-" + name, cancellation, requireSuccess: false);
|
|
|
|
|
var passed = name == "equal";
|
|
|
|
|
if ((result.ExitCode == 0) != passed || passed && (!File.Exists(destination + ".raw") || !File.ReadAllBytes(destination + ".raw").SequenceEqual(bytes) || !File.ReadAllBytes(destination).SequenceEqual(bytes))
|
|
|
|
|
|| !passed && File.Exists(destination + ".raw"))
|
|
|
|
|
throw new InvalidOperationException("RAW Recovery preparation behavior failed: " + name);
|
|
|
|
|
}
|
|
|
|
|
var entry = File.ReadAllText(Path.Combine(output, "container-entry.sh"));
|
|
|
|
|
var selectionStart = entry.IndexOf("# BEGIN raw Recovery backend", StringComparison.Ordinal);
|
|
|
|
|
var selectionEnd = entry.IndexOf("# END raw Recovery backend", selectionStart, StringComparison.Ordinal);
|
|
|
|
|
if (selectionStart < 0 || selectionEnd < 0) throw new InvalidOperationException("Missing RAW drive selection boundary.");
|
|
|
|
|
var selection = entry[selectionStart..selectionEnd];
|
|
|
|
|
var storage = Path.Combine(output, "raw-recovery-equal");
|
|
|
|
|
var device = " -device virtio-blk-pci,drive=install,bus=pcie.0,iothread=io2 -drive file=/data.img,id=data3,format=raw";
|
|
|
|
|
var originalDrive = " -drive file=" + storage + "/setup.dmg,id=install,format=dmg,cache=unsafe,readonly=on,if=none";
|
|
|
|
|
var expectedDrive = " -drive file=" + storage + "/setup.dmg.raw,id=install,format=raw,cache=unsafe,readonly=on,if=none";
|
|
|
|
|
var selectionCases = new[] { ("one", originalDrive + device, true), ("missing", device, false), ("duplicate", originalDrive + originalDrive + device, false) };
|
|
|
|
|
foreach (var test in selectionCases)
|
|
|
|
|
{
|
|
|
|
|
var result = await Command("bash", ["-c", "set -Eeuo pipefail\nSTORAGE=\"$1\"\nDISK_OPTS=\"$2\"\nerror() { printf '%s\\n' \"$*\" >&2; }\n" + selection + "\nprintf '%s' \"$DISK_OPTS\"\n", "raw-recovery-selection", storage, test.Item2], output, "raw-recovery-selection-" + test.Item1, cancellation, requireSuccess: false);
|
|
|
|
|
if ((result.ExitCode == 0) != test.Item3 || test.Item3 && result.Output != expectedDrive + device)
|
|
|
|
|
throw new InvalidOperationException("RAW Recovery backend selection failed: " + test.Item1);
|
|
|
|
|
}
|
|
|
|
|
Save(Path.Combine(output, "raw-recovery-fixtures.json"), new { success = true, cases, selectionCases = selectionCases.Select(test => test.Item1), qemuBoundaryMocked = true, realQemuExecuted = false, guestExecuted = false });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static (byte[] Binary, byte[] Manifest) ReadProbeAssets()
|
|
|
|
|
{
|
|
|
|
|
var folder = Path.Combine("tools", "ci", "native-process-probe");
|
|
|
|
@@ -1275,7 +1390,7 @@ static class NativeDiagnostic
|
|
|
|
|
// Hash it once instead of rereading the image on every twenty-second poll.
|
|
|
|
|
if ((logs.Output + stage.Output).Contains("[supported-profile] accelerator=tcg", StringComparison.Ordinal)
|
|
|
|
|
&& !File.Exists(Path.Combine(output, "guest-container-resources.last-success.json")))
|
|
|
|
|
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test -f /storage/14/setup.dmg && sha256sum /storage/14/setup.dmg || exit 1; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true);
|
|
|
|
|
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test -f /storage/14/setup.dmg && sha256sum /storage/14/setup.dmg || exit 1; if test -f /storage/14/setup.dmg.raw.json; then printf '[RAW Recovery equality receipt]\n'; cat /storage/14/setup.dmg.raw.json; fi; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static async Task CaptureDiskStack(string id, string output, string token, bool full, bool final, CancellationToken cancellation)
|
|
|
|
|
dh