Daniel/meeting-assistant
1
0
Fork 0
forked from Manuel/meeting-assistant
Code Pull Requests Actions Activity Open in Coder

ci: patch known Recovery variants and preserve UDIF checksums
PR and Push Build/Test / build-and-test (push) Failing after 18m28s
Details
PR and Push Build/Test / portable-build-and-test (push) Successful in 6m35s
Details

Browse Source
This commit is contained in:
dh
2026-10-03 21:53:38 +02:00
parent 45d7bde71f
commit 94a70b2005
3 changed files with 423 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ci/MacOsNativeDiagnostic.cs
+233 -11
View File
@@ -1,5 +1,6 @@
#:property PublishAot=false
using System.Diagnostics;
using System.Buffers.Binary;
using System.IO.Compression;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
@@ -16,6 +17,7 @@ static class NativeDiagnostic
const string CryptexUrl = "https://github.com/acidanthera/CryptexFixup/releases/download/1.0.5/CryptexFixup-1.0.5-RELEASE.zip";
const string CryptexHash = "25041d94a0fe9a0261caf0ba89b36dfcb21682bf3c697a34bcaddc839576ab30";
const string OpenCoreTemplateHash = "287328995d4198f1b05166f087d85bf7ef66bedafe150d17ad112ac8de60051d";
const string UdifChecksumBindingHash = "6109d04619e800c483fdac363d593cd1cd69f34131d2521417334e11d41c8bfa";
const string OwnerLabel = "org.meeting-assistant.native-diagnostic";
const long GuestDiskBytes = 64L * 1024 * 1024 * 1024;
const long ContainerMemoryBytes = 6L * 1024 * 1024 * 1024;
@@ -46,6 +48,10 @@ static class NativeDiagnostic
static readonly string DiagnosticDaemon = (OriginalDaemon + "\n")
.Replace("<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n", "", StringComparison.Ordinal)
.Replace("\t\t<string>/usr/libexec/recoveryosd</string>", "\t\t<string>/bin/bash</string>\n\t\t<string>/Volumes/installstate/launch.sh</string>", StringComparison.Ordinal);
// Exact XML framing read from the Apple 13 comparison download, not an assertion
// about the unretained bytes downloaded by run 4173.
static readonly string OriginalDaemon13 = ReplaceOnce(OriginalDaemon + "\n", "<string>App</string>", "<string>Interactive</string>");
static readonly string DiagnosticDaemon13 = ReplaceOnce(DiagnosticDaemon, "<string>App</string>", "<string>Interactive</string>");
public static async Task<int> Execute(string[] args)
{
@@ -61,7 +67,9 @@ static class NativeDiagnostic
if (Option(args, "--source") is { } source)
{
await PrepareSource(Path.GetFullPath(source), output, "validation", false, Option(args, "--cryptex-archive"), CancellationToken.None);
Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
await ValidateResourceRetention(output);
await ValidateRecoveryPatch(output);
Save(Path.Combine(output, "validation.json"), new { success = true, profile = "kvm-host-ventura-cryptex", helperSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "MacOsNativeDiagnostic.cs"))), udifChecksumBindingSha256 = Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"))), baselineReadinessNormalized = true, resultNegativeCases = 6, containerNegativeCases = 11, recoveryPositiveCases = 4, recoveryNegativeCases = 12, independentFixtureCrc32Readback = true, resourceSnapshotRetention = true, cryptexArchiveVerified = true, configurationAndStagingContractsVerified = true, templateIsoDownloaded = false, activeLiluRuntimeChecked = false, sourceModified = false, dockerExecuted = false, guestExecuted = false, completedUtc = DateTimeOffset.UtcNow });
}
Console.WriteLine("Source patch contracts and diagnostic result validation passed; no Docker or guest execution occurred.");
return 0;
@@ -181,8 +189,13 @@ static class NativeDiagnostic
throw new InvalidOperationException("Compatibility readiness may change only the baseline's macOS minimum to 13; identity, services, disk and limits must remain identical.");
if (Hash(File.ReadAllBytes(Path.Combine("tools", "ci", "macos-native-bootstrap.sh"))) != "94f069e116fdc7685a4d233cab6fa50df9f39274386bb82157674061e74fadb5")
throw new InvalidOperationException("Compatibility profile must preserve the baseline Apple recoveryosd wrapper.");
XDocument.Parse(DiagnosticDaemon);
if (Encoding.UTF8.GetByteCount(DiagnosticDaemon) > Encoding.UTF8.GetByteCount(OriginalDaemon + "\n")) throw new InvalidOperationException("Daemon replacement exceeds original file.");
if (Hash(Encoding.UTF8.GetBytes(OriginalDaemon13)) != "af9d7f6c1948079bd4384d27b6882678d6fb4e338fcf6a8be8f84fceef174ad6") throw new InvalidOperationException("macOS 13 allowlist bytes differ from the independently read comparison plist.");
foreach (var variant in new[] { (OriginalDaemon + "\n", DiagnosticDaemon, "App"), (OriginalDaemon13, DiagnosticDaemon13, "Interactive") })
{
ValidateDaemon(variant.Item1, variant.Item3, false);
ValidateDaemon(variant.Item2, variant.Item3, true);
if (Encoding.UTF8.GetByteCount(variant.Item2) > Encoding.UTF8.GetByteCount(variant.Item1)) throw new InvalidOperationException("Daemon replacement exceeds original file.");
}
var good = JsonSerializer.Serialize(new { token = "validation", success = true, osVersion = "13.6.1", architecture = "x86_64", uid = 0, disk = "/dev/disk1", diskBytes = GuestDiskBytes, readOnly = false, systemExit = 0, diskArbitrationExit = 0, recoveryExit = 0, diskListExit = 0 });
ValidateResult(good, "validation");
foreach (var invalid in new[] { good.Replace("13.6.1", "12.6.1"), good.Replace("x86_64", "arm64"), good.Replace("\"readOnly\":false", "\"readOnly\":true"), good.Replace("\"success\":true", "\"success\":false"), good.Replace("68719476736", "17179869184"), good.Replace("validation", "stale") })
@@ -207,11 +220,11 @@ static class NativeDiagnostic
var patchPath = Path.Combine(source, "src/install/recovery/patch.py");
var originalPatch = File.ReadAllText(patchPath);
if (Hash(Encoding.UTF8.GetBytes(originalPatch)) != "84f13db88c02edbf5ce21a39571fe58f12bebf5b0886c2d012f16ddbaed45323") throw new InvalidOperationException("Pinned Recovery patcher hash mismatch.");
var patch = ReplaceOnce(originalPatch, OriginalBootstrap, MountOnlyBootstrap);
var oldConstants = "RECOVERY_ORIGINAL = b\"/usr/libexec/recoveryosd\"\nRECOVERY_REPLACEMENT = b\"/private/etc/rc.cdrom.sh\"";
var daemon = OriginalDaemon + "\n";
var constants = "RECOVERY_ORIGINAL = b'''" + daemon + "'''\nRECOVERY_REPLACEMENT = b'''" + DiagnosticDaemon + "'''.ljust(len(RECOVERY_ORIGINAL), b\" \")";
patch = ReplaceOnce(patch, oldConstants, constants);
var patch = PrepareRecoveryPatch(originalPatch);
var checksumBinding = File.ReadAllText(Path.Combine("tools", "ci", "macos-native-udif-checksums.py"));
if (Hash(Encoding.UTF8.GetBytes(checksumBinding)) != UdifChecksumBindingHash) throw new InvalidOperationException("Recovery UDIF checksum binding hash mismatch.");
File.WriteAllText(Path.Combine(output, "udif_checksums.py"), checksumBinding, new UTF8Encoding(false));
var dockerPath = Path.Combine(source, "Dockerfile");
if (Hash(File.ReadAllBytes(dockerPath)) != "a0e804235967400eb70e755d63eff8a33a7761922ddd6e9723faa8e828fd8aa3") throw new InvalidOperationException("Pinned Dockerfile hash mismatch.");
// The existing runner's BuildKit cannot checksum dangling manpage links during COPY /.
@@ -232,9 +245,10 @@ static class NativeDiagnostic
if (Hash(Encoding.UTF8.GetBytes(originalImage)) != "c08bf9436fb8b72ea82fdf0e677641ab2fc42a0a59e2cf0309c00df519884c5c") throw new InvalidOperationException("Pinned Recovery staging script hash mismatch.");
var image = ReplaceOnce(originalImage, " if ! cp -f \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\"; then\n", " if ! cp -f \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cp -f \"$IMAGE_TOOLS/recovery/readiness.sh\" \"${script%/*}/readiness.sh\"; then\n");
image = ReplaceOnce(image, " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n", " if ! cmp -s \"$IMAGE_TOOLS/recovery/launch.sh\" \"$script\" ||\n ! cmp -s \"$IMAGE_TOOLS/recovery/readiness.sh\" \"$state/readiness.sh\" ||\n");
foreach (var pair in new[] { ("recovery-patch.py", patch), ("Dockerfile.patched", dockerfile), ("container-entry.sh", entry), ("guest-launch.sh", wrapper), ("guest-readiness.sh", hook), ("image.sh.patched", image), ("recoveryosd-original.plist", daemon), ("recoveryosd-diagnostic.plist", DiagnosticDaemon), ("early-bootstrap.sh", MountOnlyBootstrap), ("boot.sh.patched", compatibility.Boot), ("opencore-config.plist", compatibility.Config) })
image = ReplaceOnce(image, " if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n", " info \"[recovery-original] bytes=$(stat -c%s -- \"$image\") sha256=$(sha256sum \"$image\" | awk '{print $1}')\"\n if ! result=$(python3 \"$IMAGE_TOOLS/recovery/patch.py\" \"$image\"); then\n");
foreach (var pair in new[] { ("recovery-patch.py", patch), ("Dockerfile.patched", dockerfile), ("container-entry.sh", entry), ("guest-launch.sh", wrapper), ("guest-readiness.sh", hook), ("image.sh.patched", image), ("recoveryosd-original.plist", daemon), ("recoveryosd-diagnostic.plist", DiagnosticDaemon), ("recoveryosd-13-original.plist", OriginalDaemon13), ("recoveryosd-13-diagnostic.plist", DiagnosticDaemon13), ("early-bootstrap.sh", MountOnlyBootstrap), ("boot.sh.patched", compatibility.Boot), ("opencore-config.plist", compatibility.Config) })
File.WriteAllText(Path.Combine(output, pair.Item1), pair.Item2, new UTF8Encoding(false));
Save(Path.Combine(output, "source-hashes.json"), Directory.GetFiles(output).Where(path => Path.GetFileName(path) is "recovery-patch.py" or "Dockerfile.patched" or "container-entry.sh" or "guest-launch.sh" or "guest-readiness.sh" or "image.sh.patched" or "recoveryosd-original.plist" or "recoveryosd-diagnostic.plist" or "early-bootstrap.sh" or "boot.sh.patched" or "opencore-config.plist" or "compatibility-boot-assets.json").ToDictionary(path => Path.GetFileName(path)!, path => Hash(File.ReadAllBytes(path))));
Save(Path.Combine(output, "source-hashes.json"), Directory.GetFiles(output).Where(path => Path.GetFileName(path) is "recovery-patch.py" or "udif_checksums.py" or "Dockerfile.patched" or "container-entry.sh" or "guest-launch.sh" or "guest-readiness.sh" or "image.sh.patched" or "recoveryosd-original.plist" or "recoveryosd-diagnostic.plist" or "recoveryosd-13-original.plist" or "recoveryosd-13-diagnostic.plist" or "early-bootstrap.sh" or "boot.sh.patched" or "opencore-config.plist" or "compatibility-boot-assets.json").ToDictionary(path => Path.GetFileName(path)!, path => Hash(File.ReadAllBytes(path))));
await Command("bash", ["-n", Path.Combine(output, "guest-launch.sh")], output, "guest-hook-syntax", cancellation);
await Command("bash", ["-n", Path.Combine(output, "guest-readiness.sh")], output, "guest-readiness-syntax", cancellation);
await Command("bash", ["-n", Path.Combine(output, "image.sh.patched")], output, "guest-staging-syntax", cancellation);
@@ -242,6 +256,7 @@ static class NativeDiagnostic
await Command("bash", ["-n", Path.Combine(output, "boot.sh.patched")], output, "boot-staging-syntax", cancellation);
if (!writeSource) return;
File.WriteAllText(patchPath, patch, new UTF8Encoding(false));
File.WriteAllText(Path.Combine(source, "src/install/recovery/udif_checksums.py"), checksumBinding, new UTF8Encoding(false));
File.WriteAllText(dockerPath, dockerfile, new UTF8Encoding(false));
File.WriteAllText(entryPath, entry, new UTF8Encoding(false));
File.WriteAllText(imagePath, image, new UTF8Encoding(false));
@@ -259,6 +274,49 @@ static class NativeDiagnostic
}
}
static void ValidateDaemon(string xml, string processType, bool patched)
{
var pairs = XDocument.Parse(xml).Root!.Element("dict")!.Elements().ToArray();
if (pairs.Length != 10 || !pairs.Where((_, index) => index % 2 == 0).Select(element => element.Value).SequenceEqual(new[] { "Label", "OnDemand", "ProcessType", "EnablePressuredExit", "ProgramArguments" })) throw new InvalidOperationException("Recovery daemon fields changed.");
if (pairs[1].Value != "com.apple.recoveryosd" || pairs[3].Name != "false" || pairs[5].Value != processType || pairs[7].Name != "false" || pairs[9].Name != "array" || !pairs[9].Elements().Select(element => element.Value).SequenceEqual(patched ? new[] { "/bin/bash", "/Volumes/installstate/launch.sh" } : new[] { "/usr/libexec/recoveryosd" })) throw new InvalidOperationException("Recovery daemon identity/arguments changed.");
}
static string PrepareRecoveryPatch(string original)
{
var patch = ReplaceOnce(original, OriginalBootstrap, MountOnlyBootstrap);
patch = ReplaceOnce(patch, "import zlib\n", "import zlib\nfrom udif_checksums import ChecksumPlan\n");
var constants = "RECOVERY_13_ORIGINAL = b'''" + OriginalDaemon13 + "'''\nRECOVERY_13_REPLACEMENT = b'''" + DiagnosticDaemon13 + "'''.ljust(len(RECOVERY_13_ORIGINAL), b\" \")\nRECOVERY_14_ORIGINAL = b'''" + OriginalDaemon + "\n'''\nRECOVERY_14_REPLACEMENT = b'''" + DiagnosticDaemon + "'''.ljust(len(RECOVERY_14_ORIGINAL), b\" \")\nRECOVERY_LABEL = b'<string>com.apple.recoveryosd</string>'";
patch = ReplaceOnce(patch, "RECOVERY_ORIGINAL = b\"/usr/libexec/recoveryosd\"\nRECOVERY_REPLACEMENT = b\"/private/etc/rc.cdrom.sh\"", constants);
patch = ReplaceOnce(patch, " if len(RECOVERY_REPLACEMENT) != len(RECOVERY_ORIGINAL):\n raise RuntimeError(\"recoveryosd launch-path replacement length mismatch\")", " if len(RECOVERY_13_REPLACEMENT) != len(RECOVERY_13_ORIGINAL) or len(RECOVERY_14_REPLACEMENT) != len(RECOVERY_14_ORIGINAL):\n raise RuntimeError(\"recoveryosd launch-path replacement length mismatch\")");
patch = ReplaceOnce(patch, " (\"recoveryosd launch path\", RECOVERY_ORIGINAL, RECOVERY_REPLACEMENT),", " (\"recoveryosd macOS 13 launch path\", RECOVERY_13_ORIGINAL, RECOVERY_13_REPLACEMENT),\n (\"recoveryosd macOS 14 launch path\", RECOVERY_14_ORIGINAL, RECOVERY_14_REPLACEMENT),");
patch = ReplaceOnce(patch, " chunks = {}\n", " chunks = {}\n recovery_label_count = 0\n");
patch = ReplaceOnce(patch, " plist = plistlib.loads(image.read(xml_length))\n", " plist = plistlib.loads(image.read(xml_length))\n checksums = ChecksumPlan(image, koly, plist, xml_offset, xml_length, size)\n");
patch = ReplaceOnce(patch, " key = (blkx_index, run_index)\n", " recovery_label_count += decoded.count(RECOVERY_LABEL)\n key = (blkx_index, run_index)\n");
var variantValidation = """
if len(matches[patches[0][0]]) != 1:
raise RuntimeError("Expected exactly one rc.cdrom.sh bootstrap")
variants = [item for item in patches[1:] if matches[item[0]]]
if recovery_label_count != 1 or len(variants) != 1 or len(matches[variants[0][0]]) != 1:
raise RuntimeError("Expected exactly one known recoveryosd plist and launch path")
patches = (patches[0], variants[0])
print("[recovery-daemon] " + variants[0][0])
""";
patch = ReplaceOnce(patch, " for name, _, _ in patches:\n count = len(matches[name])\n if count != 1:\n raise RuntimeError(f\"Expected exactly one {name}, found {count}\")", IndentPython(variantValidation, 8));
// Plan all recompressed chunks before the first image write. A later
// compression failure must not leave an earlier chunk patched.
patch = ReplaceOnce(patch, " for key, chunk in chunks.items():\n patched = bytearray", " planned = []\n for key, chunk in chunks.items():\n patched = bytearray");
patch = ReplaceOnce(patch, " image.seek(chunk[\"physical_offset\"])\n image.write(stored)", " planned.append((chunk[\"physical_offset\"], stored))\n\n checksum_xml, checksum_koly = checksums.prepare(planned)\n for physical_offset, stored in planned:\n image.seek(physical_offset)\n image.write(stored)\n image.seek(xml_offset)\n image.write(checksum_xml)\n image.seek(size - 512)\n image.write(checksum_koly)");
patch = ReplaceOnce(patch, " image.flush()\n", " image.flush()\n checksums.verify()\n");
return patch;
}
static string IndentPython(string text, int spaces)
{
var lines = text.Split('\n');
var common = lines.Where(line => line.Length > 0).Min(line => line.TakeWhile(character => character == ' ').Count());
return string.Join("\n", lines.Select(line => new string(' ', spaces) + (line.Length > 0 ? line[common..] : "")));
}
static async Task<(string Boot, string Config, string Assets)> PrepareCompatibility(string source, string output, string? archivePath, CancellationToken cancellation)
{
var boot = File.ReadAllText(Path.Combine(source, "src", "boot.sh"));
@@ -399,7 +457,7 @@ static class NativeDiagnostic
var result = await Command("docker", ["exec", id, "cat", "/dev/shm/installstate/" + file.Item1], output, "capture-" + file.Item1, cancellation, requireSuccess: false);
if (result.ExitCode == 0 && !string.IsNullOrWhiteSpace(result.Output)) File.WriteAllText(Path.Combine(output, file.Item2), result.Output);
}
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test ! -f /storage/13/setup.dmg || sha256sum /storage/13/setup.dmg; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false);
await Command("docker", ["exec", id, "sh", "-c", "printf '[qemu]\n'; qemu-system-x86_64 --version | head -n 1; printf '[Recovery hash]\n'; test ! -f /storage/13/setup.dmg || sha256sum /storage/13/setup.dmg; printf '[resources]\n'; df -Pk /storage; cat /sys/fs/cgroup/memory.max /sys/fs/cgroup/cpu.max 2>/dev/null || true"], output, "guest-container-resources", cancellation, requireSuccess: false, retainSuccessful: true);
}
static async Task CaptureMonitor(string id, string output, string token, CancellationToken cancellation)
@@ -475,7 +533,166 @@ static class NativeDiagnostic
}
}
static async Task<CommandResult> Command(string executable, string[] arguments, string output, string label, CancellationToken cancellation, bool requireSuccess = true, bool echo = false)
static async Task ValidateRecoveryPatch(string output)
{
if (Crc32(Encoding.ASCII.GetBytes("123456789")) != 0xcbf43926) throw new InvalidOperationException("Independent C# CRC32 known vector failed.");
var fixture = Path.Combine(output, "validation-recovery-patch");
Directory.CreateDirectory(fixture);
var patch = File.ReadAllText(Path.Combine(output, "recovery-patch.py"));
const string marker = "SCRIPT_ORIGINAL = b'''";
var start = patch.IndexOf(marker, StringComparison.Ordinal) + marker.Length;
var end = patch.IndexOf("'''", start, StringComparison.Ordinal);
var bootstrap = patch[start..end];
var cases = new[] {
("13-zlib", OriginalDaemon13, true, true), ("14-zlib", OriginalDaemon + "\n", true, true),
("13-raw", OriginalDaemon13, false, true), ("14-raw", OriginalDaemon + "\n", false, true),
("unknown", OriginalDaemon13.Replace("Interactive", "Unknown"), true, false),
("duplicate", OriginalDaemon13 + OriginalDaemon + "\n", true, false),
("duplicate-unknown", OriginalDaemon13 + OriginalDaemon13.Replace("Interactive", "Unknown"), true, false),
("malformed", OriginalDaemon13.Replace("</array>", "</broken>"), true, false),
("wrong-arguments", OriginalDaemon13.Replace("/usr/libexec/recoveryosd", "/usr/libexec/wrongdaemon"), true, false),
("duplicate-arguments", OriginalDaemon13.Replace("</array>", "<string>/usr/libexec/recoveryosd</string></array>"), true, false),
("corrupt-data-crc", OriginalDaemon13, true, false), ("unsupported-crc", OriginalDaemon13, true, false),
("xml-boundary", OriginalDaemon13, true, false), ("physical-boundary", OriginalDaemon13, true, false),
("unknown-zero-run", OriginalDaemon13, true, false), ("logical-boundary", OriginalDaemon13, false, false)
};
foreach (var item in cases)
{
var path = Path.Combine(fixture, item.Item1 + ".dmg");
CreateRecoveryFixture(path, bootstrap, item.Item2, item.Item3);
if (item.Item1 is "corrupt-data-crc" or "unsupported-crc" or "xml-boundary" or "physical-boundary" or "unknown-zero-run" or "logical-boundary")
{
var corrupt = File.ReadAllBytes(path);
var trailer = corrupt.Length - 512;
if (item.Item1 == "corrupt-data-crc") corrupt[trailer + 88] ^= 1;
else if (item.Item1 == "unsupported-crc") BinaryPrimitives.WriteUInt32BigEndian(corrupt.AsSpan(trailer + 80), 3);
else if (item.Item1 == "xml-boundary") BinaryPrimitives.WriteUInt64BigEndian(corrupt.AsSpan(trailer + 224), (ulong)corrupt.Length);
else if (item.Item1 == "logical-boundary") BinaryPrimitives.WriteUInt64BigEndian(corrupt.AsSpan(trailer + 492), 1);
else
{
var xmlOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(corrupt.AsSpan(trailer + 216)));
var xmlLength = checked((int)BinaryPrimitives.ReadUInt64BigEndian(corrupt.AsSpan(trailer + 224)));
var xmlText = Encoding.UTF8.GetString(corrupt, xmlOffset, xmlLength);
var data = XDocument.Parse(xmlText).Descendants("data").Single().Value;
var mish = Convert.FromBase64String(data);
if (item.Item1 == "physical-boundary") BinaryPrimitives.WriteUInt64BigEndian(mish.AsSpan(236), (ulong)corrupt.Length);
else BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(204), 0);
var replacement = Encoding.UTF8.GetBytes(ReplaceOnce(xmlText, data, Convert.ToBase64String(mish)));
replacement.CopyTo(corrupt, xmlOffset);
}
File.WriteAllBytes(path, corrupt);
}
var before = File.ReadAllBytes(path);
var result = await Command("python3", ["-B", Path.Combine(output, "recovery-patch.py"), path], fixture, item.Item1, CancellationToken.None, requireSuccess: false);
var after = File.ReadAllBytes(path);
if ((result.ExitCode == 0) != item.Item4) throw new InvalidOperationException("Recovery fixture result mismatch: " + item.Item1 + ": " + result.Error);
if (!item.Item4 && !before.SequenceEqual(after)) throw new InvalidOperationException("Rejected Recovery fixture was modified: " + item.Item1);
if (item.Item4)
{
var decoded = DecodeRecoveryFixture(after, item.Item3);
var original = Encoding.UTF8.GetBytes(item.Item2);
var expectedText = item.Item1.StartsWith("13", StringComparison.Ordinal) ? DiagnosticDaemon13 : DiagnosticDaemon;
var expected = Encoding.UTF8.GetBytes(expectedText.PadRight(item.Item2.Length, ' '));
if (before.Length != after.Length || !decoded.AsSpan(4096, original.Length).SequenceEqual(expected)) throw new InvalidOperationException("Recovery fixture changed byte extent or daemon fields: " + item.Item1);
ValidateDaemon(expectedText, item.Item1.StartsWith("13", StringComparison.Ordinal) ? "Interactive" : "App", true);
VerifyRecoveryFixtureChecksums(after, decoded);
}
}
Save(Path.Combine(fixture, "receipt.json"), new { success = true, positiveCases = 4, negativeCases = 12, rejectedImagesUnmodified = true, knownDaemonFieldsPreserved = true, readBackCrc32IndependentlyVerified = true, syntheticUdifFixtures = true, guestExecuted = false });
}
static uint Crc32(ReadOnlySpan<byte> bytes)
{
var crc = uint.MaxValue;
foreach (var value in bytes)
{
crc ^= value;
for (var bit = 0; bit < 8; bit++) crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xedb88320u : 0);
}
return ~crc;
}
static void CreateRecoveryFixture(string path, string bootstrap, string daemon, bool compressed)
{
var decoded = new byte[16384];
Encoding.UTF8.GetBytes(bootstrap).CopyTo(decoded, 64);
Encoding.UTF8.GetBytes(daemon).CopyTo(decoded, 4096);
byte[] stored;
if (compressed)
{
using var memory = new MemoryStream();
using (var zipper = new ZLibStream(memory, CompressionLevel.Fastest, true)) zipper.Write(decoded);
stored = memory.ToArray();
}
else stored = decoded;
var mish = new byte[284];
Encoding.ASCII.GetBytes("mish").CopyTo(mish, 0);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(4), 1);
BinaryPrimitives.WriteUInt64BigEndian(mish.AsSpan(16), 32);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(64), 2);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(68), 32);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(72), Crc32(decoded));
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(200), 2);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(204), compressed ? 0x80000005u : 1u);
BinaryPrimitives.WriteUInt64BigEndian(mish.AsSpan(220), 32);
BinaryPrimitives.WriteUInt64BigEndian(mish.AsSpan(236), (ulong)stored.Length);
BinaryPrimitives.WriteUInt32BigEndian(mish.AsSpan(244), 0xffffffff);
var xml = Encoding.UTF8.GetBytes("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<plist version=\"1.0\"><dict><key>resource-fork</key><dict><key>blkx</key><array><dict><key>Data</key><data>" + Convert.ToBase64String(mish) + "</data></dict></array></dict></dict></plist>\n");
var koly = new byte[512];
Encoding.ASCII.GetBytes("koly").CopyTo(koly, 0);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(4), 4);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(8), 512);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(12), 1);
BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(32), (ulong)stored.Length);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(80), 2);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(84), 32);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(88), Crc32(stored));
BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(216), (ulong)stored.Length);
BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(224), (ulong)xml.Length);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(352), 2);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(356), 32);
BinaryPrimitives.WriteUInt32BigEndian(koly.AsSpan(360), Crc32(mish.AsSpan(72, 4)));
BinaryPrimitives.WriteUInt64BigEndian(koly.AsSpan(492), 32);
File.WriteAllBytes(path, stored.Concat(xml).Concat(koly).ToArray());
}
static byte[] DecodeRecoveryFixture(byte[] image, bool compressed)
{
var length = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 512 + 32)));
if (!compressed) return image[..length];
using var input = new MemoryStream(image, 0, length);
using var decoder = new ZLibStream(input, CompressionMode.Decompress);
using var output = new MemoryStream();
decoder.CopyTo(output);
return output.ToArray();
}
static void VerifyRecoveryFixtureChecksums(byte[] image, byte[] decoded)
{
var trailer = image.Length - 512;
var length = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(trailer + 32)));
var xmlLength = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(trailer + 224)));
var xml = XDocument.Parse(Encoding.UTF8.GetString(image, length, xmlLength));
var mish = Convert.FromBase64String(xml.Descendants("data").Single().Value);
if (Crc32(image.AsSpan(0, length)) != BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(trailer + 88)) || Crc32(decoded) != BinaryPrimitives.ReadUInt32BigEndian(mish.AsSpan(72)) || Crc32(mish.AsSpan(72, 4)) != BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(trailer + 360))) throw new InvalidOperationException("Independent C# fixture CRC32 readback failed.");
}
static async Task ValidateResourceRetention(string output)
{
var fixture = Path.Combine(output, "validation-resource-retention");
Directory.CreateDirectory(fixture);
const string label = "capture-resource-fixture";
const string successful = "Successful snapshot before stopped-container capture.\n";
await Command("bash", ["-c", "printf '%s\\n' 'Successful snapshot before stopped-container capture.'"], fixture, label, CancellationToken.None, retainSuccessful: true);
await Command("bash", ["-c", "printf '%s\\n' 'Container is not running.' >&2; exit 1"], fixture, label, CancellationToken.None, requireSuccess: false, retainSuccessful: true);
var retained = Path.Combine(fixture, label + ".last-success.stdout.log");
if (!File.Exists(retained) || File.ReadAllText(retained) != successful || File.ReadAllText(Path.Combine(fixture, label + ".stdout.log")) != "" || !File.ReadAllText(Path.Combine(fixture, label + ".stderr.log")).Contains("Container is not running."))
throw new InvalidOperationException("A failed final capture lost the last successful resource snapshot.");
using var receipt = JsonDocument.Parse(File.ReadAllText(Path.Combine(fixture, label + ".last-success.json")));
if (receipt.RootElement.GetProperty("stdoutSha256").GetString() != Hash(Encoding.UTF8.GetBytes(successful)) || receipt.RootElement.GetProperty("exitCode").GetInt32() != 0) throw new InvalidOperationException("Last successful snapshot receipt does not identify the retained bytes.");
}
static async Task<CommandResult> Command(string executable, string[] arguments, string output, string label, CancellationToken cancellation, bool requireSuccess = true, bool echo = false, bool retainSuccessful = false)
{
if (!label.StartsWith("capture-", StringComparison.Ordinal) && label is not "container" and not "container-running" and not "guest-container-resources")
Console.WriteLine("[native-diagnostic] " + label);
@@ -512,6 +729,11 @@ static class NativeDiagnostic
{
await Task.WhenAll(stdout, stderr, process.WaitForExitAsync(commandToken));
var result = new CommandResult(process.ExitCode, await stdout, await stderr);
if (retainSuccessful && result.ExitCode == 0 && !string.IsNullOrWhiteSpace(result.Output))
{
File.WriteAllText(Path.Combine(output, label + ".last-success.stdout.log"), result.Output, new UTF8Encoding(false));
Save(Path.Combine(output, label + ".last-success.json"), new { exitCode = result.ExitCode, stdoutSha256 = Hash(Encoding.UTF8.GetBytes(result.Output)), capturedUtc = DateTimeOffset.UtcNow });
}
if (requireSuccess && result.ExitCode != 0) throw new InvalidOperationException($"{label} exited {result.ExitCode}: {result.Error[..Math.Min(result.Error.Length, 1500)]}");
return result;
}
tools/ci/macos-native-udif-checksums.py
+181
View File
@@ -0,0 +1,181 @@
"""Checksum binding for the pinned Linux Recovery UDIF patcher, not a new CLI.
Source semantics: planetbeing/libdmg-hfsplus dmg/dmglib.c and dmg/blkx.c.
Only flattened, single-segment XML UDIF with CRC32 and raw/zlib data is accepted.
The caller plans same-length chunk writes; XML formatting and all offsets remain.
"""
import base64
import json
import plistlib
import re
import struct
import zlib
def u32(data, offset):
return struct.unpack_from(">I", data, offset)[0]
def u64(data, offset):
return struct.unpack_from(">Q", data, offset)[0]
def crc_contract(data, offset):
if u32(data, offset) != 2 or u32(data, offset + 4) != 32 or any(data[offset + 12:offset + 136]):
raise RuntimeError("Unsupported UDIF checksum type/size/padding")
return u32(data, offset + 8)
class ChecksumPlan:
def __init__(self, image, koly, plist, xml_offset, xml_length, size):
self.image, self.koly, self.plist = image, koly, plist
self.xml_offset, self.xml_length, self.size = xml_offset, xml_length, size
self.data_offset, self.data_length = u64(koly, 24), u64(koly, 32)
if (u32(koly, 4) != 4 or u32(koly, 8) != 512 or u32(koly, 12) != 1
or self.data_offset != 0 or u64(koly, 40) or u64(koly, 48)
or u32(koly, 60) not in (0, 1) or self.data_length != xml_offset
or xml_offset + xml_length > size - 512 or xml_length > 8 * 1024 * 1024):
raise RuntimeError("Unsupported or out-of-bounds flattened UDIF layout")
crc_contract(koly, 80)
crc_contract(koly, 352)
image.seek(xml_offset)
self.xml = image.read(xml_length)
if len(self.xml) != xml_length or not self.xml.lstrip().startswith(b"<?xml"):
raise RuntimeError("Unsupported UDIF metadata framing")
self.blocks = plist["resource-fork"]["blkx"]
self.physical_runs = {}
intervals = []
for block in self.blocks:
mish = block["Data"]
if len(mish) < 244 or mish[:4] != b"mish" or (len(mish) - 204) % 40 or u32(mish, 200) != (len(mish) - 204) // 40:
raise RuntimeError("Malformed UDIF block table")
crc_contract(mish, 64)
if u64(mish, 8) + u64(mish, 16) > u64(koly, 492):
raise RuntimeError("UDIF partition exceeds logical disk boundary")
count = u32(mish, 200)
if u32(mish, 204 + (count - 1) * 40) != 0xffffffff:
raise RuntimeError("UDIF block table has no final terminator")
for kind, offset, length, sectors in self.runs(mish):
if kind in (2, 0x7ffffffe, 0xffffffff):
if length:
raise RuntimeError("Non-data UDIF run has stored bytes")
continue
if kind not in (1, 0x80000005) or not sectors or length <= 0 or sectors * 512 > 32 * 1024 * 1024:
raise RuntimeError("Unsupported UDIF compression/run boundary")
if offset < self.data_offset or offset + length > self.data_offset + self.data_length:
raise RuntimeError("UDIF data run exceeds data-fork boundary")
intervals.append((offset, offset + length))
self.physical_runs[offset] = length
intervals.sort()
if any(left[1] > right[0] for left, right in zip(intervals, intervals[1:])):
raise RuntimeError("Overlapping UDIF physical data runs")
def runs(self, mish):
for entry in range(204, len(mish), 40):
kind = u32(mish, entry)
sector, sectors = u64(mish, entry + 8), u64(mish, entry + 16)
if sector + sectors > u64(mish, 16):
raise RuntimeError("UDIF run exceeds its partition boundary")
offset = self.data_offset + u64(mish, 24) + u64(mish, entry + 24)
yield kind, offset, u64(mish, entry + 32), sectors
def read(self, offset, length):
self.image.seek(offset)
result = self.image.read(length)
if len(result) != length:
raise RuntimeError("Short UDIF checksum read")
return result
def logical_crcs(self, mish, replacements):
original_crc = patched_crc = 0
for kind, offset, length, sectors in self.runs(mish):
# IGNORE runs are excluded by the independently verified Apple 13
# baseline. Unknown ZERO/compression types are rejected above.
if kind not in (1, 0x80000005):
continue
old = self.read(offset, length)
new = replacements.get(offset, old)
old_decoded = old if kind == 1 else zlib.decompress(old)
new_decoded = new if kind == 1 else zlib.decompress(new)
if len(old_decoded) != sectors * 512 or len(new_decoded) != sectors * 512 or len(old) != len(new):
raise RuntimeError("UDIF checksum run changed physical/logical extent")
original_crc = zlib.crc32(old_decoded, original_crc)
patched_crc = zlib.crc32(new_decoded, patched_crc)
return original_crc, patched_crc
def data_crcs(self, replacements):
old_crc = new_crc = 0
cursor = self.data_offset
def same_until(stop):
nonlocal cursor, old_crc, new_crc
while cursor < stop:
data = self.read(cursor, min(1024 * 1024, stop - cursor))
old_crc, new_crc = zlib.crc32(data, old_crc), zlib.crc32(data, new_crc)
cursor += len(data)
for offset, new in sorted(replacements.items()):
same_until(offset)
old = self.read(offset, len(new))
old_crc, new_crc = zlib.crc32(old, old_crc), zlib.crc32(new, new_crc)
cursor += len(new)
same_until(self.data_offset + self.data_length)
return old_crc, new_crc
def prepare(self, planned):
replacements = dict(planned)
if len(replacements) != len(planned):
raise RuntimeError("Duplicate planned UDIF physical writes")
if any(self.physical_runs.get(offset) != len(data) for offset, data in replacements.items()):
raise RuntimeError("Planned UDIF write does not preserve an existing data-run boundary")
old_master = bytearray()
new_master = bytearray()
changed = []
for block in self.blocks:
mish = block["Data"]
old_crc, new_crc = self.logical_crcs(mish, replacements)
if old_crc != crc_contract(mish, 64):
raise RuntimeError("Original UDIF logical CRC32 mismatch")
old_master.extend(struct.pack(">I", old_crc))
new_master.extend(struct.pack(">I", new_crc))
if new_crc != old_crc:
new_mish = bytearray(mish)
struct.pack_into(">I", new_mish, 72, new_crc)
changed.append((mish, bytes(new_mish)))
old_data_crc, new_data_crc = self.data_crcs(replacements)
if old_data_crc != crc_contract(self.koly, 80) or zlib.crc32(old_master) != crc_contract(self.koly, 352):
raise RuntimeError("Original UDIF data-fork/master CRC32 mismatch")
xml = self.xml
for old_mish, new_mish in changed:
matches = [match for match in re.finditer(rb"<data>([\sA-Za-z0-9+/=]*)</data>", xml)
if base64.b64decode(match.group(1)) == old_mish]
if len(matches) != 1:
raise RuntimeError("UDIF block checksum XML identity is ambiguous")
match = matches[0]
encoded = iter(base64.b64encode(new_mish))
text = bytes(value if chr(value).isspace() else next(encoded) for value in match.group(1))
xml = xml[:match.start(1)] + text + xml[match.end(1):]
if len(xml) != self.xml_length:
raise RuntimeError("UDIF checksum update changed XML region length")
new_plist = plistlib.loads(xml)
expected = dict(self.plist)
expected["resource-fork"] = dict(self.plist["resource-fork"])
expected["resource-fork"]["blkx"] = [dict(block, Data=dict(changed).get(block["Data"], block["Data"])) for block in self.blocks]
if new_plist != expected:
raise RuntimeError("UDIF checksum update changed unrelated metadata")
koly = bytearray(self.koly)
struct.pack_into(">I", koly, 88, new_data_crc)
struct.pack_into(">I", koly, 360, zlib.crc32(new_master))
self.receipt = dict(originalDataCrc32=f"{old_data_crc:08x}", patchedDataCrc32=f"{new_data_crc:08x}",
originalMasterCrc32=f"{zlib.crc32(old_master):08x}", patchedMasterCrc32=f"{zlib.crc32(new_master):08x}",
changedBlockChecksums=len(changed), imageBytes=self.size, xmlOffset=self.xml_offset,
xmlBytes=self.xml_length, physicalAndLogicalExtentsPreserved=True)
return xml, bytes(koly)
def verify(self):
koly = self.read(self.size - 512, 512)
xml = self.read(self.xml_offset, self.xml_length)
verifier = ChecksumPlan(self.image, koly, plistlib.loads(xml), self.xml_offset, self.xml_length, self.size)
verifier.prepare([])
self.image.seek(0, 2)
if self.image.tell() != self.size:
raise RuntimeError("Patched UDIF image length changed")
print("[recovery-udif] " + json.dumps(dict(self.receipt, readBackChecksumsVerified=True), sort_keys=True))