Public Access
257 lines
18 KiB
C#
257 lines
18 KiB
C#
#:property PublishAot=false
|
|
using System.Diagnostics;
|
|
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
|
|
return await ExistingKvmDiagnostic.Run(args);
|
|
|
|
static class ExistingKvmDiagnostic
|
|
{
|
|
const string Image = "qemux/qemu:7.50@sha256:e7f6fda52503a546fd649670ba46e4bc23dc6dcef275bc3fac48877fbbc430df";
|
|
const string Label = "cloud.schweigert.meeting-assistant.existing-kvm-probe";
|
|
const string StorageTmpfsOptions = "ro,nosuid,nodev,noexec,size=4096,mode=0555";
|
|
static readonly string[] QemuArguments = ["-machine", "pc", "-accel", "kvm", "-cpu", "host", "-m", "64", "-smp", "1", "-S", "-nodefaults",
|
|
"-display", "none", "-monitor", "stdio", "-serial", "none", "-parallel", "none", "-nic", "none"];
|
|
static readonly JsonSerializerOptions Json = new() { WriteIndented = true };
|
|
|
|
public static async Task<int> Run(string[] args)
|
|
{
|
|
if (args.SequenceEqual(new[] { "--help" }))
|
|
{
|
|
Console.WriteLine("ExistingKvmDiagnostic.cs --run|--cleanup --output DIRECTORY\nExistingKvmDiagnostic.cs --validate-evidence DIRECTORY\nRequires .NET 10; --run/--cleanup also require the existing Docker CLI/daemon. --help never calls Docker.\n--run tests only a paused, diskless QEMU with existing /dev/kvm; retains evidence and cleans up its own container. --cleanup retries that saved cleanup.\n--validate-evidence reads saved monitor/result/container evidence through the run's success parser; no Docker commands or file writes.");
|
|
return 0;
|
|
}
|
|
if (args.Length == 2 && args[0] == "--validate-evidence") return ValidateEvidence(Path.GetFullPath(args[1]));
|
|
if (args.Length != 3 || args[0] is not ("--run" or "--cleanup") || args[1] != "--output")
|
|
throw new ArgumentException("Use --help, --validate-evidence DIRECTORY or --run|--cleanup --output DIRECTORY.");
|
|
var output = Path.GetFullPath(args[2]);
|
|
Directory.CreateDirectory(output);
|
|
if (args[0] == "--cleanup") return await Cleanup(output) ? 0 : 1;
|
|
if (Directory.EnumerateFileSystemEntries(output).Any()) throw new InvalidOperationException("Run output must be empty; existing receipts cannot be reused.");
|
|
|
|
var started = DateTimeOffset.UtcNow;
|
|
var token = Guid.NewGuid().ToString("N");
|
|
var owner = new Owner(token, "meeting-assistant-kvm-" + token);
|
|
Save(output, "owner.json", owner);
|
|
Save(output, "source.json", new { image = Image, helperSha256 = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes("tools/ci/ExistingKvmDiagnostic.cs"))).ToLowerInvariant() });
|
|
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(95));
|
|
var status = "inconclusive";
|
|
string? detail = null;
|
|
try
|
|
{
|
|
await Require(Command("git", ["rev-parse", "HEAD"], output, "source-commit", budget.Token));
|
|
await Require(Command("docker", ["context", "show"], output, "docker-context", budget.Token));
|
|
await Require(Command("docker", ["version", "--format", "{{json .}}"], output, "docker-version", budget.Token));
|
|
await Require(Command("docker", ["info", "--format", "{\"ID\":{{json .ID}},\"Name\":{{json .Name}},\"ServerVersion\":{{json .ServerVersion}},\"KernelVersion\":{{json .KernelVersion}},\"OperatingSystem\":{{json .OperatingSystem}},\"OSType\":{{json .OSType}},\"Architecture\":{{json .Architecture}}}"], output, "docker-daemon", budget.Token));
|
|
var image = await Command("docker", ["image", "inspect", Image], output, "image-before", budget.Token);
|
|
if (image.ExitCode != 0) await Require(Command("docker", ["pull", "--platform", "linux/amd64", Image], output, "image-pull", budget.Token));
|
|
await Require(Command("docker", ["image", "inspect", Image], output, "image-exact", budget.Token));
|
|
using var imageDocument = JsonDocument.Parse(Read(output, "image-exact", "stdout"));
|
|
var imageId = imageDocument.RootElement[0].GetProperty("Id").GetString();
|
|
var create = await Command("docker", ["create", "--platform", "linux/amd64", "--pull", "never", "--name", owner.Name,
|
|
"--label", Label + "=" + token, "--cidfile", Path.Combine(output, "container.id"), "--interactive", "--read-only",
|
|
"--network", "none", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "0.5",
|
|
"--memory", "256m", "--memory-swap", "256m", "--pids-limit", "32", "--no-healthcheck",
|
|
"--tmpfs", "/storage:" + StorageTmpfsOptions,
|
|
"--device", "/dev/kvm:/dev/kvm:rw", "--entrypoint", "/usr/bin/qemu-system-x86_64", Image, .. QemuArguments], output, "container-create", budget.Token);
|
|
if (create.ExitCode != 0)
|
|
{
|
|
status = ClassifyFailure(Read(output, "container-create", "stderr"));
|
|
detail = "Docker did not successfully create the device-mapped container; see container-create logs.";
|
|
}
|
|
else
|
|
{
|
|
var created = await InspectOwned(output, owner, "container-created", budget.Token);
|
|
ValidateBoundary(created, imageId);
|
|
var id = created.GetProperty("Id").GetString()!;
|
|
var monitor = await Command("docker", ["start", "--attach", "--interactive", id], output, "qemu-monitor", budget.Token, "info version\ninfo kvm\ninfo status\nquit\n");
|
|
var exited = await InspectOwned(output, owner, "container-exited", budget.Token);
|
|
var state = exited.GetProperty("State");
|
|
var text = Read(output, "qemu-monitor", "stdout");
|
|
status = KvmUsable(text, monitor, state) ? "kvm_usable" : ClassifyFailure(Read(output, "qemu-monitor", "stderr") + "\n" + state.GetProperty("Error").GetString());
|
|
detail = status == "kvm_usable" ? "QEMU initialized KVM, reported enabled and paused, and exited successfully after quit. No guest CPU or OS was run."
|
|
: "KVM initialization or its enabled/paused/clean-exit proof did not pass; inspect raw monitor output and container state.";
|
|
}
|
|
}
|
|
catch (Exception error) { detail = error.Message; }
|
|
var cleaned = await Cleanup(output);
|
|
Save(output, "result.json", new { started, finished = DateTimeOffset.UtcNow, status, usable = status == "kvm_usable", cleaned, detail, image = Image, token });
|
|
Console.WriteLine(JsonSerializer.Serialize(new { status, cleaned, detail }));
|
|
return status == "kvm_usable" && cleaned ? 0 : 1;
|
|
}
|
|
|
|
static bool KvmUsable(string text, CommandResult monitor, JsonElement state)
|
|
{
|
|
var cleanExit = monitor.ExitCode == 0 && !monitor.TimedOut && monitor.Error is null && !state.GetProperty("Running").GetBoolean()
|
|
&& state.GetProperty("ExitCode").GetInt32() == 0 && !state.GetProperty("OOMKilled").GetBoolean();
|
|
var enabled = Regex.IsMatch(text, @"(?m)^kvm support: enabled\r?$", RegexOptions.CultureInvariant);
|
|
var paused = Regex.IsMatch(text, @"(?m)^VM status: paused(?: \(prelaunch\))?\r?$", RegexOptions.CultureInvariant);
|
|
return cleanExit && enabled && paused;
|
|
}
|
|
|
|
static int ValidateEvidence(string evidence)
|
|
{
|
|
try
|
|
{
|
|
using var monitorDocument = JsonDocument.Parse(File.ReadAllText(Path.Combine(evidence, "qemu-monitor.result.json")));
|
|
var result = monitorDocument.RootElement;
|
|
var monitor = new CommandResult(result.GetProperty("ExitCode").GetInt32(), result.GetProperty("TimedOut").GetBoolean(), result.GetProperty("Error").GetString());
|
|
using var exited = JsonDocument.Parse(Read(evidence, "container-exited", "stdout"));
|
|
if (exited.RootElement.GetArrayLength() != 1) throw new InvalidOperationException("Expected exactly one saved exited container.");
|
|
var usable = KvmUsable(Read(evidence, "qemu-monitor", "stdout"), monitor, exited.RootElement[0].GetProperty("State"));
|
|
Console.WriteLine(JsonSerializer.Serialize(new { usable, scope = "Saved HMP protocol and clean exit interpretation only; source, ownership and container boundary are not requalified." }));
|
|
return usable ? 0 : 1;
|
|
}
|
|
catch (Exception error) { Console.Error.WriteLine("Evidence interpretation failed: " + error.Message); return 1; }
|
|
}
|
|
|
|
static string ClassifyFailure(string text)
|
|
{
|
|
if (text.Contains("/dev/kvm", StringComparison.Ordinal) && text.Contains("error gathering device information", StringComparison.OrdinalIgnoreCase)
|
|
&& text.Contains("no such file or directory", StringComparison.OrdinalIgnoreCase)) return "daemon_device_missing";
|
|
if (text.Contains("Permission denied", StringComparison.OrdinalIgnoreCase) || text.Contains("Operation not permitted", StringComparison.OrdinalIgnoreCase)) return "access_denied_observed";
|
|
if (text.Contains("invalid accelerator kvm", StringComparison.OrdinalIgnoreCase)) return "qemu_kvm_backend_unavailable";
|
|
if (text.Contains("failed to initialize kvm", StringComparison.OrdinalIgnoreCase)) return "kvm_initialization_failed";
|
|
return "inconclusive";
|
|
}
|
|
|
|
static void ValidateBoundary(JsonElement container, string? imageId)
|
|
{
|
|
var host = container.GetProperty("HostConfig");
|
|
var devices = host.GetProperty("Devices");
|
|
if (container.GetProperty("Mounts").GetArrayLength() != 0)
|
|
throw new InvalidOperationException("Container mount boundary failed: persistent volumes or binds were created; expected Mounts=[] with only the read-only /storage tmpfs.");
|
|
if (!host.TryGetProperty("Tmpfs", out var tmpfs) || tmpfs.ValueKind != JsonValueKind.Object || tmpfs.EnumerateObject().Count() != 1
|
|
|| !tmpfs.TryGetProperty("/storage", out var options) || options.GetString() != StorageTmpfsOptions)
|
|
throw new InvalidOperationException("Container tmpfs boundary failed: expected only /storage:" + StorageTmpfsOptions + ".");
|
|
if (imageId == null || container.GetProperty("Image").GetString() != imageId || container.GetProperty("Config").GetProperty("Image").GetString() != Image
|
|
|| container.GetProperty("Path").GetString() != "/usr/bin/qemu-system-x86_64" || !container.GetProperty("Args").EnumerateArray().Select(x => x.GetString()).SequenceEqual(QemuArguments)
|
|
|| host.GetProperty("Privileged").GetBoolean() || !host.GetProperty("ReadonlyRootfs").GetBoolean()
|
|
|| host.GetProperty("NetworkMode").GetString() != "none"
|
|
|| devices.GetArrayLength() != 1 || devices[0].GetProperty("PathOnHost").GetString() != "/dev/kvm"
|
|
|| devices[0].GetProperty("PathInContainer").GetString() != "/dev/kvm" || devices[0].GetProperty("CgroupPermissions").GetString() != "rw"
|
|
|| !host.GetProperty("CapDrop").EnumerateArray().Any(x => x.GetString() == "ALL")
|
|
|| !Empty(host.GetProperty("CapAdd")) || !Empty(host.GetProperty("Binds")) || !Empty(host.GetProperty("PortBindings"))
|
|
|| !Empty(host.GetProperty("DeviceCgroupRules"))
|
|
|| !host.GetProperty("SecurityOpt").EnumerateArray().Any(x => x.GetString() == "no-new-privileges")
|
|
|| host.GetProperty("Memory").GetInt64() != 268435456 || host.GetProperty("MemorySwap").GetInt64() != 268435456
|
|
|| host.GetProperty("NanoCpus").GetInt64() != 500000000 || host.GetProperty("PidsLimit").GetInt64() != 32)
|
|
throw new InvalidOperationException("Created container does not match the diagnostic's restricted resource boundary.");
|
|
}
|
|
|
|
static bool Empty(JsonElement value) => value.ValueKind == JsonValueKind.Null
|
|
|| value.ValueKind == JsonValueKind.Array && value.GetArrayLength() == 0
|
|
|| value.ValueKind == JsonValueKind.Object && !value.EnumerateObject().Any();
|
|
|
|
static async Task<JsonElement> InspectOwned(string output, Owner owner, string step, CancellationToken cancellation)
|
|
{
|
|
var idPath = Path.Combine(output, "container.id");
|
|
var savedId = File.Exists(idPath) ? File.ReadAllText(idPath).Trim() : null;
|
|
if (savedId != null && !Regex.IsMatch(savedId, "^[a-f0-9]{64}$")) throw new InvalidOperationException("Saved container ID is invalid.");
|
|
await Require(Command("docker", ["inspect", "--type", "container", savedId ?? owner.Name], output, step, cancellation));
|
|
using var document = JsonDocument.Parse(Read(output, step, "stdout"));
|
|
var values = document.RootElement;
|
|
if (values.GetArrayLength() != 1) throw new InvalidOperationException("Container inspection did not return exactly one object.");
|
|
var value = values[0];
|
|
var id = value.GetProperty("Id").GetString()!;
|
|
if (!Regex.IsMatch(id, "^[a-f0-9]{64}$") || (savedId != null && id != savedId) || value.GetProperty("Name").GetString() != "/" + owner.Name
|
|
|| !value.GetProperty("Config").GetProperty("Labels").TryGetProperty(Label, out var label) || label.GetString() != owner.Token)
|
|
throw new InvalidOperationException("Container ownership ID/name/label mismatch; refusing resource operations.");
|
|
// Recover an interrupted create receipt by the saved random name and exact label, then use only its full ID.
|
|
if (savedId == null) File.WriteAllText(idPath, id + "\n");
|
|
return value.Clone();
|
|
}
|
|
|
|
static async Task<bool> Cleanup(string output)
|
|
{
|
|
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(20));
|
|
var prefix = "cleanup-" + Guid.NewGuid().ToString("N");
|
|
try
|
|
{
|
|
if (!File.Exists(Path.Combine(output, "owner.json"))) { Save(output, "cleanup.json", new { cleaned = true, reason = "No owned resource receipt exists." }); return true; }
|
|
var owner = JsonSerializer.Deserialize<Owner>(File.ReadAllText(Path.Combine(output, "owner.json")))!;
|
|
if (!Regex.IsMatch(owner.Token, "^[a-f0-9]{32}$") || owner.Name != "meeting-assistant-kvm-" + owner.Token)
|
|
throw new InvalidOperationException("Invalid saved ownership receipt.");
|
|
var container = await InspectOwned(output, owner, prefix + "-inspect", budget.Token);
|
|
var id = container.GetProperty("Id").GetString()!;
|
|
if (container.GetProperty("State").GetProperty("Running").GetBoolean())
|
|
{
|
|
await Command("docker", ["stop", "--time", "1", id], output, prefix + "-stop", budget.Token);
|
|
await InspectOwned(output, owner, prefix + "-reinspect", budget.Token);
|
|
}
|
|
await Require(Command("docker", ["rm", "--force", "--volumes", id], output, prefix + "-remove", budget.Token));
|
|
var receipt = new { cleaned = true, id, finished = DateTimeOffset.UtcNow };
|
|
Save(output, prefix + ".receipt.json", receipt);
|
|
Save(output, "cleanup.json", receipt);
|
|
return true;
|
|
}
|
|
catch (Exception error)
|
|
{
|
|
var path = Path.Combine(output, prefix + "-inspect.stderr.log");
|
|
var absent = File.Exists(path) && new FileInfo(path).Length <= 1024 * 1024
|
|
&& File.ReadAllText(path).Contains("No such container", StringComparison.OrdinalIgnoreCase);
|
|
var receipt = new { cleaned = absent, reason = error.Message, finished = DateTimeOffset.UtcNow };
|
|
Save(output, prefix + ".receipt.json", receipt);
|
|
Save(output, "cleanup.json", receipt);
|
|
return absent;
|
|
}
|
|
}
|
|
|
|
static async Task<CommandResult> Command(string program, string[] arguments, string output, string step, CancellationToken cancellation, string? input = null)
|
|
{
|
|
var started = DateTimeOffset.UtcNow;
|
|
Save(output, step + ".command.json", new { program, arguments, input, started });
|
|
var start = new ProcessStartInfo(program) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = input != null };
|
|
foreach (var argument in arguments) start.ArgumentList.Add(argument);
|
|
using var process = new Process { StartInfo = start };
|
|
await using var stdout = File.Create(Path.Combine(output, step + ".stdout.log"));
|
|
await using var stderr = File.Create(Path.Combine(output, step + ".stderr.log"));
|
|
int? exit = null;
|
|
var timedOut = false;
|
|
string? error = null;
|
|
try
|
|
{
|
|
cancellation.ThrowIfCancellationRequested();
|
|
process.Start();
|
|
var copies = Task.WhenAll(process.StandardOutput.BaseStream.CopyToAsync(stdout), process.StandardError.BaseStream.CopyToAsync(stderr));
|
|
try
|
|
{
|
|
if (input != null)
|
|
{
|
|
try { await process.StandardInput.WriteAsync(input.AsMemory(), cancellation); await process.StandardInput.FlushAsync(cancellation); }
|
|
catch (IOException) { /* QEMU can reject KVM before accepting stdin; preserve its stderr and state. */ }
|
|
process.StandardInput.Close();
|
|
}
|
|
await process.WaitForExitAsync(cancellation);
|
|
}
|
|
catch (OperationCanceledException) { timedOut = true; if (!process.HasExited) process.Kill(entireProcessTree: true); }
|
|
await copies.WaitAsync(TimeSpan.FromSeconds(2));
|
|
if (process.HasExited) exit = process.ExitCode;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
error = exception.Message;
|
|
try { if (!process.HasExited) process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { /* Process never started or already exited. */ }
|
|
}
|
|
var result = new CommandResult(exit, timedOut, error);
|
|
Save(output, step + ".result.json", new { result.ExitCode, result.TimedOut, result.Error, started, finished = DateTimeOffset.UtcNow });
|
|
return result;
|
|
}
|
|
|
|
static async Task Require(Task<CommandResult> command)
|
|
{
|
|
var result = await command;
|
|
if (result.ExitCode != 0 || result.TimedOut || result.Error != null) throw new InvalidOperationException($"Command did not succeed: exit={result.ExitCode}, timedOut={result.TimedOut}, error={result.Error}");
|
|
}
|
|
static string Read(string output, string step, string stream)
|
|
{
|
|
var path = Path.Combine(output, step + "." + stream + ".log");
|
|
if (new FileInfo(path).Length > 1024 * 1024) throw new InvalidOperationException("Diagnostic output exceeds the one-MiB interpretation limit; inspect the retained raw log.");
|
|
return File.ReadAllText(path);
|
|
}
|
|
static void Save(string output, string name, object value) => File.WriteAllText(Path.Combine(output, name), JsonSerializer.Serialize(value, Json) + "\n");
|
|
sealed record Owner(string Token, string Name);
|
|
sealed record CommandResult(int? ExitCode, bool TimedOut, string? Error);
|
|
}
|