forked from Manuel/meeting-assistant
ci: diagnose existing Docker-host KVM without provisioning
This commit is contained in:
dh
@@ -0,0 +1,227 @@
|
||||
#:property PublishAot=false
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
return await ExistingKvmDiagnostic.Run(args);
|
||||
|
||||
static class ExistingKvmDiagnostic
|
||||
{
|
||||
const string Image = "qemux/qemu:7.50@sha256:e7f6fda52503a546fd649670ba46e4bc23dc6dcef275bc3fac48877fbbc430df";
|
||||
const string Label = "cloud.schweigert.meeting-assistant.existing-kvm-probe";
|
||||
static readonly string[] QemuArguments = ["-machine", "pc", "-accel", "kvm", "-cpu", "host", "-m", "64", "-smp", "1", "-S", "-nodefaults",
|
||||
"-display", "none", "-monitor", "stdio", "-serial", "none", "-parallel", "none", "-nic", "none"];
|
||||
static readonly JsonSerializerOptions Json = new() { WriteIndented = true };
|
||||
|
||||
public static async Task<int> Run(string[] args)
|
||||
{
|
||||
if (args.SequenceEqual(new[] { "--help" }))
|
||||
{
|
||||
Console.WriteLine("ExistingKvmDiagnostic.cs --run|--cleanup --output DIRECTORY\nRequires .NET 10 and the existing Docker CLI/daemon. --help never calls Docker.\n--run tests only a paused, diskless QEMU with existing /dev/kvm; retains evidence and cleans up its own container. --cleanup retries that saved cleanup.");
|
||||
return 0;
|
||||
}
|
||||
if (args.Length != 3 || args[0] is not ("--run" or "--cleanup") || args[1] != "--output")
|
||||
throw new ArgumentException("Use --help or --run|--cleanup --output DIRECTORY.");
|
||||
var output = Path.GetFullPath(args[2]);
|
||||
Directory.CreateDirectory(output);
|
||||
if (args[0] == "--cleanup") return await Cleanup(output) ? 0 : 1;
|
||||
if (Directory.EnumerateFileSystemEntries(output).Any()) throw new InvalidOperationException("Run output must be empty; existing receipts cannot be reused.");
|
||||
|
||||
var started = DateTimeOffset.UtcNow;
|
||||
var token = Guid.NewGuid().ToString("N");
|
||||
var owner = new Owner(token, "meeting-assistant-kvm-" + token);
|
||||
Save(output, "owner.json", owner);
|
||||
Save(output, "source.json", new { image = Image, helperSha256 = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes("tools/ci/ExistingKvmDiagnostic.cs"))).ToLowerInvariant() });
|
||||
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(95));
|
||||
var status = "inconclusive";
|
||||
string? detail = null;
|
||||
try
|
||||
{
|
||||
await Require(Command("git", ["rev-parse", "HEAD"], output, "source-commit", budget.Token));
|
||||
await Require(Command("docker", ["context", "show"], output, "docker-context", budget.Token));
|
||||
await Require(Command("docker", ["version", "--format", "{{json .}}"], output, "docker-version", budget.Token));
|
||||
await Require(Command("docker", ["info", "--format", "{\"ID\":{{json .ID}},\"Name\":{{json .Name}},\"ServerVersion\":{{json .ServerVersion}},\"KernelVersion\":{{json .KernelVersion}},\"OperatingSystem\":{{json .OperatingSystem}},\"OSType\":{{json .OSType}},\"Architecture\":{{json .Architecture}}}"], output, "docker-daemon", budget.Token));
|
||||
var image = await Command("docker", ["image", "inspect", Image], output, "image-before", budget.Token);
|
||||
if (image.ExitCode != 0) await Require(Command("docker", ["pull", "--platform", "linux/amd64", Image], output, "image-pull", budget.Token));
|
||||
await Require(Command("docker", ["image", "inspect", Image], output, "image-exact", budget.Token));
|
||||
using var imageDocument = JsonDocument.Parse(Read(output, "image-exact", "stdout"));
|
||||
var imageId = imageDocument.RootElement[0].GetProperty("Id").GetString();
|
||||
var create = await Command("docker", ["create", "--platform", "linux/amd64", "--pull", "never", "--name", owner.Name,
|
||||
"--label", Label + "=" + token, "--cidfile", Path.Combine(output, "container.id"), "--interactive", "--read-only",
|
||||
"--network", "none", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "0.5",
|
||||
"--memory", "256m", "--memory-swap", "256m", "--pids-limit", "32", "--no-healthcheck",
|
||||
"--device", "/dev/kvm:/dev/kvm:rw", "--entrypoint", "/usr/bin/qemu-system-x86_64", Image, .. QemuArguments], output, "container-create", budget.Token);
|
||||
if (create.ExitCode != 0)
|
||||
{
|
||||
status = ClassifyFailure(Read(output, "container-create", "stderr"));
|
||||
detail = "Docker did not successfully create the device-mapped container; see container-create logs.";
|
||||
}
|
||||
else
|
||||
{
|
||||
var created = await InspectOwned(output, owner, "container-created", budget.Token);
|
||||
ValidateBoundary(created, imageId);
|
||||
var id = created.GetProperty("Id").GetString()!;
|
||||
var monitor = await Command("docker", ["start", "--attach", "--interactive", id], output, "qemu-monitor", budget.Token, "info version\ninfo kvm\ninfo status\nquit\n");
|
||||
var exited = await InspectOwned(output, owner, "container-exited", budget.Token);
|
||||
var state = exited.GetProperty("State");
|
||||
var text = Read(output, "qemu-monitor", "stdout");
|
||||
var cleanExit = monitor.ExitCode == 0 && !monitor.TimedOut && !state.GetProperty("Running").GetBoolean()
|
||||
&& state.GetProperty("ExitCode").GetInt32() == 0 && !state.GetProperty("OOMKilled").GetBoolean();
|
||||
var enabled = Regex.IsMatch(text, @"(?m)^kvm support: enabled\r?$", RegexOptions.CultureInvariant);
|
||||
var paused = Regex.IsMatch(text, @"(?m)^VM status: paused\r?$", RegexOptions.CultureInvariant);
|
||||
status = cleanExit && enabled && paused ? "kvm_usable" : ClassifyFailure(Read(output, "qemu-monitor", "stderr") + "\n" + state.GetProperty("Error").GetString());
|
||||
detail = status == "kvm_usable" ? "QEMU initialized KVM, reported enabled and paused, and exited successfully after quit. No guest CPU or OS was run."
|
||||
: "KVM initialization or its enabled/paused/clean-exit proof did not pass; inspect raw monitor output and container state.";
|
||||
}
|
||||
}
|
||||
catch (Exception error) { detail = error.Message; }
|
||||
var cleaned = await Cleanup(output);
|
||||
Save(output, "result.json", new { started, finished = DateTimeOffset.UtcNow, status, usable = status == "kvm_usable", cleaned, detail, image = Image, token });
|
||||
Console.WriteLine(JsonSerializer.Serialize(new { status, cleaned, detail }));
|
||||
return status == "kvm_usable" && cleaned ? 0 : 1;
|
||||
}
|
||||
|
||||
static string ClassifyFailure(string text)
|
||||
{
|
||||
if (text.Contains("/dev/kvm", StringComparison.Ordinal) && text.Contains("error gathering device information", StringComparison.OrdinalIgnoreCase)
|
||||
&& text.Contains("no such file or directory", StringComparison.OrdinalIgnoreCase)) return "daemon_device_missing";
|
||||
if (text.Contains("Permission denied", StringComparison.OrdinalIgnoreCase) || text.Contains("Operation not permitted", StringComparison.OrdinalIgnoreCase)) return "access_denied_observed";
|
||||
if (text.Contains("invalid accelerator kvm", StringComparison.OrdinalIgnoreCase)) return "qemu_kvm_backend_unavailable";
|
||||
if (text.Contains("failed to initialize kvm", StringComparison.OrdinalIgnoreCase)) return "kvm_initialization_failed";
|
||||
return "inconclusive";
|
||||
}
|
||||
|
||||
static void ValidateBoundary(JsonElement container, string? imageId)
|
||||
{
|
||||
var host = container.GetProperty("HostConfig");
|
||||
var devices = host.GetProperty("Devices");
|
||||
if (imageId == null || container.GetProperty("Image").GetString() != imageId || container.GetProperty("Config").GetProperty("Image").GetString() != Image
|
||||
|| container.GetProperty("Path").GetString() != "/usr/bin/qemu-system-x86_64" || !container.GetProperty("Args").EnumerateArray().Select(x => x.GetString()).SequenceEqual(QemuArguments)
|
||||
|| host.GetProperty("Privileged").GetBoolean() || !host.GetProperty("ReadonlyRootfs").GetBoolean()
|
||||
|| host.GetProperty("NetworkMode").GetString() != "none" || container.GetProperty("Mounts").GetArrayLength() != 0
|
||||
|| devices.GetArrayLength() != 1 || devices[0].GetProperty("PathOnHost").GetString() != "/dev/kvm"
|
||||
|| devices[0].GetProperty("PathInContainer").GetString() != "/dev/kvm" || devices[0].GetProperty("CgroupPermissions").GetString() != "rw"
|
||||
|| !host.GetProperty("CapDrop").EnumerateArray().Any(x => x.GetString() == "ALL")
|
||||
|| !Empty(host.GetProperty("CapAdd")) || !Empty(host.GetProperty("Binds")) || !Empty(host.GetProperty("PortBindings"))
|
||||
|| !Empty(host.GetProperty("DeviceCgroupRules"))
|
||||
|| !host.GetProperty("SecurityOpt").EnumerateArray().Any(x => x.GetString() == "no-new-privileges")
|
||||
|| host.GetProperty("Memory").GetInt64() != 268435456 || host.GetProperty("MemorySwap").GetInt64() != 268435456
|
||||
|| host.GetProperty("NanoCpus").GetInt64() != 500000000 || host.GetProperty("PidsLimit").GetInt64() != 32)
|
||||
throw new InvalidOperationException("Created container does not match the diagnostic's restricted resource boundary.");
|
||||
}
|
||||
|
||||
static bool Empty(JsonElement value) => value.ValueKind == JsonValueKind.Null
|
||||
|| value.ValueKind == JsonValueKind.Array && value.GetArrayLength() == 0
|
||||
|| value.ValueKind == JsonValueKind.Object && !value.EnumerateObject().Any();
|
||||
|
||||
static async Task<JsonElement> InspectOwned(string output, Owner owner, string step, CancellationToken cancellation)
|
||||
{
|
||||
var idPath = Path.Combine(output, "container.id");
|
||||
var savedId = File.Exists(idPath) ? File.ReadAllText(idPath).Trim() : null;
|
||||
if (savedId != null && !Regex.IsMatch(savedId, "^[a-f0-9]{64}$")) throw new InvalidOperationException("Saved container ID is invalid.");
|
||||
await Require(Command("docker", ["inspect", "--type", "container", savedId ?? owner.Name], output, step, cancellation));
|
||||
using var document = JsonDocument.Parse(Read(output, step, "stdout"));
|
||||
var values = document.RootElement;
|
||||
if (values.GetArrayLength() != 1) throw new InvalidOperationException("Container inspection did not return exactly one object.");
|
||||
var value = values[0];
|
||||
var id = value.GetProperty("Id").GetString()!;
|
||||
if (!Regex.IsMatch(id, "^[a-f0-9]{64}$") || (savedId != null && id != savedId) || value.GetProperty("Name").GetString() != "/" + owner.Name
|
||||
|| !value.GetProperty("Config").GetProperty("Labels").TryGetProperty(Label, out var label) || label.GetString() != owner.Token)
|
||||
throw new InvalidOperationException("Container ownership ID/name/label mismatch; refusing resource operations.");
|
||||
// Recover an interrupted create receipt by the saved random name and exact label, then use only its full ID.
|
||||
if (savedId == null) File.WriteAllText(idPath, id + "\n");
|
||||
return value.Clone();
|
||||
}
|
||||
|
||||
static async Task<bool> Cleanup(string output)
|
||||
{
|
||||
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(20));
|
||||
var prefix = "cleanup-" + Guid.NewGuid().ToString("N");
|
||||
try
|
||||
{
|
||||
if (!File.Exists(Path.Combine(output, "owner.json"))) { Save(output, "cleanup.json", new { cleaned = true, reason = "No owned resource receipt exists." }); return true; }
|
||||
var owner = JsonSerializer.Deserialize<Owner>(File.ReadAllText(Path.Combine(output, "owner.json")))!;
|
||||
if (!Regex.IsMatch(owner.Token, "^[a-f0-9]{32}$") || owner.Name != "meeting-assistant-kvm-" + owner.Token)
|
||||
throw new InvalidOperationException("Invalid saved ownership receipt.");
|
||||
var container = await InspectOwned(output, owner, prefix + "-inspect", budget.Token);
|
||||
var id = container.GetProperty("Id").GetString()!;
|
||||
if (container.GetProperty("State").GetProperty("Running").GetBoolean())
|
||||
{
|
||||
await Command("docker", ["stop", "--time", "1", id], output, prefix + "-stop", budget.Token);
|
||||
await InspectOwned(output, owner, prefix + "-reinspect", budget.Token);
|
||||
}
|
||||
await Require(Command("docker", ["rm", "--force", id], output, prefix + "-remove", budget.Token));
|
||||
var receipt = new { cleaned = true, id, finished = DateTimeOffset.UtcNow };
|
||||
Save(output, prefix + ".receipt.json", receipt);
|
||||
Save(output, "cleanup.json", receipt);
|
||||
return true;
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
var path = Path.Combine(output, prefix + "-inspect.stderr.log");
|
||||
var absent = File.Exists(path) && new FileInfo(path).Length <= 1024 * 1024
|
||||
&& File.ReadAllText(path).Contains("No such container", StringComparison.OrdinalIgnoreCase);
|
||||
var receipt = new { cleaned = absent, reason = error.Message, finished = DateTimeOffset.UtcNow };
|
||||
Save(output, prefix + ".receipt.json", receipt);
|
||||
Save(output, "cleanup.json", receipt);
|
||||
return absent;
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<CommandResult> Command(string program, string[] arguments, string output, string step, CancellationToken cancellation, string? input = null)
|
||||
{
|
||||
var started = DateTimeOffset.UtcNow;
|
||||
Save(output, step + ".command.json", new { program, arguments, input, started });
|
||||
var start = new ProcessStartInfo(program) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = input != null };
|
||||
foreach (var argument in arguments) start.ArgumentList.Add(argument);
|
||||
using var process = new Process { StartInfo = start };
|
||||
await using var stdout = File.Create(Path.Combine(output, step + ".stdout.log"));
|
||||
await using var stderr = File.Create(Path.Combine(output, step + ".stderr.log"));
|
||||
int? exit = null;
|
||||
var timedOut = false;
|
||||
string? error = null;
|
||||
try
|
||||
{
|
||||
cancellation.ThrowIfCancellationRequested();
|
||||
process.Start();
|
||||
var copies = Task.WhenAll(process.StandardOutput.BaseStream.CopyToAsync(stdout), process.StandardError.BaseStream.CopyToAsync(stderr));
|
||||
try
|
||||
{
|
||||
if (input != null)
|
||||
{
|
||||
try { await process.StandardInput.WriteAsync(input.AsMemory(), cancellation); await process.StandardInput.FlushAsync(cancellation); }
|
||||
catch (IOException) { /* QEMU can reject KVM before accepting stdin; preserve its stderr and state. */ }
|
||||
process.StandardInput.Close();
|
||||
}
|
||||
await process.WaitForExitAsync(cancellation);
|
||||
}
|
||||
catch (OperationCanceledException) { timedOut = true; if (!process.HasExited) process.Kill(entireProcessTree: true); }
|
||||
await copies.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
if (process.HasExited) exit = process.ExitCode;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
error = exception.Message;
|
||||
try { if (!process.HasExited) process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { /* Process never started or already exited. */ }
|
||||
}
|
||||
var result = new CommandResult(exit, timedOut, error);
|
||||
Save(output, step + ".result.json", new { result.ExitCode, result.TimedOut, result.Error, started, finished = DateTimeOffset.UtcNow });
|
||||
return result;
|
||||
}
|
||||
|
||||
static async Task Require(Task<CommandResult> command)
|
||||
{
|
||||
var result = await command;
|
||||
if (result.ExitCode != 0 || result.TimedOut || result.Error != null) throw new InvalidOperationException($"Command did not succeed: exit={result.ExitCode}, timedOut={result.TimedOut}, error={result.Error}");
|
||||
}
|
||||
static string Read(string output, string step, string stream)
|
||||
{
|
||||
var path = Path.Combine(output, step + "." + stream + ".log");
|
||||
if (new FileInfo(path).Length > 1024 * 1024) throw new InvalidOperationException("Diagnostic output exceeds the one-MiB interpretation limit; inspect the retained raw log.");
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
static void Save(string output, string name, object value) => File.WriteAllText(Path.Combine(output, name), JsonSerializer.Serialize(value, Json) + "\n");
|
||||
sealed record Owner(string Token, string Name);
|
||||
sealed record CommandResult(int? ExitCode, bool TimedOut, string? Error);
|
||||
}
|
||||
Reference in New Issue
Block a user