Public Access
302 lines
22 KiB
C#
302 lines
22 KiB
C#
#:property PublishAot=false
|
|
using System.Diagnostics;
|
|
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
|
|
return await ExistingKvmDiagnostic.Run(args);
|
|
|
|
static class ExistingKvmDiagnostic
|
|
{
|
|
const string Image = "qemux/qemu:7.50@sha256:e7f6fda52503a546fd649670ba46e4bc23dc6dcef275bc3fac48877fbbc430df";
|
|
const string Label = "cloud.schweigert.meeting-assistant.existing-kvm-probe";
|
|
const string StorageTmpfsOptions = "ro,nosuid,nodev,noexec,size=4096,mode=0555";
|
|
static readonly string[] QemuArguments = ["-machine", "pc", "-accel", "kvm", "-cpu", "host", "-m", "64", "-smp", "1", "-S", "-nodefaults",
|
|
"-display", "none", "-monitor", "stdio", "-serial", "none", "-parallel", "none", "-nic", "none"];
|
|
static readonly JsonSerializerOptions Json = new() { WriteIndented = true };
|
|
|
|
public static async Task<int> Run(string[] args)
|
|
{
|
|
if (args.SequenceEqual(new[] { "--help" }))
|
|
{
|
|
Console.WriteLine("ExistingKvmDiagnostic.cs --run|--cleanup|--cleanup-run4165-volume --output DIRECTORY\nRequires .NET 10 and the existing Docker CLI/daemon. --help never calls Docker.\n--run tests only a paused, diskless QEMU with existing /dev/kvm; retains evidence and cleans up its own container. --cleanup retries that saved cleanup.\n--cleanup-run4165-volume removes only run 4165's frozen unused anonymous volume after daemon, old-container absence and reference checks.");
|
|
return 0;
|
|
}
|
|
if (args.Length != 3 || args[0] is not ("--run" or "--cleanup" or "--cleanup-run4165-volume") || args[1] != "--output")
|
|
throw new ArgumentException("Use --help or --run|--cleanup|--cleanup-run4165-volume --output DIRECTORY.");
|
|
var output = Path.GetFullPath(args[2]);
|
|
Directory.CreateDirectory(output);
|
|
if (args[0] == "--cleanup") return await Cleanup(output) ? 0 : 1;
|
|
if (args[0] == "--cleanup-run4165-volume") return await CleanupRun4165Volume(output) ? 0 : 1;
|
|
if (Directory.EnumerateFileSystemEntries(output).Any()) throw new InvalidOperationException("Run output must be empty; existing receipts cannot be reused.");
|
|
|
|
var started = DateTimeOffset.UtcNow;
|
|
var token = Guid.NewGuid().ToString("N");
|
|
var owner = new Owner(token, "meeting-assistant-kvm-" + token);
|
|
Save(output, "owner.json", owner);
|
|
Save(output, "source.json", new { image = Image, helperSha256 = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes("tools/ci/ExistingKvmDiagnostic.cs"))).ToLowerInvariant() });
|
|
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(95));
|
|
var status = "inconclusive";
|
|
string? detail = null;
|
|
try
|
|
{
|
|
await Require(Command("git", ["rev-parse", "HEAD"], output, "source-commit", budget.Token));
|
|
await Require(Command("docker", ["context", "show"], output, "docker-context", budget.Token));
|
|
await Require(Command("docker", ["version", "--format", "{{json .}}"], output, "docker-version", budget.Token));
|
|
await Require(Command("docker", ["info", "--format", "{\"ID\":{{json .ID}},\"Name\":{{json .Name}},\"ServerVersion\":{{json .ServerVersion}},\"KernelVersion\":{{json .KernelVersion}},\"OperatingSystem\":{{json .OperatingSystem}},\"OSType\":{{json .OSType}},\"Architecture\":{{json .Architecture}}}"], output, "docker-daemon", budget.Token));
|
|
var image = await Command("docker", ["image", "inspect", Image], output, "image-before", budget.Token);
|
|
if (image.ExitCode != 0) await Require(Command("docker", ["pull", "--platform", "linux/amd64", Image], output, "image-pull", budget.Token));
|
|
await Require(Command("docker", ["image", "inspect", Image], output, "image-exact", budget.Token));
|
|
using var imageDocument = JsonDocument.Parse(Read(output, "image-exact", "stdout"));
|
|
var imageId = imageDocument.RootElement[0].GetProperty("Id").GetString();
|
|
var create = await Command("docker", ["create", "--platform", "linux/amd64", "--pull", "never", "--name", owner.Name,
|
|
"--label", Label + "=" + token, "--cidfile", Path.Combine(output, "container.id"), "--interactive", "--read-only",
|
|
"--network", "none", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "0.5",
|
|
"--memory", "256m", "--memory-swap", "256m", "--pids-limit", "32", "--no-healthcheck",
|
|
"--tmpfs", "/storage:" + StorageTmpfsOptions,
|
|
"--device", "/dev/kvm:/dev/kvm:rw", "--entrypoint", "/usr/bin/qemu-system-x86_64", Image, .. QemuArguments], output, "container-create", budget.Token);
|
|
if (create.ExitCode != 0)
|
|
{
|
|
status = ClassifyFailure(Read(output, "container-create", "stderr"));
|
|
detail = "Docker did not successfully create the device-mapped container; see container-create logs.";
|
|
}
|
|
else
|
|
{
|
|
var created = await InspectOwned(output, owner, "container-created", budget.Token);
|
|
ValidateBoundary(created, imageId);
|
|
var id = created.GetProperty("Id").GetString()!;
|
|
var monitor = await Command("docker", ["start", "--attach", "--interactive", id], output, "qemu-monitor", budget.Token, "info version\ninfo kvm\ninfo status\nquit\n");
|
|
var exited = await InspectOwned(output, owner, "container-exited", budget.Token);
|
|
var state = exited.GetProperty("State");
|
|
var text = Read(output, "qemu-monitor", "stdout");
|
|
var cleanExit = monitor.ExitCode == 0 && !monitor.TimedOut && !state.GetProperty("Running").GetBoolean()
|
|
&& state.GetProperty("ExitCode").GetInt32() == 0 && !state.GetProperty("OOMKilled").GetBoolean();
|
|
var enabled = Regex.IsMatch(text, @"(?m)^kvm support: enabled\r?$", RegexOptions.CultureInvariant);
|
|
var paused = Regex.IsMatch(text, @"(?m)^VM status: paused\r?$", RegexOptions.CultureInvariant);
|
|
status = cleanExit && enabled && paused ? "kvm_usable" : ClassifyFailure(Read(output, "qemu-monitor", "stderr") + "\n" + state.GetProperty("Error").GetString());
|
|
detail = status == "kvm_usable" ? "QEMU initialized KVM, reported enabled and paused, and exited successfully after quit. No guest CPU or OS was run."
|
|
: "KVM initialization or its enabled/paused/clean-exit proof did not pass; inspect raw monitor output and container state.";
|
|
}
|
|
}
|
|
catch (Exception error) { detail = error.Message; }
|
|
var cleaned = await Cleanup(output);
|
|
Save(output, "result.json", new { started, finished = DateTimeOffset.UtcNow, status, usable = status == "kvm_usable", cleaned, detail, image = Image, token });
|
|
Console.WriteLine(JsonSerializer.Serialize(new { status, cleaned, detail }));
|
|
return status == "kvm_usable" && cleaned ? 0 : 1;
|
|
}
|
|
|
|
static string ClassifyFailure(string text)
|
|
{
|
|
if (text.Contains("/dev/kvm", StringComparison.Ordinal) && text.Contains("error gathering device information", StringComparison.OrdinalIgnoreCase)
|
|
&& text.Contains("no such file or directory", StringComparison.OrdinalIgnoreCase)) return "daemon_device_missing";
|
|
if (text.Contains("Permission denied", StringComparison.OrdinalIgnoreCase) || text.Contains("Operation not permitted", StringComparison.OrdinalIgnoreCase)) return "access_denied_observed";
|
|
if (text.Contains("invalid accelerator kvm", StringComparison.OrdinalIgnoreCase)) return "qemu_kvm_backend_unavailable";
|
|
if (text.Contains("failed to initialize kvm", StringComparison.OrdinalIgnoreCase)) return "kvm_initialization_failed";
|
|
return "inconclusive";
|
|
}
|
|
|
|
static void ValidateBoundary(JsonElement container, string? imageId)
|
|
{
|
|
var host = container.GetProperty("HostConfig");
|
|
var devices = host.GetProperty("Devices");
|
|
if (container.GetProperty("Mounts").GetArrayLength() != 0)
|
|
throw new InvalidOperationException("Container mount boundary failed: persistent volumes or binds were created; expected Mounts=[] with only the read-only /storage tmpfs.");
|
|
if (!host.TryGetProperty("Tmpfs", out var tmpfs) || tmpfs.ValueKind != JsonValueKind.Object || tmpfs.EnumerateObject().Count() != 1
|
|
|| !tmpfs.TryGetProperty("/storage", out var options) || options.GetString() != StorageTmpfsOptions)
|
|
throw new InvalidOperationException("Container tmpfs boundary failed: expected only /storage:" + StorageTmpfsOptions + ".");
|
|
if (imageId == null || container.GetProperty("Image").GetString() != imageId || container.GetProperty("Config").GetProperty("Image").GetString() != Image
|
|
|| container.GetProperty("Path").GetString() != "/usr/bin/qemu-system-x86_64" || !container.GetProperty("Args").EnumerateArray().Select(x => x.GetString()).SequenceEqual(QemuArguments)
|
|
|| host.GetProperty("Privileged").GetBoolean() || !host.GetProperty("ReadonlyRootfs").GetBoolean()
|
|
|| host.GetProperty("NetworkMode").GetString() != "none"
|
|
|| devices.GetArrayLength() != 1 || devices[0].GetProperty("PathOnHost").GetString() != "/dev/kvm"
|
|
|| devices[0].GetProperty("PathInContainer").GetString() != "/dev/kvm" || devices[0].GetProperty("CgroupPermissions").GetString() != "rw"
|
|
|| !host.GetProperty("CapDrop").EnumerateArray().Any(x => x.GetString() == "ALL")
|
|
|| !Empty(host.GetProperty("CapAdd")) || !Empty(host.GetProperty("Binds")) || !Empty(host.GetProperty("PortBindings"))
|
|
|| !Empty(host.GetProperty("DeviceCgroupRules"))
|
|
|| !host.GetProperty("SecurityOpt").EnumerateArray().Any(x => x.GetString() == "no-new-privileges")
|
|
|| host.GetProperty("Memory").GetInt64() != 268435456 || host.GetProperty("MemorySwap").GetInt64() != 268435456
|
|
|| host.GetProperty("NanoCpus").GetInt64() != 500000000 || host.GetProperty("PidsLimit").GetInt64() != 32)
|
|
throw new InvalidOperationException("Created container does not match the diagnostic's restricted resource boundary.");
|
|
}
|
|
|
|
static bool Empty(JsonElement value) => value.ValueKind == JsonValueKind.Null
|
|
|| value.ValueKind == JsonValueKind.Array && value.GetArrayLength() == 0
|
|
|| value.ValueKind == JsonValueKind.Object && !value.EnumerateObject().Any();
|
|
|
|
static async Task<JsonElement> InspectOwned(string output, Owner owner, string step, CancellationToken cancellation)
|
|
{
|
|
var idPath = Path.Combine(output, "container.id");
|
|
var savedId = File.Exists(idPath) ? File.ReadAllText(idPath).Trim() : null;
|
|
if (savedId != null && !Regex.IsMatch(savedId, "^[a-f0-9]{64}$")) throw new InvalidOperationException("Saved container ID is invalid.");
|
|
await Require(Command("docker", ["inspect", "--type", "container", savedId ?? owner.Name], output, step, cancellation));
|
|
using var document = JsonDocument.Parse(Read(output, step, "stdout"));
|
|
var values = document.RootElement;
|
|
if (values.GetArrayLength() != 1) throw new InvalidOperationException("Container inspection did not return exactly one object.");
|
|
var value = values[0];
|
|
var id = value.GetProperty("Id").GetString()!;
|
|
if (!Regex.IsMatch(id, "^[a-f0-9]{64}$") || (savedId != null && id != savedId) || value.GetProperty("Name").GetString() != "/" + owner.Name
|
|
|| !value.GetProperty("Config").GetProperty("Labels").TryGetProperty(Label, out var label) || label.GetString() != owner.Token)
|
|
throw new InvalidOperationException("Container ownership ID/name/label mismatch; refusing resource operations.");
|
|
// Recover an interrupted create receipt by the saved random name and exact label, then use only its full ID.
|
|
if (savedId == null) File.WriteAllText(idPath, id + "\n");
|
|
return value.Clone();
|
|
}
|
|
|
|
static async Task<bool> Cleanup(string output)
|
|
{
|
|
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(20));
|
|
var prefix = "cleanup-" + Guid.NewGuid().ToString("N");
|
|
try
|
|
{
|
|
if (!File.Exists(Path.Combine(output, "owner.json"))) { Save(output, "cleanup.json", new { cleaned = true, reason = "No owned resource receipt exists." }); return true; }
|
|
var owner = JsonSerializer.Deserialize<Owner>(File.ReadAllText(Path.Combine(output, "owner.json")))!;
|
|
if (!Regex.IsMatch(owner.Token, "^[a-f0-9]{32}$") || owner.Name != "meeting-assistant-kvm-" + owner.Token)
|
|
throw new InvalidOperationException("Invalid saved ownership receipt.");
|
|
var container = await InspectOwned(output, owner, prefix + "-inspect", budget.Token);
|
|
var id = container.GetProperty("Id").GetString()!;
|
|
if (container.GetProperty("State").GetProperty("Running").GetBoolean())
|
|
{
|
|
await Command("docker", ["stop", "--time", "1", id], output, prefix + "-stop", budget.Token);
|
|
await InspectOwned(output, owner, prefix + "-reinspect", budget.Token);
|
|
}
|
|
await Require(Command("docker", ["rm", "--force", "--volumes", id], output, prefix + "-remove", budget.Token));
|
|
var receipt = new { cleaned = true, id, finished = DateTimeOffset.UtcNow };
|
|
Save(output, prefix + ".receipt.json", receipt);
|
|
Save(output, "cleanup.json", receipt);
|
|
return true;
|
|
}
|
|
catch (Exception error)
|
|
{
|
|
var path = Path.Combine(output, prefix + "-inspect.stderr.log");
|
|
var absent = File.Exists(path) && new FileInfo(path).Length <= 1024 * 1024
|
|
&& File.ReadAllText(path).Contains("No such container", StringComparison.OrdinalIgnoreCase);
|
|
var receipt = new { cleaned = absent, reason = error.Message, finished = DateTimeOffset.UtcNow };
|
|
Save(output, prefix + ".receipt.json", receipt);
|
|
Save(output, "cleanup.json", receipt);
|
|
return absent;
|
|
}
|
|
}
|
|
|
|
static async Task<bool> CleanupRun4165Volume(string output)
|
|
{
|
|
// Frozen from actual run 4165: the exact owner-labelled container never started (created/PID 0/zero StartedAt).
|
|
// Its image's VOLUME instruction created this anonymous /storage volume before the mount guard failed.
|
|
const string daemonId = "528941c8-73ac-49ff-8eb7-69113eb4a2a1";
|
|
const string containerId = "1753f95ef334244e7a1b393a839f218ea885363de7d5335eec53132d64627010";
|
|
const string token = "43b7f4676c514f2a95c63c02577ac36e";
|
|
const string volumeName = "ef7daa62ef89a2ffb8aae50a9b7803f1d9b3075ee509aa3183f3e170f69ce595";
|
|
const string containerName = "meeting-assistant-kvm-" + token;
|
|
var prefix = "run4165-volume-" + Guid.NewGuid().ToString("N");
|
|
Save(output, prefix + ".target.json", new { runId = 4165, daemonId, containerId, containerName, token, volumeName,
|
|
archiveSha256 = "6745d90e8b81c867740405c99b4364cc165c47ebb165455052314459d5cd547b",
|
|
createdInspectSha256 = "7afdfc6c30c933bee2ef1d6c18ed011c8b2f709d1a5e88531928f9f40471c055",
|
|
ownerReceiptSha256 = "d96f15588412a5928ebe8a64b115764f113adaaaf6d164209fd87de8a4505572",
|
|
priorState = "created", priorPid = 0, priorStartedAt = "0001-01-01T00:00:00Z" });
|
|
using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(20));
|
|
var success = false;
|
|
var outcome = "refused";
|
|
string? detail = null;
|
|
try
|
|
{
|
|
await Require(Command("docker", ["info", "--format", "{{.ID}}"], output, prefix + "-daemon", budget.Token));
|
|
if (Read(output, prefix + "-daemon", "stdout").Trim() != daemonId) throw new InvalidOperationException("Frozen run 4165 daemon ID differs; refusing volume cleanup.");
|
|
foreach (var selector in new[] { containerId, containerName })
|
|
{
|
|
var step = prefix + (selector == containerId ? "-old-id" : "-old-name");
|
|
var inspection = await Command("docker", ["inspect", "--type", "container", selector], output, step, budget.Token);
|
|
if (inspection.TimedOut || inspection.Error != null || inspection.ExitCode == 0 || !Read(output, step, "stderr").Contains("No such container", StringComparison.OrdinalIgnoreCase)
|
|
|| !Read(output, step, "stderr").Contains(selector, StringComparison.Ordinal)) throw new InvalidOperationException("Old run 4165 container absence was not proved for " + selector + ".");
|
|
}
|
|
var inspectStep = prefix + "-inspect";
|
|
var inspect = await Command("docker", ["volume", "inspect", volumeName], output, inspectStep, budget.Token);
|
|
if (VolumeAbsent(inspect, output, inspectStep, volumeName)) { success = true; outcome = "already-absent"; }
|
|
else
|
|
{
|
|
if (inspect.ExitCode != 0 || inspect.TimedOut || inspect.Error != null) throw new InvalidOperationException("Cannot inspect the exact run 4165 volume.");
|
|
using var document = JsonDocument.Parse(Read(output, inspectStep, "stdout"));
|
|
var values = document.RootElement;
|
|
if (values.GetArrayLength() != 1 || values[0].GetProperty("Name").GetString() != volumeName
|
|
|| values[0].GetProperty("Driver").GetString() != "local" || values[0].GetProperty("Scope").GetString() != "local"
|
|
|| !Empty(values[0].GetProperty("Options"))) throw new InvalidOperationException("Frozen volume name/local driver/scope/options boundary failed.");
|
|
await Require(Command("docker", ["ps", "--all", "--no-trunc", "--filter", "volume=" + volumeName, "--format", "{{.ID}}"], output, prefix + "-references", budget.Token));
|
|
if (Read(output, prefix + "-references", "stdout").Trim().Length != 0) throw new InvalidOperationException("A container references the frozen run 4165 volume; refusing removal.");
|
|
// No --force: Docker rejects an attachment made after the reference check, too.
|
|
await Require(Command("docker", ["volume", "rm", volumeName], output, prefix + "-remove", budget.Token));
|
|
var afterStep = prefix + "-after";
|
|
var after = await Command("docker", ["volume", "inspect", volumeName], output, afterStep, budget.Token);
|
|
if (!VolumeAbsent(after, output, afterStep, volumeName)) throw new InvalidOperationException("Volume absence after removal was not proved.");
|
|
success = true;
|
|
outcome = "removed";
|
|
}
|
|
}
|
|
catch (Exception error) { detail = error.Message; }
|
|
var receipt = new { runId = 4165, daemonId, volumeName, success, outcome, detail, finished = DateTimeOffset.UtcNow };
|
|
Save(output, prefix + ".receipt.json", receipt);
|
|
Console.WriteLine(JsonSerializer.Serialize(receipt));
|
|
return success;
|
|
}
|
|
|
|
static bool VolumeAbsent(CommandResult result, string output, string step, string name)
|
|
{
|
|
var error = Read(output, step, "stderr");
|
|
return result.ExitCode is not (null or 0) && !result.TimedOut && result.Error == null
|
|
&& error.Contains("no such volume", StringComparison.OrdinalIgnoreCase) && error.Contains(name, StringComparison.Ordinal);
|
|
}
|
|
|
|
static async Task<CommandResult> Command(string program, string[] arguments, string output, string step, CancellationToken cancellation, string? input = null)
|
|
{
|
|
var started = DateTimeOffset.UtcNow;
|
|
Save(output, step + ".command.json", new { program, arguments, input, started });
|
|
var start = new ProcessStartInfo(program) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = input != null };
|
|
foreach (var argument in arguments) start.ArgumentList.Add(argument);
|
|
using var process = new Process { StartInfo = start };
|
|
await using var stdout = File.Create(Path.Combine(output, step + ".stdout.log"));
|
|
await using var stderr = File.Create(Path.Combine(output, step + ".stderr.log"));
|
|
int? exit = null;
|
|
var timedOut = false;
|
|
string? error = null;
|
|
try
|
|
{
|
|
cancellation.ThrowIfCancellationRequested();
|
|
process.Start();
|
|
var copies = Task.WhenAll(process.StandardOutput.BaseStream.CopyToAsync(stdout), process.StandardError.BaseStream.CopyToAsync(stderr));
|
|
try
|
|
{
|
|
if (input != null)
|
|
{
|
|
try { await process.StandardInput.WriteAsync(input.AsMemory(), cancellation); await process.StandardInput.FlushAsync(cancellation); }
|
|
catch (IOException) { /* QEMU can reject KVM before accepting stdin; preserve its stderr and state. */ }
|
|
process.StandardInput.Close();
|
|
}
|
|
await process.WaitForExitAsync(cancellation);
|
|
}
|
|
catch (OperationCanceledException) { timedOut = true; if (!process.HasExited) process.Kill(entireProcessTree: true); }
|
|
await copies.WaitAsync(TimeSpan.FromSeconds(2));
|
|
if (process.HasExited) exit = process.ExitCode;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
error = exception.Message;
|
|
try { if (!process.HasExited) process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { /* Process never started or already exited. */ }
|
|
}
|
|
var result = new CommandResult(exit, timedOut, error);
|
|
Save(output, step + ".result.json", new { result.ExitCode, result.TimedOut, result.Error, started, finished = DateTimeOffset.UtcNow });
|
|
return result;
|
|
}
|
|
|
|
static async Task Require(Task<CommandResult> command)
|
|
{
|
|
var result = await command;
|
|
if (result.ExitCode != 0 || result.TimedOut || result.Error != null) throw new InvalidOperationException($"Command did not succeed: exit={result.ExitCode}, timedOut={result.TimedOut}, error={result.Error}");
|
|
}
|
|
static string Read(string output, string step, string stream)
|
|
{
|
|
var path = Path.Combine(output, step + "." + stream + ".log");
|
|
if (new FileInfo(path).Length > 1024 * 1024) throw new InvalidOperationException("Diagnostic output exceeds the one-MiB interpretation limit; inspect the retained raw log.");
|
|
return File.ReadAllText(path);
|
|
}
|
|
static void Save(string output, string name, object value) => File.WriteAllText(Path.Combine(output, name), JsonSerializer.Serialize(value, Json) + "\n");
|
|
sealed record Owner(string Token, string Name);
|
|
sealed record CommandResult(int? ExitCode, bool TimedOut, string? Error);
|
|
}
|