using System.Xml.Linq; #if !PLATFORM_TEST_VERIFIER_LIBRARY if (args.Length != 3) { Console.Error.WriteLine("Usage: dotnet run tools/VerifyPlatformTests.cs -- "); return 2; } try { var verification = PlatformTestVerification.Verify(args[0], File.ReadAllText(args[1]), XDocument.Load(args[2])); foreach (var line in verification.Lines) { Console.WriteLine(line); } return 0; } catch (Exception exception) when (exception is IOException or InvalidDataException or System.Xml.XmlException or ArgumentException) { Console.Error.WriteLine($"Platform test assurance failed: {exception.Message}"); return 1; } #endif public sealed record PlatformTestVerificationResult(int Total, int Passed, int Skipped, IReadOnlyList Lines); public static class PlatformTestVerification { private static readonly XNamespace Trx = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; private const string CompositionGuard = "MeetingAssistant.Tests.TestTargetCompositionTests.TestAndApplicationTargetsUseMatchingComposition"; private static readonly string[] WindowsRuntimeTests = [ "MeetingAssistant.Tests.AzureSpeechStreamingTranscriptionProviderTests.ResolveKeyFallsBackToUserEnvironment", "MeetingAssistant.Tests.TaskbarIconTests.TaskbarIconGlyphsAreVisuallyCentered" ]; private const string OutlookClass = "MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests."; private static readonly string[] OutlookCases = [ OutlookClass + "ExtractAgendaStopsBeforeTeamsJoinInformation", OutlookClass + "NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail", OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Canceled: Project sync\")", OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Cancelled: Project sync\")", OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Abgesagt: Project sync\")" ]; private const string WindowsCropClass = "MeetingAssistant.Tests.WindowsScreenshotImageCropperTests."; private static readonly string[] WindowsCropCases = [ WindowsCropClass + "WindowsCropUsesOcrPixelCoordinates", WindowsCropClass + "WindowsCropPreservesExpectedPixelsAtNonzeroOrigin", WindowsCropClass + "WindowsCropRejectsInvalidBounds" ]; public static PlatformTestVerificationResult Verify(string target, string discovery, XDocument trx) { if (target is not ("portable" or "windows")) { throw new ArgumentException("Target must be portable or windows.", nameof(target)); } var records = trx.Root?.Element(Trx + "Results")?.Elements(Trx + "UnitTestResult").ToArray() ?? []; if (records.Length == 0) { throw new InvalidDataException("TRX has no execution records."); } var guards = records.Where(record => (string?)record.Attribute("testName") == CompositionGuard).ToArray(); if (guards.Length != 1 || (string?)guards[0].Attribute("outcome") != "Passed") { throw new InvalidDataException("The composition guard must have exactly one Passed result."); } var markers = guards[0].Descendants().Where(element => element.Name.LocalName == "StdOut") .SelectMany(element => element.Value.Split('\n', StringSplitOptions.TrimEntries)) .Where(line => line.StartsWith("platform-assurance:", StringComparison.Ordinal)).ToArray(); var prefix = $"platform-assurance: target={target}; host="; if (markers.Length != 1 || !markers[0].StartsWith(prefix, StringComparison.Ordinal) || markers[0][prefix.Length..] is not ("windows" or "macos" or "linux" or "other")) { throw new InvalidDataException("The composition guard must report one matching target and runtime host marker."); } var host = markers[0][prefix.Length..]; if (target == "windows" && host != "windows") { throw new InvalidDataException("The Windows target requires a Windows runtime reported by the passed composition guard."); } var lines = new List { $"Target: {target}; runtime host reported by passed composition guard: {host}.", $"Passed: {CompositionGuard}" }; var discovered = discovery.Split('\n', StringSplitOptions.TrimEntries) .Where(line => line.StartsWith("MeetingAssistant.Tests.", StringComparison.Ordinal)).ToArray(); var names = records.Select(record => (string?)record.Attribute("testName") ?? "").ToArray(); if (discovered.Length == 0 || names.Any(string.IsNullOrWhiteSpace)) { throw new InvalidDataException("Discovery and execution records must contain named tests."); } foreach (var set in new[] { discovered, names }) { var duplicate = set.GroupBy(name => name, StringComparer.Ordinal).FirstOrDefault(group => group.Count() != 1); if (duplicate is not null) { throw new InvalidDataException($"Duplicate discovery or execution case: {duplicate.Key}"); } } var missing = discovered.Except(names, StringComparer.Ordinal).FirstOrDefault() ?? names.Except(discovered, StringComparer.Ordinal).FirstOrDefault(); if (missing is not null) { throw new InvalidDataException($"Discovery and execution cases differ: {missing}"); } foreach (var record in records) { if ((string?)record.Attribute("outcome") is not ("Passed" or "NotExecuted")) { throw new InvalidDataException($"Case did not pass or skip: {(string?)record.Attribute("testName")} ({(string?)record.Attribute("outcome")})."); } if ((string?)record.Attribute("outcome") == "NotExecuted" && string.IsNullOrWhiteSpace(SkipReason(record))) { throw new InvalidDataException($"Skipped case has no reason: {(string?)record.Attribute("testName")}"); } } if (target == "portable") { if (discovered.Any(name => name.StartsWith(OutlookClass, StringComparison.Ordinal)) || records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(OutlookClass, StringComparison.Ordinal) == true)) { throw new InvalidDataException("The portable target must exclude the Windows-only Outlook cases."); } lines.Add("Outlook cases: excluded from portable target (5 Windows-only cases)."); if (discovered.Any(name => name.StartsWith(WindowsCropClass, StringComparison.Ordinal)) || records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(WindowsCropClass, StringComparison.Ordinal) == true)) { throw new InvalidDataException("The portable target must exclude the native Windows crop cases."); } lines.Add("Windows crop cases: excluded from portable target (3 Windows-only native cases)."); } else { foreach (var name in OutlookCases) { var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray(); if (discovered.Count(discoveredName => discoveredName == name) != 1 || matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed") { throw new InvalidDataException($"Outlook case must be discovered once and have exactly one Passed result: {name}"); } lines.Add($"Passed: {name}"); } foreach (var name in WindowsCropCases) { var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray(); if (discovered.Count(discoveredName => discoveredName == name) != 1 || matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed") { throw new InvalidDataException($"Windows crop case must be discovered once and have exactly one Passed result: {name}"); } lines.Add($"Passed: {name}"); } } foreach (var name in WindowsRuntimeTests) { var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray(); var expectedOutcome = host == "windows" ? "Passed" : "NotExecuted"; if (matches.Length != 1 || (string?)matches[0].Attribute("outcome") != expectedOutcome) { throw new InvalidDataException($"{name} must have exactly one {expectedOutcome} result on runtime host {host}."); } if (expectedOutcome == "NotExecuted" && !SkipReason(matches[0]).Contains("Windows", StringComparison.OrdinalIgnoreCase)) { throw new InvalidDataException($"{name} must report its incompatible Windows capability as the skip reason."); } if (expectedOutcome == "Passed") { lines.Add($"Passed: {name}"); } } foreach (var record in records.Where(record => (string?)record.Attribute("outcome") == "NotExecuted")) { lines.Add($"Skipped: {(string?)record.Attribute("testName")} — {SkipReason(record)}"); } var passed = records.Count(record => (string?)record.Attribute("outcome") == "Passed"); var skipped = records.Count(record => (string?)record.Attribute("outcome") == "NotExecuted"); var summary = trx.Root?.Element(Trx + "ResultSummary"); var counters = summary?.Element(Trx + "Counters"); if ((string?)summary?.Attribute("outcome") != "Completed" || counters is null) { throw new InvalidDataException("TRX summary must report a completed run with counters."); } CheckCounter(counters, "total", records.Length); CheckCounter(counters, "executed", passed); CheckCounter(counters, "passed", passed); CheckCounter(counters, "failed", 0); foreach (var counter in new[] { "error", "timeout", "aborted", "inconclusive", "passedButRunAborted", "notRunnable", "disconnected", "inProgress", "pending" }) { if (counters.Attribute(counter) is not null) { CheckCounter(counters, counter, 0); } } // xUnit VSTest emits notExecuted=0 even for NotExecuted result records. // Count actual skip records rather than trusting this adapter summary field. lines.Add($"Totals: {discovered.Length} discovered, {records.Length} results, {passed} passed, {skipped} skipped, {records.Length - passed - skipped} failed."); lines.Add("Test artifacts only; full Windows resource/PRI builds and native operational acceptance require separate evidence."); return new(records.Length, passed, skipped, lines); } private static string SkipReason(XElement result) => string.Join(" ", result.Descendants() .Where(element => element.Name.LocalName is "Message" or "StdOut") .Select(element => element.Value.Trim()).Where(value => value.Length > 0)); private static void CheckCounter(XElement counters, string name, int expected) { if (!int.TryParse((string?)counters.Attribute(name), out var actual) || actual != expected) { throw new InvalidDataException($"TRX summary counter {name} must match the execution records ({expected})."); } } }