forked from Manuel/meeting-assistant
218 lines
12 KiB
C#
218 lines
12 KiB
C#
using System.Xml.Linq;
|
|
|
|
#if !PLATFORM_TEST_VERIFIER_LIBRARY
|
|
if (args.Length != 3)
|
|
{
|
|
Console.Error.WriteLine("Usage: dotnet run tools/VerifyPlatformTests.cs -- <portable|windows> <discovery.txt> <results.trx>");
|
|
return 2;
|
|
}
|
|
|
|
try
|
|
{
|
|
var verification = PlatformTestVerification.Verify(args[0], File.ReadAllText(args[1]), XDocument.Load(args[2]));
|
|
foreach (var line in verification.Lines)
|
|
{
|
|
Console.WriteLine(line);
|
|
}
|
|
return 0;
|
|
}
|
|
catch (Exception exception) when (exception is IOException or InvalidDataException or System.Xml.XmlException or ArgumentException)
|
|
{
|
|
Console.Error.WriteLine($"Platform test assurance failed: {exception.Message}");
|
|
return 1;
|
|
}
|
|
#endif
|
|
|
|
public sealed record PlatformTestVerificationResult(int Total, int Passed, int Skipped, IReadOnlyList<string> Lines);
|
|
|
|
public static class PlatformTestVerification
|
|
{
|
|
private static readonly XNamespace Trx = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010";
|
|
private const string CompositionGuard = "MeetingAssistant.Tests.TestTargetCompositionTests.TestAndApplicationTargetsUseMatchingComposition";
|
|
private static readonly string[] WindowsRuntimeTests =
|
|
[
|
|
"MeetingAssistant.Tests.AzureSpeechStreamingTranscriptionProviderTests.ResolveKeyFallsBackToUserEnvironment",
|
|
"MeetingAssistant.Tests.TaskbarIconTests.TaskbarIconGlyphsAreVisuallyCentered"
|
|
];
|
|
private const string OutlookClass = "MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.";
|
|
private static readonly string[] OutlookCases =
|
|
[
|
|
OutlookClass + "ExtractAgendaStopsBeforeTeamsJoinInformation",
|
|
OutlookClass + "NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail",
|
|
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Canceled: Project sync\")",
|
|
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Cancelled: Project sync\")",
|
|
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Abgesagt: Project sync\")"
|
|
];
|
|
private const string WindowsCropClass = "MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.";
|
|
private static readonly string[] WindowsCropCases =
|
|
[
|
|
WindowsCropClass + "WindowsCropUsesOcrPixelCoordinates",
|
|
WindowsCropClass + "WindowsCropPreservesExpectedPixelsAtNonzeroOrigin",
|
|
WindowsCropClass + "WindowsCropRejectsInvalidBounds"
|
|
];
|
|
|
|
public static PlatformTestVerificationResult Verify(string target, string discovery, XDocument trx)
|
|
{
|
|
if (target is not ("portable" or "windows"))
|
|
{
|
|
throw new ArgumentException("Target must be portable or windows.", nameof(target));
|
|
}
|
|
var records = trx.Root?.Element(Trx + "Results")?.Elements(Trx + "UnitTestResult").ToArray() ?? [];
|
|
if (records.Length == 0)
|
|
{
|
|
throw new InvalidDataException("TRX has no execution records.");
|
|
}
|
|
var guards = records.Where(record => (string?)record.Attribute("testName") == CompositionGuard).ToArray();
|
|
if (guards.Length != 1 || (string?)guards[0].Attribute("outcome") != "Passed")
|
|
{
|
|
throw new InvalidDataException("The composition guard must have exactly one Passed result.");
|
|
}
|
|
var markers = guards[0].Descendants().Where(element => element.Name.LocalName == "StdOut")
|
|
.SelectMany(element => element.Value.Split('\n', StringSplitOptions.TrimEntries))
|
|
.Where(line => line.StartsWith("platform-assurance:", StringComparison.Ordinal)).ToArray();
|
|
var prefix = $"platform-assurance: target={target}; host=";
|
|
if (markers.Length != 1 || !markers[0].StartsWith(prefix, StringComparison.Ordinal)
|
|
|| markers[0][prefix.Length..] is not ("windows" or "macos" or "linux" or "other"))
|
|
{
|
|
throw new InvalidDataException("The composition guard must report one matching target and runtime host marker.");
|
|
}
|
|
var host = markers[0][prefix.Length..];
|
|
if (target == "windows" && host != "windows")
|
|
{
|
|
throw new InvalidDataException("The Windows target requires a Windows runtime reported by the passed composition guard.");
|
|
}
|
|
var lines = new List<string>
|
|
{
|
|
$"Target: {target}; runtime host reported by passed composition guard: {host}.",
|
|
$"Passed: {CompositionGuard}"
|
|
};
|
|
var discovered = discovery.Split('\n', StringSplitOptions.TrimEntries)
|
|
.Where(line => line.StartsWith("MeetingAssistant.Tests.", StringComparison.Ordinal)).ToArray();
|
|
var names = records.Select(record => (string?)record.Attribute("testName") ?? "").ToArray();
|
|
if (discovered.Length == 0 || names.Any(string.IsNullOrWhiteSpace))
|
|
{
|
|
throw new InvalidDataException("Discovery and execution records must contain named tests.");
|
|
}
|
|
foreach (var set in new[] { discovered, names })
|
|
{
|
|
var duplicate = set.GroupBy(name => name, StringComparer.Ordinal).FirstOrDefault(group => group.Count() != 1);
|
|
if (duplicate is not null)
|
|
{
|
|
throw new InvalidDataException($"Duplicate discovery or execution case: {duplicate.Key}");
|
|
}
|
|
}
|
|
var missing = discovered.Except(names, StringComparer.Ordinal).FirstOrDefault()
|
|
?? names.Except(discovered, StringComparer.Ordinal).FirstOrDefault();
|
|
if (missing is not null)
|
|
{
|
|
throw new InvalidDataException($"Discovery and execution cases differ: {missing}");
|
|
}
|
|
foreach (var record in records)
|
|
{
|
|
if ((string?)record.Attribute("outcome") is not ("Passed" or "NotExecuted"))
|
|
{
|
|
throw new InvalidDataException($"Case did not pass or skip: {(string?)record.Attribute("testName")} ({(string?)record.Attribute("outcome")}).");
|
|
}
|
|
if ((string?)record.Attribute("outcome") == "NotExecuted" && string.IsNullOrWhiteSpace(SkipReason(record)))
|
|
{
|
|
throw new InvalidDataException($"Skipped case has no reason: {(string?)record.Attribute("testName")}");
|
|
}
|
|
}
|
|
if (target == "portable")
|
|
{
|
|
if (discovered.Any(name => name.StartsWith(OutlookClass, StringComparison.Ordinal))
|
|
|| records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(OutlookClass, StringComparison.Ordinal) == true))
|
|
{
|
|
throw new InvalidDataException("The portable target must exclude the Windows-only Outlook cases.");
|
|
}
|
|
lines.Add("Outlook cases: excluded from portable target (5 Windows-only cases).");
|
|
if (discovered.Any(name => name.StartsWith(WindowsCropClass, StringComparison.Ordinal))
|
|
|| records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(WindowsCropClass, StringComparison.Ordinal) == true))
|
|
{
|
|
throw new InvalidDataException("The portable target must exclude the native Windows crop cases.");
|
|
}
|
|
lines.Add("Windows crop cases: excluded from portable target (3 Windows-only native cases).");
|
|
}
|
|
else
|
|
{
|
|
foreach (var name in OutlookCases)
|
|
{
|
|
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
|
if (discovered.Count(discoveredName => discoveredName == name) != 1
|
|
|| matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed")
|
|
{
|
|
throw new InvalidDataException($"Outlook case must be discovered once and have exactly one Passed result: {name}");
|
|
}
|
|
lines.Add($"Passed: {name}");
|
|
}
|
|
foreach (var name in WindowsCropCases)
|
|
{
|
|
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
|
if (discovered.Count(discoveredName => discoveredName == name) != 1
|
|
|| matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed")
|
|
{
|
|
throw new InvalidDataException($"Windows crop case must be discovered once and have exactly one Passed result: {name}");
|
|
}
|
|
lines.Add($"Passed: {name}");
|
|
}
|
|
}
|
|
foreach (var name in WindowsRuntimeTests)
|
|
{
|
|
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
|
var expectedOutcome = host == "windows" ? "Passed" : "NotExecuted";
|
|
if (matches.Length != 1 || (string?)matches[0].Attribute("outcome") != expectedOutcome)
|
|
{
|
|
throw new InvalidDataException($"{name} must have exactly one {expectedOutcome} result on runtime host {host}.");
|
|
}
|
|
if (expectedOutcome == "NotExecuted" && !SkipReason(matches[0]).Contains("Windows", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
throw new InvalidDataException($"{name} must report its incompatible Windows capability as the skip reason.");
|
|
}
|
|
if (expectedOutcome == "Passed")
|
|
{
|
|
lines.Add($"Passed: {name}");
|
|
}
|
|
}
|
|
foreach (var record in records.Where(record => (string?)record.Attribute("outcome") == "NotExecuted"))
|
|
{
|
|
lines.Add($"Skipped: {(string?)record.Attribute("testName")} — {SkipReason(record)}");
|
|
}
|
|
var passed = records.Count(record => (string?)record.Attribute("outcome") == "Passed");
|
|
var skipped = records.Count(record => (string?)record.Attribute("outcome") == "NotExecuted");
|
|
var summary = trx.Root?.Element(Trx + "ResultSummary");
|
|
var counters = summary?.Element(Trx + "Counters");
|
|
if ((string?)summary?.Attribute("outcome") != "Completed" || counters is null)
|
|
{
|
|
throw new InvalidDataException("TRX summary must report a completed run with counters.");
|
|
}
|
|
CheckCounter(counters, "total", records.Length);
|
|
CheckCounter(counters, "executed", passed);
|
|
CheckCounter(counters, "passed", passed);
|
|
CheckCounter(counters, "failed", 0);
|
|
foreach (var counter in new[] { "error", "timeout", "aborted", "inconclusive", "passedButRunAborted", "notRunnable", "disconnected", "inProgress", "pending" })
|
|
{
|
|
if (counters.Attribute(counter) is not null)
|
|
{
|
|
CheckCounter(counters, counter, 0);
|
|
}
|
|
}
|
|
// xUnit VSTest emits notExecuted=0 even for NotExecuted result records.
|
|
// Count actual skip records rather than trusting this adapter summary field.
|
|
lines.Add($"Totals: {discovered.Length} discovered, {records.Length} results, {passed} passed, {skipped} skipped, {records.Length - passed - skipped} failed.");
|
|
lines.Add("Test artifacts only; full Windows resource/PRI builds and native operational acceptance require separate evidence.");
|
|
return new(records.Length, passed, skipped, lines);
|
|
}
|
|
|
|
private static string SkipReason(XElement result) => string.Join(" ", result.Descendants()
|
|
.Where(element => element.Name.LocalName is "Message" or "StdOut")
|
|
.Select(element => element.Value.Trim()).Where(value => value.Length > 0));
|
|
|
|
private static void CheckCounter(XElement counters, string name, int expected)
|
|
{
|
|
if (!int.TryParse((string?)counters.Attribute(name), out var actual) || actual != expected)
|
|
{
|
|
throw new InvalidDataException($"TRX summary counter {name} must match the execution records ({expected}).");
|
|
}
|
|
}
|
|
}
|