forked from Manuel/meeting-assistant
ci: consolidate platform pipelines and test prerequisites on macOS support
This commit is contained in:
dh committed
1 parent
1b19b08f2e
commit
728e66dd72
76 files changed
+7477
-239
No files matched your search
@@ -0,0 +1,217 @@
|
||||
using System.Xml.Linq;
|
||||
|
||||
#if !PLATFORM_TEST_VERIFIER_LIBRARY
|
||||
if (args.Length != 3)
|
||||
{
|
||||
Console.Error.WriteLine("Usage: dotnet run tools/VerifyPlatformTests.cs -- <portable|windows> <discovery.txt> <results.trx>");
|
||||
return 2;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var verification = PlatformTestVerification.Verify(args[0], File.ReadAllText(args[1]), XDocument.Load(args[2]));
|
||||
foreach (var line in verification.Lines)
|
||||
{
|
||||
Console.WriteLine(line);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or InvalidDataException or System.Xml.XmlException or ArgumentException)
|
||||
{
|
||||
Console.Error.WriteLine($"Platform test assurance failed: {exception.Message}");
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
public sealed record PlatformTestVerificationResult(int Total, int Passed, int Skipped, IReadOnlyList<string> Lines);
|
||||
|
||||
public static class PlatformTestVerification
|
||||
{
|
||||
private static readonly XNamespace Trx = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010";
|
||||
private const string CompositionGuard = "MeetingAssistant.Tests.TestTargetCompositionTests.TestAndApplicationTargetsUseMatchingComposition";
|
||||
private static readonly string[] WindowsRuntimeTests =
|
||||
[
|
||||
"MeetingAssistant.Tests.AzureSpeechStreamingTranscriptionProviderTests.ResolveKeyFallsBackToUserEnvironment",
|
||||
"MeetingAssistant.Tests.TaskbarIconTests.TaskbarIconGlyphsAreVisuallyCentered"
|
||||
];
|
||||
private const string OutlookClass = "MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.";
|
||||
private static readonly string[] OutlookCases =
|
||||
[
|
||||
OutlookClass + "ExtractAgendaStopsBeforeTeamsJoinInformation",
|
||||
OutlookClass + "NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail",
|
||||
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Canceled: Project sync\")",
|
||||
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Cancelled: Project sync\")",
|
||||
OutlookClass + "SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Abgesagt: Project sync\")"
|
||||
];
|
||||
private const string WindowsCropClass = "MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.";
|
||||
private static readonly string[] WindowsCropCases =
|
||||
[
|
||||
WindowsCropClass + "WindowsCropUsesOcrPixelCoordinates",
|
||||
WindowsCropClass + "WindowsCropPreservesExpectedPixelsAtNonzeroOrigin",
|
||||
WindowsCropClass + "WindowsCropRejectsInvalidBounds"
|
||||
];
|
||||
|
||||
public static PlatformTestVerificationResult Verify(string target, string discovery, XDocument trx)
|
||||
{
|
||||
if (target is not ("portable" or "windows"))
|
||||
{
|
||||
throw new ArgumentException("Target must be portable or windows.", nameof(target));
|
||||
}
|
||||
var records = trx.Root?.Element(Trx + "Results")?.Elements(Trx + "UnitTestResult").ToArray() ?? [];
|
||||
if (records.Length == 0)
|
||||
{
|
||||
throw new InvalidDataException("TRX has no execution records.");
|
||||
}
|
||||
var guards = records.Where(record => (string?)record.Attribute("testName") == CompositionGuard).ToArray();
|
||||
if (guards.Length != 1 || (string?)guards[0].Attribute("outcome") != "Passed")
|
||||
{
|
||||
throw new InvalidDataException("The composition guard must have exactly one Passed result.");
|
||||
}
|
||||
var markers = guards[0].Descendants().Where(element => element.Name.LocalName == "StdOut")
|
||||
.SelectMany(element => element.Value.Split('\n', StringSplitOptions.TrimEntries))
|
||||
.Where(line => line.StartsWith("platform-assurance:", StringComparison.Ordinal)).ToArray();
|
||||
var prefix = $"platform-assurance: target={target}; host=";
|
||||
if (markers.Length != 1 || !markers[0].StartsWith(prefix, StringComparison.Ordinal)
|
||||
|| markers[0][prefix.Length..] is not ("windows" or "macos" or "linux" or "other"))
|
||||
{
|
||||
throw new InvalidDataException("The composition guard must report one matching target and runtime host marker.");
|
||||
}
|
||||
var host = markers[0][prefix.Length..];
|
||||
if (target == "windows" && host != "windows")
|
||||
{
|
||||
throw new InvalidDataException("The Windows target requires a Windows runtime reported by the passed composition guard.");
|
||||
}
|
||||
var lines = new List<string>
|
||||
{
|
||||
$"Target: {target}; runtime host reported by passed composition guard: {host}.",
|
||||
$"Passed: {CompositionGuard}"
|
||||
};
|
||||
var discovered = discovery.Split('\n', StringSplitOptions.TrimEntries)
|
||||
.Where(line => line.StartsWith("MeetingAssistant.Tests.", StringComparison.Ordinal)).ToArray();
|
||||
var names = records.Select(record => (string?)record.Attribute("testName") ?? "").ToArray();
|
||||
if (discovered.Length == 0 || names.Any(string.IsNullOrWhiteSpace))
|
||||
{
|
||||
throw new InvalidDataException("Discovery and execution records must contain named tests.");
|
||||
}
|
||||
foreach (var set in new[] { discovered, names })
|
||||
{
|
||||
var duplicate = set.GroupBy(name => name, StringComparer.Ordinal).FirstOrDefault(group => group.Count() != 1);
|
||||
if (duplicate is not null)
|
||||
{
|
||||
throw new InvalidDataException($"Duplicate discovery or execution case: {duplicate.Key}");
|
||||
}
|
||||
}
|
||||
var missing = discovered.Except(names, StringComparer.Ordinal).FirstOrDefault()
|
||||
?? names.Except(discovered, StringComparer.Ordinal).FirstOrDefault();
|
||||
if (missing is not null)
|
||||
{
|
||||
throw new InvalidDataException($"Discovery and execution cases differ: {missing}");
|
||||
}
|
||||
foreach (var record in records)
|
||||
{
|
||||
if ((string?)record.Attribute("outcome") is not ("Passed" or "NotExecuted"))
|
||||
{
|
||||
throw new InvalidDataException($"Case did not pass or skip: {(string?)record.Attribute("testName")} ({(string?)record.Attribute("outcome")}).");
|
||||
}
|
||||
if ((string?)record.Attribute("outcome") == "NotExecuted" && string.IsNullOrWhiteSpace(SkipReason(record)))
|
||||
{
|
||||
throw new InvalidDataException($"Skipped case has no reason: {(string?)record.Attribute("testName")}");
|
||||
}
|
||||
}
|
||||
if (target == "portable")
|
||||
{
|
||||
if (discovered.Any(name => name.StartsWith(OutlookClass, StringComparison.Ordinal))
|
||||
|| records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(OutlookClass, StringComparison.Ordinal) == true))
|
||||
{
|
||||
throw new InvalidDataException("The portable target must exclude the Windows-only Outlook cases.");
|
||||
}
|
||||
lines.Add("Outlook cases: excluded from portable target (5 Windows-only cases).");
|
||||
if (discovered.Any(name => name.StartsWith(WindowsCropClass, StringComparison.Ordinal))
|
||||
|| records.Any(record => ((string?)record.Attribute("testName"))?.StartsWith(WindowsCropClass, StringComparison.Ordinal) == true))
|
||||
{
|
||||
throw new InvalidDataException("The portable target must exclude the native Windows crop cases.");
|
||||
}
|
||||
lines.Add("Windows crop cases: excluded from portable target (3 Windows-only native cases).");
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var name in OutlookCases)
|
||||
{
|
||||
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
||||
if (discovered.Count(discoveredName => discoveredName == name) != 1
|
||||
|| matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed")
|
||||
{
|
||||
throw new InvalidDataException($"Outlook case must be discovered once and have exactly one Passed result: {name}");
|
||||
}
|
||||
lines.Add($"Passed: {name}");
|
||||
}
|
||||
foreach (var name in WindowsCropCases)
|
||||
{
|
||||
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
||||
if (discovered.Count(discoveredName => discoveredName == name) != 1
|
||||
|| matches.Length != 1 || (string?)matches[0].Attribute("outcome") != "Passed")
|
||||
{
|
||||
throw new InvalidDataException($"Windows crop case must be discovered once and have exactly one Passed result: {name}");
|
||||
}
|
||||
lines.Add($"Passed: {name}");
|
||||
}
|
||||
}
|
||||
foreach (var name in WindowsRuntimeTests)
|
||||
{
|
||||
var matches = records.Where(record => (string?)record.Attribute("testName") == name).ToArray();
|
||||
var expectedOutcome = host == "windows" ? "Passed" : "NotExecuted";
|
||||
if (matches.Length != 1 || (string?)matches[0].Attribute("outcome") != expectedOutcome)
|
||||
{
|
||||
throw new InvalidDataException($"{name} must have exactly one {expectedOutcome} result on runtime host {host}.");
|
||||
}
|
||||
if (expectedOutcome == "NotExecuted" && !SkipReason(matches[0]).Contains("Windows", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidDataException($"{name} must report its incompatible Windows capability as the skip reason.");
|
||||
}
|
||||
if (expectedOutcome == "Passed")
|
||||
{
|
||||
lines.Add($"Passed: {name}");
|
||||
}
|
||||
}
|
||||
foreach (var record in records.Where(record => (string?)record.Attribute("outcome") == "NotExecuted"))
|
||||
{
|
||||
lines.Add($"Skipped: {(string?)record.Attribute("testName")} — {SkipReason(record)}");
|
||||
}
|
||||
var passed = records.Count(record => (string?)record.Attribute("outcome") == "Passed");
|
||||
var skipped = records.Count(record => (string?)record.Attribute("outcome") == "NotExecuted");
|
||||
var summary = trx.Root?.Element(Trx + "ResultSummary");
|
||||
var counters = summary?.Element(Trx + "Counters");
|
||||
if ((string?)summary?.Attribute("outcome") != "Completed" || counters is null)
|
||||
{
|
||||
throw new InvalidDataException("TRX summary must report a completed run with counters.");
|
||||
}
|
||||
CheckCounter(counters, "total", records.Length);
|
||||
CheckCounter(counters, "executed", passed);
|
||||
CheckCounter(counters, "passed", passed);
|
||||
CheckCounter(counters, "failed", 0);
|
||||
foreach (var counter in new[] { "error", "timeout", "aborted", "inconclusive", "passedButRunAborted", "notRunnable", "disconnected", "inProgress", "pending" })
|
||||
{
|
||||
if (counters.Attribute(counter) is not null)
|
||||
{
|
||||
CheckCounter(counters, counter, 0);
|
||||
}
|
||||
}
|
||||
// xUnit VSTest emits notExecuted=0 even for NotExecuted result records.
|
||||
// Count actual skip records rather than trusting this adapter summary field.
|
||||
lines.Add($"Totals: {discovered.Length} discovered, {records.Length} results, {passed} passed, {skipped} skipped, {records.Length - passed - skipped} failed.");
|
||||
lines.Add("Test artifacts only; full Windows resource/PRI builds and native operational acceptance require separate evidence.");
|
||||
return new(records.Length, passed, skipped, lines);
|
||||
}
|
||||
|
||||
private static string SkipReason(XElement result) => string.Join(" ", result.Descendants()
|
||||
.Where(element => element.Name.LocalName is "Message" or "StdOut")
|
||||
.Select(element => element.Value.Trim()).Where(value => value.Length > 0));
|
||||
|
||||
private static void CheckCounter(XElement counters, string name, int expected)
|
||||
{
|
||||
if (!int.TryParse((string?)counters.Attribute(name), out var actual) || actual != expected)
|
||||
{
|
||||
throw new InvalidDataException($"TRX summary counter {name} must match the execution records ({expected}).");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user