forked from Manuel/meeting-assistant
ci: consolidate platform pipelines and test prerequisites on macOS support
This commit is contained in:
dh committed
1 parent
1b19b08f2e
commit
728e66dd72
76 files changed
+7477
-239
No files matched your search
@@ -0,0 +1,377 @@
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
// These are synthetic verifier fixtures, not execution evidence from a Windows host.
|
||||
public sealed class PlatformTestVerificationTests
|
||||
{
|
||||
private const string Guard = "MeetingAssistant.Tests.TestTargetCompositionTests.TestAndApplicationTargetsUseMatchingComposition";
|
||||
private const string UserEnvironment = "MeetingAssistant.Tests.AzureSpeechStreamingTranscriptionProviderTests.ResolveKeyFallsBackToUserEnvironment";
|
||||
private const string Glyphs = "MeetingAssistant.Tests.TaskbarIconTests.TaskbarIconGlyphsAreVisuallyCentered";
|
||||
private static readonly XNamespace Trx = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010";
|
||||
private static readonly string[] OutlookCases =
|
||||
[
|
||||
"MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.ExtractAgendaStopsBeforeTeamsJoinInformation",
|
||||
"MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail",
|
||||
"MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Canceled: Project sync\")",
|
||||
"MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Cancelled: Project sync\")",
|
||||
"MeetingAssistant.Tests.OutlookClassicMeetingMetadataProviderTests.SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(subject: \"Abgesagt: Project sync\")"
|
||||
];
|
||||
private static readonly string[] WindowsCropCases =
|
||||
[
|
||||
"MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.WindowsCropUsesOcrPixelCoordinates",
|
||||
"MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.WindowsCropPreservesExpectedPixelsAtNonzeroOrigin",
|
||||
"MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.WindowsCropRejectsInvalidBounds"
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void RejectsEmptyExecutionRecordsInsteadOfAcceptingSummaryCounts()
|
||||
{
|
||||
var trx = XDocument.Parse("""
|
||||
<TestRun xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
|
||||
<Results />
|
||||
<ResultSummary outcome="Completed"><Counters total="602" passed="602" failed="0" notExecuted="0" /></ResultSummary>
|
||||
</TestRun>
|
||||
""");
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() =>
|
||||
PlatformTestVerification.Verify("portable", "MeetingAssistant.Tests.Example.Passes", trx));
|
||||
|
||||
Assert.Contains("execution records", failure.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DerivesCurrentTotalsFromExecutionRecords()
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
|
||||
var result = PlatformTestVerification.Verify("portable", discovery, trx);
|
||||
|
||||
Assert.Equal(3, result.Total);
|
||||
Assert.Equal(1, result.Passed);
|
||||
Assert.Equal(2, result.Skipped);
|
||||
Assert.Contains("Totals: 3 discovered, 3 results, 1 passed, 2 skipped, 0 failed.", result.Lines);
|
||||
Assert.Contains("Windows crop cases: excluded from portable target (3 Windows-only native cases).", result.Lines);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Passed", "")]
|
||||
[InlineData("Passed", "platform-assurance: target=windows; host=linux")]
|
||||
[InlineData("Passed", "platform-assurance: target=portable; host=unknown")]
|
||||
[InlineData("NotExecuted", "platform-assurance: target=portable; host=linux")]
|
||||
[InlineData("Passed", "platform-assurance: target=portable; host=linux\nplatform-assurance: target=portable; host=windows")]
|
||||
public void RequiresMatchingPassedCompositionGuard(string outcome, string marker)
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
var guard = trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard);
|
||||
guard.SetAttributeValue("outcome", outcome);
|
||||
guard.Descendants(Trx + "StdOut").Single().Value = marker;
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
|
||||
Assert.Contains("composition guard", failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(UserEnvironment, "linux", "Passed", "")]
|
||||
[InlineData(Glyphs, "macos", "Passed", "")]
|
||||
[InlineData(UserEnvironment, "linux", "NotExecuted", "")]
|
||||
[InlineData(Glyphs, "macos", "NotExecuted", "No explanation")]
|
||||
[InlineData(UserEnvironment, "windows", "NotExecuted", "Requires Windows per-user environment variables.")]
|
||||
[InlineData(Glyphs, "windows", "NotExecuted", "Requires Windows rendering.")]
|
||||
public void RequiresCompatibleExecutionOrReasonedWindowsSkip(string name, string host, string outcome, string reason)
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
var guard = trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard);
|
||||
guard.Descendants(Trx + "StdOut").Single().Value = $"platform-assurance: target=portable; host={host}";
|
||||
if (host == "windows")
|
||||
{
|
||||
foreach (var runtimeTest in trx.Descendants(Trx + "UnitTestResult").Where(record => (string?)record.Attribute("testName") != Guard))
|
||||
{
|
||||
runtimeTest.SetAttributeValue("outcome", "Passed");
|
||||
}
|
||||
}
|
||||
var record = trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == name);
|
||||
record.SetAttributeValue("outcome", outcome);
|
||||
record.Descendants(Trx + "StdOut").Single().Value = reason;
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
|
||||
Assert.Contains(name, failure.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReportsWindowsSkipNamesAndReasonsFromTrxErrorInfo()
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
foreach (var record in trx.Descendants(Trx + "UnitTestResult").Where(record => (string?)record.Attribute("outcome") == "NotExecuted"))
|
||||
{
|
||||
var output = record.Element(Trx + "Output")!;
|
||||
var reason = output.Element(Trx + "StdOut")!.Value;
|
||||
output.ReplaceNodes(new XElement(Trx + "ErrorInfo", new XElement(Trx + "Message", reason)));
|
||||
}
|
||||
|
||||
var result = PlatformTestVerification.Verify("portable", discovery, trx);
|
||||
|
||||
Assert.Contains($"Skipped: {UserEnvironment} — Requires Windows per-user environment variables; run on Windows.", result.Lines);
|
||||
Assert.Contains($"Skipped: {Glyphs} — Requires Windows System.Drawing bitmap rendering; run on Windows.", result.Lines);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WindowsEvidenceNamesAllPassedOutlookAndNativeCropCases()
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
|
||||
var result = PlatformTestVerification.Verify("windows", discovery, trx);
|
||||
|
||||
Assert.Equal(11, result.Total);
|
||||
Assert.Equal(11, result.Passed);
|
||||
Assert.Equal(0, result.Skipped);
|
||||
foreach (var name in OutlookCases)
|
||||
{
|
||||
Assert.Contains($"Passed: {name}", result.Lines);
|
||||
}
|
||||
foreach (var name in WindowsCropCases)
|
||||
{
|
||||
Assert.Contains($"Passed: {name}", result.Lines);
|
||||
}
|
||||
Assert.Contains("Test artifacts only; full Windows resource/PRI builds and native operational acceptance require separate evidence.", result.Lines);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, "both")]
|
||||
[InlineData(1, "both")]
|
||||
[InlineData(2, "both")]
|
||||
[InlineData(0, "discovery")]
|
||||
[InlineData(1, "discovery")]
|
||||
[InlineData(2, "discovery")]
|
||||
[InlineData(0, "execution")]
|
||||
[InlineData(1, "execution")]
|
||||
[InlineData(2, "execution")]
|
||||
public void RequiresDiscoveryAndExecutionOfEachNativeWindowsCropCase(int missingCase, string missingArtifact)
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
var name = WindowsCropCases[missingCase];
|
||||
if (missingArtifact is "discovery" or "both")
|
||||
{
|
||||
discovery = discovery.Replace(name, "", StringComparison.Ordinal);
|
||||
}
|
||||
if (missingArtifact is "execution" or "both")
|
||||
{
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == name).Remove();
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("total", 10);
|
||||
counters.SetAttributeValue("executed", 10);
|
||||
counters.SetAttributeValue("passed", 10);
|
||||
}
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("windows", discovery, trx));
|
||||
|
||||
Assert.Contains(name, failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
public void ReasonedWindowsCropSkipsDoNotCountAsNativeExecutionEvidence(int skippedCase)
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
var name = WindowsCropCases[skippedCase];
|
||||
var record = trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == name);
|
||||
record.SetAttributeValue("outcome", "NotExecuted");
|
||||
record.Descendants(Trx + "StdOut").Single().Value = "Requires Windows System.Drawing bitmap rendering.";
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("executed", 10);
|
||||
counters.SetAttributeValue("passed", 10);
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("windows", discovery, trx));
|
||||
|
||||
Assert.Contains(name, failure.Message);
|
||||
Assert.Contains("Passed", failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
[InlineData(3)]
|
||||
[InlineData(4)]
|
||||
public void RequiresEachExistingOutlookCase(int missingCase)
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
var name = OutlookCases[missingCase];
|
||||
discovery = discovery.Replace(name, "", StringComparison.Ordinal);
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == name).Remove();
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("windows", discovery, trx));
|
||||
|
||||
Assert.Contains(name, failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("NotExecuted")]
|
||||
[InlineData("Failed")]
|
||||
public void RequiresExecutedPassedOutlookAssertions(string outcome)
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == OutlookCases[0])
|
||||
.SetAttributeValue("outcome", outcome);
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("windows", discovery, trx));
|
||||
|
||||
Assert.Contains(OutlookCases[0], failure.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PortableEvidenceRejectsOutlookCasesEvenWithWindowsRuntime()
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard)
|
||||
.Descendants(Trx + "StdOut").Single().Value = "platform-assurance: target=portable; host=windows";
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
|
||||
Assert.Contains("Outlook", failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("linux")]
|
||||
[InlineData("windows")]
|
||||
public void PortableEvidenceRejectsWindowsCropClassRegardlessOfRuntimeHost(string host)
|
||||
{
|
||||
const string cropCase = "MeetingAssistant.Tests.WindowsScreenshotImageCropperTests.AdditionalNativeCropCase";
|
||||
var (discovery, trx) = PortableFixture();
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard)
|
||||
.Descendants(Trx + "StdOut").Single().Value = $"platform-assurance: target=portable; host={host}";
|
||||
if (host == "windows")
|
||||
{
|
||||
foreach (var record in trx.Descendants(Trx + "UnitTestResult"))
|
||||
{
|
||||
record.SetAttributeValue("outcome", "Passed");
|
||||
}
|
||||
}
|
||||
discovery += '\n' + cropCase;
|
||||
trx.Root!.Element(Trx + "Results")!.Add(Result(cropCase, "Passed", ""));
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("total", 4);
|
||||
counters.SetAttributeValue("executed", host == "windows" ? 4 : 2);
|
||||
counters.SetAttributeValue("passed", host == "windows" ? 4 : 2);
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
|
||||
Assert.Contains("Windows crop", failure.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("duplicate discovery")]
|
||||
[InlineData("duplicate execution")]
|
||||
[InlineData("missing execution")]
|
||||
[InlineData("unlisted execution")]
|
||||
[InlineData("failed execution")]
|
||||
[InlineData("unrecognized outcome")]
|
||||
[InlineData("empty discovery")]
|
||||
public void RejectsIncompleteDuplicateOrFailedArtifactSets(string corruption)
|
||||
{
|
||||
const string example = "MeetingAssistant.Tests.Example.Passes";
|
||||
var (discovery, trx) = PortableFixture();
|
||||
var record = Result(example, "Passed", "");
|
||||
trx.Root!.Element(Trx + "Results")!.Add(record);
|
||||
discovery += '\n' + example;
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("total", 4);
|
||||
counters.SetAttributeValue("executed", 2);
|
||||
counters.SetAttributeValue("passed", 2);
|
||||
|
||||
switch (corruption)
|
||||
{
|
||||
case "duplicate discovery": discovery += '\n' + example; break;
|
||||
case "duplicate execution": record.AddAfterSelf(new XElement(record)); break;
|
||||
case "missing execution": record.Remove(); break;
|
||||
case "unlisted execution": discovery = discovery.Replace(example, "", StringComparison.Ordinal); break;
|
||||
case "failed execution": record.SetAttributeValue("outcome", "Failed"); break;
|
||||
case "unrecognized outcome": record.SetAttributeValue("outcome", "Aborted"); break;
|
||||
case "empty discovery": discovery = ""; break;
|
||||
}
|
||||
|
||||
Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("total", "602")]
|
||||
[InlineData("executed", "602")]
|
||||
[InlineData("passed", "602")]
|
||||
[InlineData("failed", "1")]
|
||||
[InlineData("total", null)]
|
||||
[InlineData("total", "invalid")]
|
||||
[InlineData("aborted", "1")]
|
||||
public void RejectsInconsistentSummaryCounters(string counter, string? value)
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
trx.Descendants(Trx + "Counters").Single().SetAttributeValue(counter, value);
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("portable", discovery, trx));
|
||||
|
||||
Assert.Contains("summary", failure.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WindowsTargetRequiresWindowsRuntimeEvidence()
|
||||
{
|
||||
var (discovery, trx) = WindowsFixture();
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard)
|
||||
.Descendants(Trx + "StdOut").Single().Value = "platform-assurance: target=windows; host=linux";
|
||||
foreach (var record in trx.Descendants(Trx + "UnitTestResult")
|
||||
.Where(record => (string?)record.Attribute("testName") is UserEnvironment or Glyphs))
|
||||
{
|
||||
record.SetAttributeValue("outcome", "NotExecuted");
|
||||
record.Descendants(Trx + "StdOut").Single().Value = "Requires Windows runtime.";
|
||||
}
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("executed", 9);
|
||||
counters.SetAttributeValue("passed", 9);
|
||||
|
||||
var failure = Assert.Throws<InvalidDataException>(() => PlatformTestVerification.Verify("windows", discovery, trx));
|
||||
|
||||
Assert.Contains("Windows runtime", failure.Message);
|
||||
}
|
||||
|
||||
private static (string Discovery, XDocument Results) WindowsFixture()
|
||||
{
|
||||
var (discovery, trx) = PortableFixture();
|
||||
foreach (var record in trx.Descendants(Trx + "UnitTestResult"))
|
||||
{
|
||||
record.SetAttributeValue("outcome", "Passed");
|
||||
record.Descendants(Trx + "StdOut").Single().Value = "";
|
||||
}
|
||||
trx.Descendants(Trx + "UnitTestResult").Single(record => (string?)record.Attribute("testName") == Guard)
|
||||
.Descendants(Trx + "StdOut").Single().Value = "platform-assurance: target=windows; host=windows";
|
||||
trx.Root!.Element(Trx + "Results")!.Add(OutlookCases.Select(name => Result(name, "Passed", "")));
|
||||
trx.Root!.Element(Trx + "Results")!.Add(WindowsCropCases.Select(name => Result(name, "Passed", "")));
|
||||
var counters = trx.Descendants(Trx + "Counters").Single();
|
||||
counters.SetAttributeValue("total", 11);
|
||||
counters.SetAttributeValue("executed", 11);
|
||||
counters.SetAttributeValue("passed", 11);
|
||||
return (discovery + '\n' + string.Join('\n', OutlookCases) + '\n' + string.Join('\n', WindowsCropCases), trx);
|
||||
}
|
||||
|
||||
private static (string Discovery, XDocument Results) PortableFixture()
|
||||
{
|
||||
var results = new[]
|
||||
{
|
||||
Result(Guard, "Passed", "platform-assurance: target=portable; host=linux"),
|
||||
Result(UserEnvironment, "NotExecuted", "Requires Windows per-user environment variables; run on Windows."),
|
||||
Result(Glyphs, "NotExecuted", "Requires Windows System.Drawing bitmap rendering; run on Windows.")
|
||||
};
|
||||
return ("The following Tests are available:\n" + string.Join('\n', new[] { Guard, UserEnvironment, Glyphs }),
|
||||
new XDocument(new XElement(Trx + "TestRun",
|
||||
new XElement(Trx + "Results", results),
|
||||
new XElement(Trx + "ResultSummary", new XAttribute("outcome", "Completed"),
|
||||
new XElement(Trx + "Counters", new XAttribute("total", 3), new XAttribute("executed", 1),
|
||||
new XAttribute("passed", 1), new XAttribute("failed", 0), new XAttribute("notExecuted", 0))))));
|
||||
}
|
||||
|
||||
private static XElement Result(string name, string outcome, string output) =>
|
||||
new(Trx + "UnitTestResult", new XAttribute("testName", name), new XAttribute("outcome", outcome),
|
||||
new XElement(Trx + "Output", new XElement(Trx + "StdOut", output)));
|
||||
}
|
||||
Reference in new issue
Block a user