feat(macos): enable meeting integrations

2026-08-06 08:19:26 +02:00
parent 9542c05ea4
commit 2af7933a1b
27 changed files with 1999 additions and 227 deletions
@@ -0,0 +1,436 @@
+using System.Net.Http.Json;
+using MeetingAssistant.Calendar;
+using MeetingAssistant.MacOs;
+using MeetingAssistant.MeetingNotes;
+using MeetingAssistant.Screenshots;
+using MeetingAssistant.Transcription;
+using MeetingAssistant.Workflow;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace MeetingAssistant.Tests;
+
+public sealed class MacOsMeetingIntegrationTests
+{
+    [Fact]
+    public void PortableRegistrationUsesRealMacOsProvidersOnlyOnMacOs()
+    {
+        var macServices = new ServiceCollection().AddLogging();
+        var unsupportedServices = new ServiceCollection().AddLogging();
+
+        MacOsMeetingIntegrationRegistration.Add(macServices, isMacOs: true);
+        MacOsMeetingIntegrationRegistration.Add(unsupportedServices, isMacOs: false);
+
+        using var macProvider = macServices.BuildServiceProvider();
+        using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
+
+        Assert.IsType<MacOsMeetingMetadataProvider>(
+            macProvider.GetRequiredService<IMeetingMetadataProvider>());
+        Assert.IsType<MacOsCalendarMeetingProvider>(
+            macProvider.GetRequiredService<ICalendarMeetingProvider>());
+        Assert.IsType<MacOsMeetingStartPromptService>(
+            macProvider.GetRequiredService<IMeetingStartPromptService>());
+        Assert.IsType<MacOsDesktopIntegrationClient>(
+            macProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
+        Assert.IsType<MacOsDesktopIntegrationClient>(
+            macProvider.GetRequiredService<IScreenshotImageCropper>());
+
+        Assert.IsType<NoopMeetingMetadataProvider>(
+            unsupportedProvider.GetRequiredService<IMeetingMetadataProvider>());
+        Assert.IsType<NoopCalendarMeetingProvider>(
+            unsupportedProvider.GetRequiredService<ICalendarMeetingProvider>());
+        Assert.IsType<NoopMeetingStartPromptService>(
+            unsupportedProvider.GetRequiredService<IMeetingStartPromptService>());
+        Assert.IsType<UnavailableActiveWindowScreenshotCapture>(
+            unsupportedProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
+        Assert.IsType<UnavailableScreenshotImageCropper>(
+            unsupportedProvider.GetRequiredService<IScreenshotImageCropper>());
+    }
+
+    [Fact]
+    public async Task MacOsCalendarProvidersReturnTeamsMetadataAndPromptCandidate()
+    {
+        var startedAt = new DateTimeOffset(2026, 7, 25, 10, 5, 0, TimeSpan.FromHours(2));
+        var calendarEvent = new MacOsCalendarEvent(
+            "event-1",
+            "Project sync",
+            startedAt.AddMinutes(-5),
+            startedAt.AddMinutes(55),
+            "Microsoft Teams",
+            "Discuss launch readiness.\nJoin Microsoft Teams Meeting",
+            "https://teams.microsoft.com/l/meetup-join/example",
+            ["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
+            IsCanceled: false);
+        var client = new StubMacOsDesktopIntegrationClient(calendarEvent);
+        IMeetingMetadataProvider metadataProvider = new MacOsMeetingMetadataProvider(
+            client,
+            NullLogger<MacOsMeetingMetadataProvider>.Instance);
+        ICalendarMeetingProvider calendarProvider = new MacOsCalendarMeetingProvider(
+            client,
+            NullLogger<MacOsCalendarMeetingProvider>.Instance);
+
+        var metadata = await metadataProvider.GetCurrentMeetingAsync(
+            startedAt,
+            CancellationToken.None);
+        var candidates = await calendarProvider.GetRecordingPromptCandidatesForDayAsync(
+            DateOnly.FromDateTime(startedAt.Date),
+            CancellationToken.None);
+
+        Assert.NotNull(metadata);
+        Assert.Equal("Project sync", metadata.Title);
+        Assert.Equal(
+            ["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
+            metadata.Attendees);
+        Assert.Equal("Discuss launch readiness.", metadata.Agenda);
+        Assert.Equal(calendarEvent.End, metadata.ScheduledEnd);
+
+        var candidate = Assert.Single(candidates);
+        Assert.Equal("event-1", candidate.Id);
+        Assert.Equal("Project sync", candidate.Subject);
+        Assert.NotNull(candidate.Metadata);
+        Assert.Equal(metadata.Title, candidate.Metadata.Title);
+        Assert.Equal(metadata.Attendees, candidate.Metadata.Attendees);
+        Assert.Equal(metadata.Agenda, candidate.Metadata.Agenda);
+        Assert.Equal(metadata.ScheduledEnd, candidate.Metadata.ScheduledEnd);
+    }
+
+    [Fact]
+    public async Task MacOsPromptAndScreenshotUseNativeClientResults()
+    {
+        var screenshot = new ActiveWindowScreenshot([137, 80, 78, 71], "Meeting deck");
+        var client = new StubMacOsDesktopIntegrationClient(
+            screenshot: screenshot,
+            promptResponse: MeetingStartPromptResponse.Record);
+        IMeetingStartPromptService prompt = new MacOsMeetingStartPromptService(
+            client,
+            NullLogger<MacOsMeetingStartPromptService>.Instance);
+        IActiveWindowScreenshotCapture capture = client;
+        IScreenshotImageCropper cropper = client;
+        MeetingStartPromptResponse? callbackResponse = null;
+        var request = new MeetingStartPromptRequest(new CalendarMeeting(
+            "event-1",
+            "Project sync",
+            DateTimeOffset.Now,
+            DateTimeOffset.Now.AddHours(1)));
+
+        await prompt.ShowPromptAsync(
+            request,
+            (response, _) =>
+            {
+                callbackResponse = response;
+                return Task.CompletedTask;
+            },
+            CancellationToken.None);
+        var captured = await capture.CaptureAsync(CancellationToken.None);
+        var croppedPath = await cropper.SaveCroppedScreenshotAsync(
+            "meeting.png",
+            new ScreenshotCropCoordinates(1, 2, 3, 4),
+            CancellationToken.None);
+
+        Assert.Equal(MeetingStartPromptResponse.Record, callbackResponse);
+        Assert.Equal(screenshot, captured);
+        Assert.Equal("meeting.png.cropped.png", croppedPath);
+    }
+
+    [Fact]
+    public async Task MacOsCapabilityEndpointReportsEnabledRealProviders()
+    {
+        if (!OperatingSystem.IsMacOS())
+        {
+            return;
+        }
+
+        await using var factory = new WebApplicationFactory<Program>();
+        using var client = factory.CreateClient();
+
+        var report = await client.GetFromJsonAsync<PlatformCapabilityReport>(
+            "/diagnostics/platform-capabilities");
+
+        Assert.NotNull(report);
+        Assert.Equal("macOS", report.Host);
+        Assert.Equal(nameof(MacOsMeetingMetadataProvider), report.MeetingMetadataProvider);
+        Assert.Equal(nameof(MacOsCalendarMeetingProvider), report.CalendarMeetingProvider);
+        Assert.Equal(nameof(MacOsMeetingStartPromptService), report.MeetingStartPromptProvider);
+        Assert.Equal(nameof(MacOsDesktopIntegrationClient), report.ScreenshotProvider);
+        Assert.Equal(nameof(MacOsWorkflowRulesEditorWindowService), report.WorkflowEditorProvider);
+        Assert.True(report.WorkflowEditorAvailable);
+        Assert.True(report.CalendarRecordingPromptsEnabled);
+        Assert.True(report.SpeakerIdentificationEnabled);
+        Assert.True(report.ScreenshotOcrEnabled);
+        Assert.True(report.FunAsrBackendEnabled);
+        Assert.True(report.FunAsrDiarizationEnabled);
+        Assert.True(report.WhisperLocalDiarizationEnabled);
+    }
+
+    [Fact]
+    public async Task NativeHelperAdvertisesCalendarPromptAndScreenshotFeatures()
+    {
+        if (!OperatingSystem.IsMacOS())
+        {
+            return;
+        }
+
+        var runner = new ProcessCommandRunner(
+            NullLogger<ProcessCommandRunner>.Instance);
+
+        var result = await runner.RunAsync(
+            GetNativeHelperPath(),
+            ["self-test"],
+            CancellationToken.None);
+
+        Assert.Equal(0, result.ExitCode);
+        using var json = System.Text.Json.JsonDocument.Parse(result.StandardOutput);
+        Assert.Equal("ok", json.RootElement.GetProperty("status").GetString());
+        Assert.Equal(
+            ["calendar", "prompt", "screenshot", "crop"],
+            json.RootElement.GetProperty("features")
+                .EnumerateArray()
+                .Select(feature => feature.GetString()!)
+                .ToArray());
+    }
+
+    [Fact]
+    public async Task NativeHelperCropsPngUsingOcrPixelCoordinates()
+    {
+        if (!OperatingSystem.IsMacOS())
+        {
+            return;
+        }
+
+        var testRoot = Path.Combine(
+            Path.GetTempPath(),
+            $"meeting-assistant-native-crop-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(testRoot);
+        var bitmapPath = Path.Combine(testRoot, "input.bmp");
+        var inputPath = Path.Combine(testRoot, "input.png");
+        var outputPath = Path.Combine(testRoot, "output.png");
+        var outputBitmapPath = Path.Combine(testRoot, "output.bmp");
+        try
+        {
+            await File.WriteAllBytesAsync(
+                bitmapPath,
+                CreateTopRedBottomBlueBitmap());
+            var runner = new ProcessCommandRunner(
+                NullLogger<ProcessCommandRunner>.Instance);
+            var inputConversion = await runner.RunAsync(
+                "/usr/bin/sips",
+                ["-s", "format", "png", bitmapPath, "--out", inputPath],
+                CancellationToken.None);
+            Assert.Equal(0, inputConversion.ExitCode);
+
+            var result = await runner.RunAsync(
+                GetNativeHelperPath(),
+                ["crop", inputPath, outputPath, "0", "0", "2", "1"],
+                CancellationToken.None);
+
+            Assert.Equal(0, result.ExitCode);
+            var png = await File.ReadAllBytesAsync(outputPath);
+            Assert.Equal((2, 1), PngTestFixture.ReadDimensions(png));
+            var outputConversion = await runner.RunAsync(
+                "/usr/bin/sips",
+                ["-s", "format", "bmp", outputPath, "--out", outputBitmapPath],
+                CancellationToken.None);
+            Assert.Equal(0, outputConversion.ExitCode);
+            Assert.Equal(
+                (Red: (byte)255, Green: (byte)0, Blue: (byte)0),
+                ReadFirstBitmapPixel(await File.ReadAllBytesAsync(outputBitmapPath)));
+        }
+        finally
+        {
+            Directory.Delete(testRoot, recursive: true);
+        }
+    }
+
+    [Fact]
+    public async Task CalendarProviderUsesActualLocalMidnightsAcrossDaylightSavingTransition()
+    {
+        if (TimeZoneInfo.Local.Id != "Europe/Berlin")
+        {
+            return;
+        }
+
+        var client = new StubMacOsDesktopIntegrationClient();
+        var provider = new MacOsCalendarMeetingProvider(
+            client,
+            NullLogger<MacOsCalendarMeetingProvider>.Instance);
+
+        await provider.GetRecordingPromptCandidatesForDayAsync(
+            new DateOnly(2026, 3, 29),
+            CancellationToken.None);
+
+        Assert.Equal(
+            DateTimeOffset.Parse("2026-03-29T00:00:00+01:00"),
+            client.CalendarFrom);
+        Assert.Equal(
+            DateTimeOffset.Parse("2026-03-30T00:00:00+02:00"),
+            client.CalendarTo);
+        Assert.Equal(TimeSpan.FromHours(23), client.CalendarTo - client.CalendarFrom);
+    }
+
+    [Fact]
+    public async Task CalendarClientFallsBackToCalendarAutomationWhenEventKitIsDenied()
+    {
+        if (!OperatingSystem.IsMacOS())
+        {
+            return;
+        }
+
+        var runner = new SequenceCommandRunner(
+            new CommandResult(
+                MacOsIntegrationException.CalendarPermissionDeniedExitCode,
+                "",
+                "Calendar Full Access permission is required."),
+            new CommandResult(
+                0,
+                """
+                [{"id":"fallback-event","subject":"Project sync","start":"2026-07-25T08:00:00Z","end":"2026-07-25T09:00:00Z","location":"Microsoft Teams","notes":"","url":"https://teams.microsoft.com/example","attendees":[],"isCanceled":false}]
+                """,
+                ""));
+        IMacOsCalendarClient client = new MacOsDesktopIntegrationClient(
+            runner,
+            Path.GetDirectoryName(GetNativeHelperPath())!);
+
+        var events = await client.GetCalendarEventsAsync(
+            DateTimeOffset.Parse("2026-07-25T00:00:00Z"),
+            DateTimeOffset.Parse("2026-07-26T00:00:00Z"),
+            CancellationToken.None);
+
+        var calendarEvent = Assert.Single(events);
+        Assert.Equal("fallback-event", calendarEvent.Id);
+        Assert.Collection(
+            runner.Commands,
+            command => Assert.Equal("calendar", command.Arguments[0]),
+            command =>
+            {
+                Assert.Equal("/usr/bin/osascript", command.FileName);
+                Assert.Equal(["-l", "JavaScript"], command.Arguments.Take(2));
+            });
+    }
+
+    private static string GetNativeHelperPath()
+    {
+        var configuration = new DirectoryInfo(AppContext.BaseDirectory)
+            .Parent?.Name ?? "Debug";
+        return Path.GetFullPath(Path.Combine(
+            AppContext.BaseDirectory,
+            "..",
+            "..",
+            "..",
+            "..",
+            "MeetingAssistant",
+            "bin",
+            configuration,
+            "net10.0",
+            "Native",
+            "macos-meeting-integrations"));
+    }
+
+    private static byte[] CreateTopRedBottomBlueBitmap()
+    {
+        var bitmap = new byte[70];
+        bitmap[0] = (byte)'B';
+        bitmap[1] = (byte)'M';
+        BitConverter.GetBytes(bitmap.Length).CopyTo(bitmap, 2);
+        BitConverter.GetBytes(54).CopyTo(bitmap, 10);
+        BitConverter.GetBytes(40).CopyTo(bitmap, 14);
+        BitConverter.GetBytes(2).CopyTo(bitmap, 18);
+        BitConverter.GetBytes(2).CopyTo(bitmap, 22);
+        BitConverter.GetBytes((short)1).CopyTo(bitmap, 26);
+        BitConverter.GetBytes((short)24).CopyTo(bitmap, 28);
+        BitConverter.GetBytes(16).CopyTo(bitmap, 34);
+
+        // BMP rows are bottom-up and padded to four-byte boundaries.
+        bitmap[54] = 255;
+        bitmap[57] = 255;
+        bitmap[64] = 255;
+        bitmap[67] = 255;
+        return bitmap;
+    }
+
+    private static (byte Red, byte Green, byte Blue) ReadFirstBitmapPixel(byte[] bitmap)
+    {
+        var pixelOffset = BitConverter.ToInt32(bitmap, 10);
+        return (
+            Red: bitmap[pixelOffset + 2],
+            Green: bitmap[pixelOffset + 1],
+            Blue: bitmap[pixelOffset]);
+    }
+
+    private sealed class StubMacOsDesktopIntegrationClient :
+        IMacOsCalendarClient,
+        IMacOsMeetingPromptClient,
+        IActiveWindowScreenshotCapture,
+        IScreenshotImageCropper
+    {
+        private readonly IReadOnlyList<MacOsCalendarEvent> events;
+        private readonly ActiveWindowScreenshot screenshot;
+        private readonly MeetingStartPromptResponse promptResponse;
+
+        public StubMacOsDesktopIntegrationClient(
+            MacOsCalendarEvent? calendarEvent = null,
+            ActiveWindowScreenshot? screenshot = null,
+            MeetingStartPromptResponse promptResponse = MeetingStartPromptResponse.Skip)
+        {
+            events = calendarEvent is null ? [] : [calendarEvent];
+            this.screenshot = screenshot ?? new ActiveWindowScreenshot([], null);
+            this.promptResponse = promptResponse;
+        }
+
+        public Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
+            DateTimeOffset from,
+            DateTimeOffset to,
+            CancellationToken cancellationToken)
+        {
+            CalendarFrom = from;
+            CalendarTo = to;
+            return Task.FromResult(events);
+        }
+
+        public DateTimeOffset CalendarFrom { get; private set; }
+
+        public DateTimeOffset CalendarTo { get; private set; }
+
+        public Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
+            MeetingStartPromptRequest request,
+            CancellationToken cancellationToken)
+        {
+            return Task.FromResult(promptResponse);
+        }
+
+        public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
+        {
+            return Task.FromResult(screenshot);
+        }
+
+        public Task<string?> SaveCroppedScreenshotAsync(
+            string screenshotPath,
+            ScreenshotCropCoordinates crop,
+            CancellationToken cancellationToken)
+        {
+            return Task.FromResult<string?>($"{screenshotPath}.cropped.png");
+        }
+    }
+
+    private sealed class SequenceCommandRunner : ICommandRunner
+    {
+        private readonly Queue<CommandResult> results;
+
+        public SequenceCommandRunner(params CommandResult[] results)
+        {
+            this.results = new Queue<CommandResult>(results);
+        }
+
+        public List<(string FileName, IReadOnlyList<string> Arguments)> Commands { get; } = [];
+
+        public Task<CommandResult> RunAsync(
+            string fileName,
+            IReadOnlyList<string> arguments,
+            CancellationToken cancellationToken,
+            IReadOnlyDictionary<string, string>? environment = null)
+        {
+            Commands.Add((fileName, arguments));
+            return Task.FromResult(results.Dequeue());
+        }
+    }
+}