diff --git a/.gitea/workflows/pr-push-build-and-test.yaml b/.gitea/workflows/pr-push-build-and-test.yaml index 83bc719..8afa519 100644 --- a/.gitea/workflows/pr-push-build-and-test.yaml +++ b/.gitea/workflows/pr-push-build-and-test.yaml @@ -90,3 +90,61 @@ jobs: run: | find MeetingAssistant.Tests -type d -name TestResults -print || true find MeetingAssistant.Tests -type f -path "*/TestResults/*" -maxdepth 5 -print || true + + macos-build-and-test: + runs-on: macos + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" + DOTNET_NOLOGO: "1" + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Verify native macOS runner + run: | + if [ "$(uname -s)" != "Darwin" ]; then + echo "The macos job requires a native Gitea runner registered with the macos:host label." + exit 1 + fi + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "10.0.x" + + - name: Restore (portable target) + run: | + dotnet restore MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \ + -p:EnableWindowsTargeting=true + + - name: Build (portable target and macOS helpers) + run: | + dotnet build MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \ + -c Release \ + -f net10.0 \ + --no-restore \ + -p:EnableWindowsTargeting=true \ + --nologo + + - name: Verify macOS native helpers + run: | + test -x MeetingAssistant/bin/Release/net10.0/Native/MeetingAssistantAudioCapture.app/Contents/MacOS/macos-meeting-audio-capture + test -x MeetingAssistant/bin/Release/net10.0/Native/macos-desktop-controls + test -x MeetingAssistant/bin/Release/net10.0/Native/macos-meeting-integrations + test -x MeetingAssistant/bin/Release/net10.0/Native/macos-meeting-assistant-launcher + + - name: Run tests on macOS + run: | + dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \ + -c Release \ + -f net10.0 \ + --no-build \ + -p:EnableWindowsTargeting=true \ + --nologo + + - name: Show test output folder on failure + if: failure() + run: | + find MeetingAssistant.Tests -type d -name TestResults -print || true + find MeetingAssistant.Tests -type f -path "*/TestResults/*" -maxdepth 5 -print || true diff --git a/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs index a280c8c..14efb99 100644 --- a/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs +++ b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs @@ -110,6 +110,11 @@ public sealed class AzureSpeechStreamingTranscriptionProviderTests [Fact] public void ResolveKeyFallsBackToUserEnvironment() { + if (!OperatingSystem.IsWindows()) + { + return; + } + var keyEnv = $"MEETING_ASSISTANT_AZURE_KEY_{Guid.NewGuid():N}"; try { diff --git a/MeetingAssistant.Tests/HealthEndpointTests.cs b/MeetingAssistant.Tests/HealthEndpointTests.cs index 911de71..718d86a 100644 --- a/MeetingAssistant.Tests/HealthEndpointTests.cs +++ b/MeetingAssistant.Tests/HealthEndpointTests.cs @@ -42,6 +42,19 @@ public sealed class HealthEndpointTests : IClassFixture(); + var second = factory.Services.GetRequiredService(); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Same(first, second); + Assert.Equal(HostOperatingSystem.DetectCurrent(), first); + } + [Fact] public async Task NeutralBuildReportsSpeakerPlaybackUnavailable() { diff --git a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs index b1d0b95..40b9aeb 100644 --- a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs +++ b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs @@ -2,8 +2,6 @@ using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; -using System.Drawing; -using System.Drawing.Imaging; using MeetingAssistant.Screenshots; using Microsoft.Extensions.Logging.Abstractions; @@ -284,15 +282,7 @@ public sealed class LiteLlmScreenshotOcrClientTests private static byte[] CreatePngBytes(int width, int height) { - using var bitmap = new Bitmap(width, height); - using (var graphics = Graphics.FromImage(bitmap)) - { - graphics.Clear(Color.White); - } - - using var stream = new MemoryStream(); - bitmap.Save(stream, ImageFormat.Png); - return stream.ToArray(); + return PngTestFixture.Solid(width, height); } private static string CreateNonStreamingTextResponse(string text) diff --git a/MeetingAssistant.Tests/MacOsDesktopControlManifestTests.cs b/MeetingAssistant.Tests/MacOsDesktopControlManifestTests.cs new file mode 100644 index 0000000..b339c41 --- /dev/null +++ b/MeetingAssistant.Tests/MacOsDesktopControlManifestTests.cs @@ -0,0 +1,89 @@ +using MeetingAssistant.LaunchProfiles; +using MeetingAssistant.MacOs; +using MeetingAssistant.Workflow; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; + +namespace MeetingAssistant.Tests; + +public sealed class MacOsDesktopControlManifestTests +{ + [Fact] + public void MacOsApplicationRegistersARealInteractiveAgentWindow() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + using var factory = new WebApplicationFactory(); + + var service = factory.Services.GetRequiredService(); + + Assert.IsType(service); + } + + [Fact] + public void ManifestPreservesConfiguredWindowsEquivalentShortcutsAndControlEndpoints() + { + var profiles = new[] + { + Profile("default", "Ctrl+Alt+M", "Ctrl+Alt+Z", "Ctrl+Alt+S"), + Profile("english", "Ctrl+Alt+L", "", "") + }; + + var hotkeys = new[] + { + new LaunchProfileHotkey("default", "Ctrl+Alt+M", LaunchProfileHotkeyAction.ToggleRecording), + new LaunchProfileHotkey("default", "Ctrl+Alt+Z", LaunchProfileHotkeyAction.AbortRecording), + new LaunchProfileHotkey("default", "Ctrl+Alt+S", LaunchProfileHotkeyAction.CaptureScreenshot), + new LaunchProfileHotkey("english", "Ctrl+Alt+L", LaunchProfileHotkeyAction.ToggleRecording) + }; + + var manifest = MacOsDesktopControlManifest.Create( + "http://127.0.0.1:5090", + profiles, + hotkeys); + + Assert.Collection( + manifest.Hotkeys, + hotkey => + { + Assert.Equal("Ctrl+Alt+M", hotkey.Shortcut); + Assert.Equal("/profiles/default/recording/toggle", hotkey.Endpoint); + }, + hotkey => + { + Assert.Equal("Ctrl+Alt+Z", hotkey.Shortcut); + Assert.Equal("/profiles/default/recording/abort", hotkey.Endpoint); + }, + hotkey => + { + Assert.Equal("Ctrl+Alt+S", hotkey.Shortcut); + Assert.Equal("/profiles/default/meetings/screenshot/capture", hotkey.Endpoint); + }, + hotkey => + { + Assert.Equal("Ctrl+Alt+L", hotkey.Shortcut); + Assert.Equal("/profiles/english/recording/toggle", hotkey.Endpoint); + }); + Assert.Equal("http://127.0.0.1:5090/recording/status", manifest.StatusUrl); + Assert.Equal("http://127.0.0.1:5090/diagnostics/settings-and-logs", manifest.OpenAgentUrl); + } + + private static LaunchProfile Profile(string name, string toggle, string abort, string screenshot) + { + return new LaunchProfile(name, new MeetingAssistantOptions + { + Hotkey = new HotkeyOptions + { + Toggle = toggle, + Abort = abort + }, + Screenshots = new ScreenshotOptions + { + Hotkey = screenshot + } + }); + } +} diff --git a/MeetingAssistant.Tests/MacOsMeetingAudioSourceTests.cs b/MeetingAssistant.Tests/MacOsMeetingAudioSourceTests.cs new file mode 100644 index 0000000..672b553 --- /dev/null +++ b/MeetingAssistant.Tests/MacOsMeetingAudioSourceTests.cs @@ -0,0 +1,504 @@ +using MeetingAssistant.Recording; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using MeetingAssistant.Transcription; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging.Abstractions; +using System.Net.Http.Json; + +namespace MeetingAssistant.Tests; + +public sealed class MacOsMeetingAudioSourceTests +{ + [Fact] + public void MacOsServiceLauncherDefinesStablePrivacyIdentity() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var projectDirectory = Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "..", + "MeetingAssistant")); + var launcherDirectory = Path.Combine(projectDirectory, "Native", "MacOsLauncher"); + var infoPlistPath = Path.Combine(launcherDirectory, "Info.plist"); + var launcherSourcePath = Path.Combine(launcherDirectory, "main.swift"); + + Assert.True(File.Exists(infoPlistPath), $"Expected launcher Info.plist at '{infoPlistPath}'."); + Assert.True(File.Exists(launcherSourcePath), $"Expected launcher source at '{launcherSourcePath}'."); + + var infoPlist = File.ReadAllText(infoPlistPath); + Assert.Contains("cloud.schweigert.meeting-assistant", infoPlist, StringComparison.Ordinal); + Assert.Contains("NSMicrophoneUsageDescription", infoPlist, StringComparison.Ordinal); + Assert.Contains("NSCalendarsFullAccessUsageDescription", infoPlist, StringComparison.Ordinal); + + var launcherSource = File.ReadAllText(launcherSourcePath); + Assert.Contains("set -a", launcherSource, StringComparison.Ordinal); + Assert.Contains("Contents/Resources/app", launcherSource, StringComparison.Ordinal); + + var projectFile = File.ReadAllText(Path.Combine(projectDirectory, "MeetingAssistant.csproj")); + Assert.Contains( + "designated => identifier "cloud.schweigert.meeting-assistant"", + projectFile, + StringComparison.Ordinal); + } + + [Fact] + public void NativeAudioCaptureIsPackagedAsSignedMacOsAppForPersistentPrivacyGrant() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var configuration = new DirectoryInfo(AppContext.BaseDirectory) + .Parent?.Name ?? "Debug"; + var appPath = Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "..", + "MeetingAssistant", + "bin", + configuration, + "net10.0", + "Native", + "MeetingAssistantAudioCapture.app")); + var infoPlistPath = Path.Combine(appPath, "Contents", "Info.plist"); + var helperPath = Path.Combine(appPath, "Contents", "MacOS", "macos-meeting-audio-capture"); + + Assert.True(Directory.Exists(appPath), $"Expected macOS audio capture app at '{appPath}'."); + Assert.True(File.Exists(infoPlistPath), $"Expected Info.plist at '{infoPlistPath}'."); + Assert.True(File.Exists(helperPath), $"Expected native helper at '{helperPath}'."); + + var infoPlist = File.ReadAllText(infoPlistPath); + Assert.Contains("cloud.schweigert.meeting-assistant.audio-capture", infoPlist, StringComparison.Ordinal); + Assert.Contains("NSMicrophoneUsageDescription", infoPlist, StringComparison.Ordinal); + + using var verification = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo + { + FileName = "/usr/bin/codesign", + ArgumentList = { "--verify", "--deep", "--strict", appPath }, + RedirectStandardError = true, + UseShellExecute = false + }); + Assert.NotNull(verification); + verification.WaitForExit(); + Assert.True( + verification.ExitCode == 0, + $"Expected a valid app-bundle signature: {verification.StandardError.ReadToEnd()}"); + } + + [Fact] + public async Task MacOsMicrophoneStreamsNativePcmUsingRunFormat() + { + var pcm = Pcm16(1_000, -2_000, 3_000, -4_000); + var process = new StubMacOsAudioCaptureProcess(pcm, exitCode: 0); + var processFactory = new StubMacOsAudioCaptureProcessFactory(process); + IMeetingAudioSource source = new MacOsMicrophoneAudioSource( + processFactory, + NullLogger.Instance); + var options = new MeetingAssistantOptions + { + Recording = + { + SampleRate = 24_000, + Channels = 2 + } + }; + + var chunks = await ReadChunksAsync(source, options); + + var chunk = Assert.Single(chunks); + Assert.Equal(pcm, chunk.Pcm); + Assert.Equal(24_000, chunk.SampleRate); + Assert.Equal(2, chunk.Channels); + Assert.Equal( + new MacOsAudioCaptureRequest(MacOsAudioCaptureKind.Microphone, 24_000, 2), + processFactory.Request); + Assert.True(process.Disposed); + } + + [Fact] + public async Task MacOsSystemAudioReportsNativePermissionFailure() + { + var process = new StubMacOsAudioCaptureProcess( + [], + exitCode: 77, + standardError: "Screen Recording/System Audio permission is required."); + IMeetingAudioSource source = new MacOsSystemAudioSource( + new StubMacOsAudioCaptureProcessFactory(process), + NullLogger.Instance); + + var exception = await Assert.ThrowsAsync(async () => + await ReadChunksAsync(source, new MeetingAssistantOptions())); + + Assert.Contains("Screen Recording/System Audio permission", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void PortableRegistrationUsesMacOsCompositeOnlyForMacOs() + { + var macServices = new ServiceCollection().AddLogging(); + var unsupportedServices = new ServiceCollection().AddLogging(); + + PortableMeetingAudioRegistration.Add(macServices, isMacOs: true); + PortableMeetingAudioRegistration.Add(unsupportedServices, isMacOs: false); + + using var macProvider = macServices.BuildServiceProvider(); + using var unsupportedProvider = unsupportedServices.BuildServiceProvider(); + Assert.IsType(macProvider.GetRequiredService()); + Assert.IsType(unsupportedProvider.GetRequiredService()); + } + + [Fact] + public async Task RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses() + { + var testRoot = Path.Combine(Path.GetTempPath(), $"meeting-assistant-macos-endpoint-{Guid.NewGuid():N}"); + var nativeProcesses = new EndpointMacOsAudioCaptureProcessFactory( + microphonePcm: Pcm16(2_000), + systemPcm: Pcm16(10_000)); + var speechPipelines = new CapturingSpeechRecognitionPipelineFactory(); + var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new Dictionary + { + ["MeetingAssistant:Vault:BaseFolder"] = testRoot, + ["MeetingAssistant:Vault:TranscriptsFolder"] = "Transcripts", + ["MeetingAssistant:Vault:MeetingNotesFolder"] = "Notes", + ["MeetingAssistant:Vault:AssistantContextFolder"] = "Context", + ["MeetingAssistant:Vault:SummariesFolder"] = "Summaries", + ["MeetingAssistant:Vault:ProjectsFolder"] = "Projects", + ["MeetingAssistant:Recording:TemporaryRecordingsFolder"] = Path.Combine(testRoot, "Recordings"), + ["MeetingAssistant:Recording:InactivitySafeguard:Enabled"] = "false", + ["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(testRoot, "speakers.db"), + ["MeetingAssistant:FunAsr:Backend:Enabled"] = "false" + }); + }); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + PortableMeetingAudioRegistration.Add(services, isMacOs: true); + services.RemoveAll(); + services.AddSingleton(nativeProcesses); + services.RemoveAll(); + services.AddSingleton(speechPipelines); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + }); + }); + + try + { + using var client = factory.CreateClient(); + + using var startResponse = await client.PostAsync("/recording/start", content: null); + var started = await startResponse.Content.ReadFromJsonAsync(); + var mixedChunk = await speechPipelines.FirstChunk.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(startResponse.IsSuccessStatusCode); + Assert.True(started?.IsRecording); + Assert.Equal(12_000, BitConverter.ToInt16(mixedChunk.Pcm)); + + using var stopResponse = await client.PostAsync("/recording/stop", content: null); + var stopped = await stopResponse.Content.ReadFromJsonAsync(); + + Assert.True(stopResponse.IsSuccessStatusCode); + Assert.False(stopped?.IsRecording); + Assert.All(nativeProcesses.Processes, process => Assert.True(process.Terminated)); + } + finally + { + await factory.DisposeAsync(); + using var connection = new SqliteConnection($"Data Source={Path.Combine(testRoot, "speakers.db")}"); + SqliteConnection.ClearPool(connection); + if (Directory.Exists(testRoot)) + { + Directory.Delete(testRoot, recursive: true); + } + } + } + + private static async Task> ReadChunksAsync( + IMeetingAudioSource source, + MeetingAssistantOptions options) + { + var chunks = new List(); + await foreach (var chunk in source.CaptureAsync(options, CancellationToken.None)) + { + chunks.Add(chunk); + } + + return chunks; + } + + private static byte[] Pcm16(params short[] samples) + { + var pcm = new byte[samples.Length * sizeof(short)]; + Buffer.BlockCopy(samples, 0, pcm, 0, pcm.Length); + return pcm; + } + + private sealed class StubMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory + { + private readonly IMacOsAudioCaptureProcess process; + + public StubMacOsAudioCaptureProcessFactory(IMacOsAudioCaptureProcess process) + { + this.process = process; + } + + public MacOsAudioCaptureRequest? Request { get; private set; } + + public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request) + { + Request = request; + return process; + } + } + + private sealed class StubMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess + { + private readonly MemoryStream standardOutput; + private readonly string standardError; + + public StubMacOsAudioCaptureProcess(byte[] pcm, int exitCode, string standardError = "") + { + standardOutput = new MemoryStream(pcm); + this.standardError = standardError; + ExitCode = exitCode; + } + + public Stream StandardOutput => standardOutput; + + public int ExitCode { get; } + + public bool HasExited => true; + + public bool Disposed { get; private set; } + + public Task ReadStandardErrorAsync(CancellationToken cancellationToken) + { + return Task.FromResult(standardError); + } + + public Task WaitForExitAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public void Terminate() + { + } + + public ValueTask DisposeAsync() + { + Disposed = true; + standardOutput.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class EndpointMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory + { + private readonly IReadOnlyDictionary pcmByKind; + + public EndpointMacOsAudioCaptureProcessFactory(byte[] microphonePcm, byte[] systemPcm) + { + pcmByKind = new Dictionary + { + [MacOsAudioCaptureKind.Microphone] = microphonePcm, + [MacOsAudioCaptureKind.System] = systemPcm + }; + } + + public List Processes { get; } = []; + + public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request) + { + var process = new BlockingMacOsAudioCaptureProcess(pcmByKind[request.Kind]); + Processes.Add(process); + return process; + } + } + + private sealed class BlockingMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess + { + private readonly TaskCompletionSource terminated = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public BlockingMacOsAudioCaptureProcess(byte[] pcm) + { + StandardOutput = new InitialPcmThenBlockingStream(pcm, terminated.Task); + } + + public Stream StandardOutput { get; } + + public int ExitCode => Terminated ? 137 : throw new InvalidOperationException("Capture is still running."); + + public bool HasExited => Terminated; + + public bool Terminated => terminated.Task.IsCompleted; + + public Task ReadStandardErrorAsync(CancellationToken cancellationToken) + { + return Task.FromResult(""); + } + + public Task WaitForExitAsync(CancellationToken cancellationToken) + { + return terminated.Task.WaitAsync(cancellationToken); + } + + public void Terminate() + { + terminated.TrySetResult(); + } + + public ValueTask DisposeAsync() + { + StandardOutput.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class InitialPcmThenBlockingStream : Stream + { + private readonly byte[] pcm; + private readonly Task processExit; + private bool read; + + public InitialPcmThenBlockingStream(byte[] pcm, Task processExit) + { + this.pcm = pcm; + this.processExit = processExit; + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + if (!read) + { + read = true; + pcm.AsSpan().CopyTo(buffer.Span); + return pcm.Length; + } + + await processExit.WaitAsync(cancellationToken); + return 0; + } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Flush() + { + } + } + + private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory + { + public TaskCompletionSource FirstChunk { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ISpeechRecognitionPipeline Create() + { + return new CapturingSpeechRecognitionPipeline(FirstChunk); + } + } + + private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline + { + private readonly TaskCompletionSource firstChunk; + private readonly TaskCompletionSource completed = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CapturingSpeechRecognitionPipeline(TaskCompletionSource firstChunk) + { + this.firstChunk = firstChunk; + } + + public Task InitializeAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task InitializeAsync( + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) => Task.CompletedTask; + + public Task WaitUntilReadyAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken) + { + firstChunk.TrySetResult(chunk); + return ValueTask.CompletedTask; + } + + public Task CompleteAsync(CancellationToken cancellationToken) + { + completed.TrySetResult(); + return Task.CompletedTask; + } + + public async IAsyncEnumerable ReadLiveTranscriptAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await completed.Task.WaitAsync(cancellationToken); + yield break; + } + + public Task> ReadFinishedTranscriptAsync( + string audioPath, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult>([]); + } + + public ValueTask DisposeAsync() + { + completed.TrySetResult(); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopMeetingNoteOpener : IMeetingNoteOpener + { + public Task OpenAsync(string notePath, CancellationToken cancellationToken) => Task.CompletedTask; + } + + private sealed class NoopMeetingSummaryPipeline : IMeetingSummaryPipeline + { + public Task RunAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + return Task.FromResult(new MeetingSummaryRunResult(artifacts.SummaryPath, "", Succeeded: true)); + } + } +} diff --git a/MeetingAssistant.Tests/MacOsMeetingIntegrationTests.cs b/MeetingAssistant.Tests/MacOsMeetingIntegrationTests.cs new file mode 100644 index 0000000..0fc4cdd --- /dev/null +++ b/MeetingAssistant.Tests/MacOsMeetingIntegrationTests.cs @@ -0,0 +1,436 @@ +using System.Net.Http.Json; +using MeetingAssistant.Calendar; +using MeetingAssistant.MacOs; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Screenshots; +using MeetingAssistant.Transcription; +using MeetingAssistant.Workflow; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace MeetingAssistant.Tests; + +public sealed class MacOsMeetingIntegrationTests +{ + [Fact] + public void PortableRegistrationUsesRealMacOsProvidersOnlyOnMacOs() + { + var macServices = new ServiceCollection().AddLogging(); + var unsupportedServices = new ServiceCollection().AddLogging(); + + MacOsMeetingIntegrationRegistration.Add(macServices, isMacOs: true); + MacOsMeetingIntegrationRegistration.Add(unsupportedServices, isMacOs: false); + + using var macProvider = macServices.BuildServiceProvider(); + using var unsupportedProvider = unsupportedServices.BuildServiceProvider(); + + Assert.IsType( + macProvider.GetRequiredService()); + Assert.IsType( + macProvider.GetRequiredService()); + Assert.IsType( + macProvider.GetRequiredService()); + Assert.IsType( + macProvider.GetRequiredService()); + Assert.IsType( + macProvider.GetRequiredService()); + + Assert.IsType( + unsupportedProvider.GetRequiredService()); + Assert.IsType( + unsupportedProvider.GetRequiredService()); + Assert.IsType( + unsupportedProvider.GetRequiredService()); + Assert.IsType( + unsupportedProvider.GetRequiredService()); + Assert.IsType( + unsupportedProvider.GetRequiredService()); + } + + [Fact] + public async Task MacOsCalendarProvidersReturnTeamsMetadataAndPromptCandidate() + { + var startedAt = new DateTimeOffset(2026, 7, 25, 10, 5, 0, TimeSpan.FromHours(2)); + var calendarEvent = new MacOsCalendarEvent( + "event-1", + "Project sync", + startedAt.AddMinutes(-5), + startedAt.AddMinutes(55), + "Microsoft Teams", + "Discuss launch readiness.\nJoin Microsoft Teams Meeting", + "https://teams.microsoft.com/l/meetup-join/example", + ["Ada Lovelace ", "Grace Hopper "], + IsCanceled: false); + var client = new StubMacOsDesktopIntegrationClient(calendarEvent); + IMeetingMetadataProvider metadataProvider = new MacOsMeetingMetadataProvider( + client, + NullLogger.Instance); + ICalendarMeetingProvider calendarProvider = new MacOsCalendarMeetingProvider( + client, + NullLogger.Instance); + + var metadata = await metadataProvider.GetCurrentMeetingAsync( + startedAt, + CancellationToken.None); + var candidates = await calendarProvider.GetRecordingPromptCandidatesForDayAsync( + DateOnly.FromDateTime(startedAt.Date), + CancellationToken.None); + + Assert.NotNull(metadata); + Assert.Equal("Project sync", metadata.Title); + Assert.Equal( + ["Ada Lovelace ", "Grace Hopper "], + metadata.Attendees); + Assert.Equal("Discuss launch readiness.", metadata.Agenda); + Assert.Equal(calendarEvent.End, metadata.ScheduledEnd); + + var candidate = Assert.Single(candidates); + Assert.Equal("event-1", candidate.Id); + Assert.Equal("Project sync", candidate.Subject); + Assert.NotNull(candidate.Metadata); + Assert.Equal(metadata.Title, candidate.Metadata.Title); + Assert.Equal(metadata.Attendees, candidate.Metadata.Attendees); + Assert.Equal(metadata.Agenda, candidate.Metadata.Agenda); + Assert.Equal(metadata.ScheduledEnd, candidate.Metadata.ScheduledEnd); + } + + [Fact] + public async Task MacOsPromptAndScreenshotUseNativeClientResults() + { + var screenshot = new ActiveWindowScreenshot([137, 80, 78, 71], "Meeting deck"); + var client = new StubMacOsDesktopIntegrationClient( + screenshot: screenshot, + promptResponse: MeetingStartPromptResponse.Record); + IMeetingStartPromptService prompt = new MacOsMeetingStartPromptService( + client, + NullLogger.Instance); + IActiveWindowScreenshotCapture capture = client; + IScreenshotImageCropper cropper = client; + MeetingStartPromptResponse? callbackResponse = null; + var request = new MeetingStartPromptRequest(new CalendarMeeting( + "event-1", + "Project sync", + DateTimeOffset.Now, + DateTimeOffset.Now.AddHours(1))); + + await prompt.ShowPromptAsync( + request, + (response, _) => + { + callbackResponse = response; + return Task.CompletedTask; + }, + CancellationToken.None); + var captured = await capture.CaptureAsync(CancellationToken.None); + var croppedPath = await cropper.SaveCroppedScreenshotAsync( + "meeting.png", + new ScreenshotCropCoordinates(1, 2, 3, 4), + CancellationToken.None); + + Assert.Equal(MeetingStartPromptResponse.Record, callbackResponse); + Assert.Equal(screenshot, captured); + Assert.Equal("meeting.png.cropped.png", croppedPath); + } + + [Fact] + public async Task MacOsCapabilityEndpointReportsEnabledRealProviders() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + await using var factory = new WebApplicationFactory(); + using var client = factory.CreateClient(); + + var report = await client.GetFromJsonAsync( + "/diagnostics/platform-capabilities"); + + Assert.NotNull(report); + Assert.Equal("macOS", report.Host); + Assert.Equal(nameof(MacOsMeetingMetadataProvider), report.MeetingMetadataProvider); + Assert.Equal(nameof(MacOsCalendarMeetingProvider), report.CalendarMeetingProvider); + Assert.Equal(nameof(MacOsMeetingStartPromptService), report.MeetingStartPromptProvider); + Assert.Equal(nameof(MacOsDesktopIntegrationClient), report.ScreenshotProvider); + Assert.Equal(nameof(MacOsWorkflowRulesEditorWindowService), report.WorkflowEditorProvider); + Assert.True(report.WorkflowEditorAvailable); + Assert.True(report.CalendarRecordingPromptsEnabled); + Assert.True(report.SpeakerIdentificationEnabled); + Assert.True(report.ScreenshotOcrEnabled); + Assert.True(report.FunAsrBackendEnabled); + Assert.True(report.FunAsrDiarizationEnabled); + Assert.True(report.WhisperLocalDiarizationEnabled); + } + + [Fact] + public async Task NativeHelperAdvertisesCalendarPromptAndScreenshotFeatures() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var runner = new ProcessCommandRunner( + NullLogger.Instance); + + var result = await runner.RunAsync( + GetNativeHelperPath(), + ["self-test"], + CancellationToken.None); + + Assert.Equal(0, result.ExitCode); + using var json = System.Text.Json.JsonDocument.Parse(result.StandardOutput); + Assert.Equal("ok", json.RootElement.GetProperty("status").GetString()); + Assert.Equal( + ["calendar", "prompt", "screenshot", "crop"], + json.RootElement.GetProperty("features") + .EnumerateArray() + .Select(feature => feature.GetString()!) + .ToArray()); + } + + [Fact] + public async Task NativeHelperCropsPngUsingOcrPixelCoordinates() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var testRoot = Path.Combine( + Path.GetTempPath(), + $"meeting-assistant-native-crop-{Guid.NewGuid():N}"); + Directory.CreateDirectory(testRoot); + var bitmapPath = Path.Combine(testRoot, "input.bmp"); + var inputPath = Path.Combine(testRoot, "input.png"); + var outputPath = Path.Combine(testRoot, "output.png"); + var outputBitmapPath = Path.Combine(testRoot, "output.bmp"); + try + { + await File.WriteAllBytesAsync( + bitmapPath, + CreateTopRedBottomBlueBitmap()); + var runner = new ProcessCommandRunner( + NullLogger.Instance); + var inputConversion = await runner.RunAsync( + "/usr/bin/sips", + ["-s", "format", "png", bitmapPath, "--out", inputPath], + CancellationToken.None); + Assert.Equal(0, inputConversion.ExitCode); + + var result = await runner.RunAsync( + GetNativeHelperPath(), + ["crop", inputPath, outputPath, "0", "0", "2", "1"], + CancellationToken.None); + + Assert.Equal(0, result.ExitCode); + var png = await File.ReadAllBytesAsync(outputPath); + Assert.Equal((2, 1), PngTestFixture.ReadDimensions(png)); + var outputConversion = await runner.RunAsync( + "/usr/bin/sips", + ["-s", "format", "bmp", outputPath, "--out", outputBitmapPath], + CancellationToken.None); + Assert.Equal(0, outputConversion.ExitCode); + Assert.Equal( + (Red: (byte)255, Green: (byte)0, Blue: (byte)0), + ReadFirstBitmapPixel(await File.ReadAllBytesAsync(outputBitmapPath))); + } + finally + { + Directory.Delete(testRoot, recursive: true); + } + } + + [Fact] + public async Task CalendarProviderUsesActualLocalMidnightsAcrossDaylightSavingTransition() + { + if (TimeZoneInfo.Local.Id != "Europe/Berlin") + { + return; + } + + var client = new StubMacOsDesktopIntegrationClient(); + var provider = new MacOsCalendarMeetingProvider( + client, + NullLogger.Instance); + + await provider.GetRecordingPromptCandidatesForDayAsync( + new DateOnly(2026, 3, 29), + CancellationToken.None); + + Assert.Equal( + DateTimeOffset.Parse("2026-03-29T00:00:00+01:00"), + client.CalendarFrom); + Assert.Equal( + DateTimeOffset.Parse("2026-03-30T00:00:00+02:00"), + client.CalendarTo); + Assert.Equal(TimeSpan.FromHours(23), client.CalendarTo - client.CalendarFrom); + } + + [Fact] + public async Task CalendarClientFallsBackToCalendarAutomationWhenEventKitIsDenied() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var runner = new SequenceCommandRunner( + new CommandResult( + MacOsIntegrationException.CalendarPermissionDeniedExitCode, + "", + "Calendar Full Access permission is required."), + new CommandResult( + 0, + """ + [{"id":"fallback-event","subject":"Project sync","start":"2026-07-25T08:00:00Z","end":"2026-07-25T09:00:00Z","location":"Microsoft Teams","notes":"","url":"https://teams.microsoft.com/example","attendees":[],"isCanceled":false}] + """, + "")); + IMacOsCalendarClient client = new MacOsDesktopIntegrationClient( + runner, + Path.GetDirectoryName(GetNativeHelperPath())!); + + var events = await client.GetCalendarEventsAsync( + DateTimeOffset.Parse("2026-07-25T00:00:00Z"), + DateTimeOffset.Parse("2026-07-26T00:00:00Z"), + CancellationToken.None); + + var calendarEvent = Assert.Single(events); + Assert.Equal("fallback-event", calendarEvent.Id); + Assert.Collection( + runner.Commands, + command => Assert.Equal("calendar", command.Arguments[0]), + command => + { + Assert.Equal("/usr/bin/osascript", command.FileName); + Assert.Equal(["-l", "JavaScript"], command.Arguments.Take(2)); + }); + } + + private static string GetNativeHelperPath() + { + var configuration = new DirectoryInfo(AppContext.BaseDirectory) + .Parent?.Name ?? "Debug"; + return Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "..", + "MeetingAssistant", + "bin", + configuration, + "net10.0", + "Native", + "macos-meeting-integrations")); + } + + private static byte[] CreateTopRedBottomBlueBitmap() + { + var bitmap = new byte[70]; + bitmap[0] = (byte)'B'; + bitmap[1] = (byte)'M'; + BitConverter.GetBytes(bitmap.Length).CopyTo(bitmap, 2); + BitConverter.GetBytes(54).CopyTo(bitmap, 10); + BitConverter.GetBytes(40).CopyTo(bitmap, 14); + BitConverter.GetBytes(2).CopyTo(bitmap, 18); + BitConverter.GetBytes(2).CopyTo(bitmap, 22); + BitConverter.GetBytes((short)1).CopyTo(bitmap, 26); + BitConverter.GetBytes((short)24).CopyTo(bitmap, 28); + BitConverter.GetBytes(16).CopyTo(bitmap, 34); + + // BMP rows are bottom-up and padded to four-byte boundaries. + bitmap[54] = 255; + bitmap[57] = 255; + bitmap[64] = 255; + bitmap[67] = 255; + return bitmap; + } + + private static (byte Red, byte Green, byte Blue) ReadFirstBitmapPixel(byte[] bitmap) + { + var pixelOffset = BitConverter.ToInt32(bitmap, 10); + return ( + Red: bitmap[pixelOffset + 2], + Green: bitmap[pixelOffset + 1], + Blue: bitmap[pixelOffset]); + } + + private sealed class StubMacOsDesktopIntegrationClient : + IMacOsCalendarClient, + IMacOsMeetingPromptClient, + IActiveWindowScreenshotCapture, + IScreenshotImageCropper + { + private readonly IReadOnlyList events; + private readonly ActiveWindowScreenshot screenshot; + private readonly MeetingStartPromptResponse promptResponse; + + public StubMacOsDesktopIntegrationClient( + MacOsCalendarEvent? calendarEvent = null, + ActiveWindowScreenshot? screenshot = null, + MeetingStartPromptResponse promptResponse = MeetingStartPromptResponse.Skip) + { + events = calendarEvent is null ? [] : [calendarEvent]; + this.screenshot = screenshot ?? new ActiveWindowScreenshot([], null); + this.promptResponse = promptResponse; + } + + public Task> GetCalendarEventsAsync( + DateTimeOffset from, + DateTimeOffset to, + CancellationToken cancellationToken) + { + CalendarFrom = from; + CalendarTo = to; + return Task.FromResult(events); + } + + public DateTimeOffset CalendarFrom { get; private set; } + + public DateTimeOffset CalendarTo { get; private set; } + + public Task ShowMeetingStartPromptAsync( + MeetingStartPromptRequest request, + CancellationToken cancellationToken) + { + return Task.FromResult(promptResponse); + } + + public Task CaptureAsync(CancellationToken cancellationToken) + { + return Task.FromResult(screenshot); + } + + public Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + return Task.FromResult($"{screenshotPath}.cropped.png"); + } + } + + private sealed class SequenceCommandRunner : ICommandRunner + { + private readonly Queue results; + + public SequenceCommandRunner(params CommandResult[] results) + { + this.results = new Queue(results); + } + + public List<(string FileName, IReadOnlyList Arguments)> Commands { get; } = []; + + public Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + Commands.Add((fileName, arguments)); + return Task.FromResult(results.Dequeue()); + } + } +} diff --git a/MeetingAssistant.Tests/MeetingNoteStoreTests.cs b/MeetingAssistant.Tests/MeetingNoteStoreTests.cs index b25b699..8c2422b 100644 --- a/MeetingAssistant.Tests/MeetingNoteStoreTests.cs +++ b/MeetingAssistant.Tests/MeetingNoteStoreTests.cs @@ -206,7 +206,7 @@ public sealed class MeetingNoteStoreTests { var link = MeetingNoteActionLinks.CreateSummaryRetryLink( "http://localhost:5090/", - "C:\\Vault\\Meetings\\Summaries\\summary with spaces.md"); + Path.Combine(Path.GetTempPath(), "summary with spaces.md")); Assert.Equal( "[Retry summary generation](http://localhost:5090/meetings/summary/retry?summaryPath=summary%20with%20spaces.md)", diff --git a/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs b/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs index 926d76b..64f2f12 100644 --- a/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs +++ b/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs @@ -4,8 +4,6 @@ using MeetingAssistant.Speakers; using MeetingAssistant.Workflow; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using System.Drawing; -using System.Drawing.Imaging; #pragma warning disable CA1416 @@ -117,11 +115,7 @@ public sealed class MeetingScreenshotServiceTests var croppedPath = Assert.Single(Directory.GetFiles( Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"), "*-cropped.png")); - using (var cropped = new Bitmap(croppedPath)) - { - Assert.Equal(2, cropped.Width); - Assert.Equal(2, cropped.Height); - } + Assert.Equal((2, 2), PngTestFixture.ReadDimensions(await File.ReadAllBytesAsync(croppedPath))); var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath); var screenshotIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal); @@ -133,6 +127,38 @@ public sealed class MeetingScreenshotServiceTests Assert.Contains("Shared screen text", context); } + [Fact] + public async Task CaptureKeepsSuccessfulOcrTextWhenCropperRejectsCoordinates() + { + var fixture = await ScreenshotFixture.CreateAsync(options => + { + options.Screenshots.Ocr.Enabled = true; + }); + var ocr = new CapturingScreenshotOcrClient( + "Readable text survives.", + new ScreenshotCropCoordinates(99, 99, 10, 10)); + var service = fixture.CreateService( + new FixedScreenshotCapture(CreatePngBytes(4, 4)), + ocr, + screenshotImageCropper: new RejectingScreenshotImageCropper()); + + await service.CaptureAsync( + fixture.Artifacts, + DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"), + DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"), + fixture.Options, + CancellationToken.None); + await service.WaitForPendingOcrAsync( + fixture.Artifacts, + TimeSpan.FromSeconds(5), + CancellationToken.None); + + var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath); + Assert.Contains("Readable text survives.", context); + Assert.DoesNotContain("_OCR failed", context); + Assert.DoesNotContain("![Cropped screenshot]", context); + } + [Fact] public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer() { @@ -454,7 +480,8 @@ public sealed class MeetingScreenshotServiceTests IActiveWindowScreenshotCapture capture, IScreenshotOcrClient ocrClient, ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null, - IMeetingWorkflowEngine? meetingWorkflowEngine = null) + IMeetingWorkflowEngine? meetingWorkflowEngine = null, + IScreenshotImageCropper? screenshotImageCropper = null) { return new MeetingScreenshotService( capture, @@ -462,12 +489,28 @@ public sealed class MeetingScreenshotServiceTests NoteStore, attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance, ocrClient, + screenshotImageCropper ?? + capture as IScreenshotImageCropper ?? + new UnavailableScreenshotImageCropper(), NullLogger.Instance, meetingWorkflowEngine); } } - private sealed class FixedScreenshotCapture : IActiveWindowScreenshotCapture + private sealed class RejectingScreenshotImageCropper : IScreenshotImageCropper + { + public Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + throw new InvalidDataException("The crop is outside image bounds."); + } + } + + private sealed class FixedScreenshotCapture : + IActiveWindowScreenshotCapture, + IScreenshotImageCropper { private readonly byte[] bytes; @@ -480,6 +523,21 @@ public sealed class MeetingScreenshotServiceTests { return Task.FromResult(new ActiveWindowScreenshot(bytes, "Demo Window")); } + + public async Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + var croppedPath = Path.Combine( + Path.GetDirectoryName(screenshotPath)!, + $"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png"); + await File.WriteAllBytesAsync( + croppedPath, + PngTestFixture.Solid(2, 2), + cancellationToken); + return croppedPath; + } } private sealed class CapturingScreenshotOcrClient : IScreenshotOcrClient @@ -614,15 +672,7 @@ public sealed class MeetingScreenshotServiceTests private static byte[] CreatePngBytes(int width, int height) { - using var bitmap = new Bitmap(width, height); - using (var graphics = Graphics.FromImage(bitmap)) - { - graphics.Clear(Color.White); - } - - using var stream = new MemoryStream(); - bitmap.Save(stream, ImageFormat.Png); - return stream.ToArray(); + return PngTestFixture.Solid(width, height); } private static string ExtractScreenshotOcrId(string context) diff --git a/MeetingAssistant.Tests/PngTestFixture.cs b/MeetingAssistant.Tests/PngTestFixture.cs new file mode 100644 index 0000000..dd6804f --- /dev/null +++ b/MeetingAssistant.Tests/PngTestFixture.cs @@ -0,0 +1,38 @@ +namespace MeetingAssistant.Tests; + +internal static class PngTestFixture +{ + private const string Solid2X2 = + "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAAqADAAQAAAABAAAAAgAAAADO0J6QAAAAEklEQVQIHWP8DwQMQMAEIkAAAD34BACALvQ5AAAAAElFTkSuQmCC"; + private const string Solid4X4 = + "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAABKADAAQAAAABAAAABAAAAADmpNw4AAAAFUlEQVQIHWP8DwQMSIAJiQ1mEhYAAAZdBAQjjGwcAAAAAElFTkSuQmCC"; + private const string Solid8X6 = + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAYAAAD+Bd/7AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAACKADAAQAAAABAAAABgAAAAAJfOcJAAAAFklEQVQIHWP8DwQMeAATHjmw1GBQAAAQiQQIVmhjoQAAAABJRU5ErkJggg=="; + + public static byte[] Solid(int width, int height) + { + var encoded = (width, height) switch + { + (2, 2) => Solid2X2, + (4, 4) => Solid4X4, + (8, 6) => Solid8X6, + _ => throw new ArgumentOutOfRangeException( + nameof(width), + $"No deterministic PNG fixture exists for {width}x{height}.") + }; + return Convert.FromBase64String(encoded); + } + + public static (int Width, int Height) ReadDimensions(byte[] png) + { + if (!png.AsSpan(0, 8).SequenceEqual( + new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 })) + { + throw new InvalidDataException("The fixture is not a PNG."); + } + + return ( + System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(16, 4)), + System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(20, 4))); + } +} diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs index 5720250..36e79c9 100644 --- a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -448,9 +448,14 @@ public sealed class RecordingCoordinatorTests [Fact] public async Task StartCreatesMeetingNoteLinkedToTranscriptAndOpensIt() { + var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-vault"); + var transcriptPath = Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"); + var meetingNotePath = Path.Combine(vaultRoot, "Meetings", "Notes", "20260519-meeting.md"); + var assistantContextFolder = Path.Combine(vaultRoot, "Meetings", "Assistant Context"); + var summariesFolder = Path.Combine(vaultRoot, "Meetings", "Summaries"); var audioSource = new ControlledAudioSource(); - var transcriptStore = new InMemoryTranscriptStore("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md"); - var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md"); + var transcriptStore = new InMemoryTranscriptStore(transcriptPath); + var noteStore = new InMemoryMeetingNoteStore(meetingNotePath); var clock = new ManualMeetingInactivityClock(DateTimeOffset.Parse("2026-05-19T10:03:42+02:00")); var noteOpener = new CapturingMeetingNoteOpener(); var artifactStore = new InMemoryMeetingArtifactStore(); @@ -468,25 +473,27 @@ public sealed class RecordingCoordinatorTests { Vault = new VaultOptions { - AssistantContextFolder = "C:\\Vault\\Meetings\\Assistant Context", - SummariesFolder = "C:\\Vault\\Meetings\\Summaries" + AssistantContextFolder = assistantContextFolder, + SummariesFolder = summariesFolder } }), NullLogger.Instance, inactivityClock: clock); var status = await coordinator.StartAsync(CancellationToken.None); + var assistantContextPath = Path.Combine(assistantContextFolder, "20260519-1003-context.md"); + var summaryPath = Path.Combine(summariesFolder, "20260519-1003-summary.md"); Assert.True(status.IsRecording); - Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", status.MeetingNotePath); - Assert.Equal("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md", noteStore.SavedNote?.Frontmatter.Transcript); - Assert.Equal("C:\\Vault\\Meetings\\Assistant Context\\20260519-1003-context.md", noteStore.SavedNote?.Frontmatter.AssistantContext); - Assert.Equal("C:\\Vault\\Meetings\\Summaries\\20260519-1003-summary.md", noteStore.SavedNote?.Frontmatter.Summary); - Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", noteOpener.OpenedPath); - Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", artifactStore.CreatedArtifacts?.MeetingNotePath); - Assert.Equal("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md", artifactStore.CreatedArtifacts?.TranscriptPath); - Assert.Equal("C:\\Vault\\Meetings\\Assistant Context\\20260519-1003-context.md", artifactStore.CreatedArtifacts?.AssistantContextPath); - Assert.Equal("C:\\Vault\\Meetings\\Summaries\\20260519-1003-summary.md", artifactStore.CreatedArtifacts?.SummaryPath); + Assert.Equal(meetingNotePath, status.MeetingNotePath); + Assert.Equal(transcriptPath, noteStore.SavedNote?.Frontmatter.Transcript); + Assert.Equal(assistantContextPath, noteStore.SavedNote?.Frontmatter.AssistantContext); + Assert.Equal(summaryPath, noteStore.SavedNote?.Frontmatter.Summary); + Assert.Equal(meetingNotePath, noteOpener.OpenedPath); + Assert.Equal(meetingNotePath, artifactStore.CreatedArtifacts?.MeetingNotePath); + Assert.Equal(transcriptPath, artifactStore.CreatedArtifacts?.TranscriptPath); + Assert.Equal(assistantContextPath, artifactStore.CreatedArtifacts?.AssistantContextPath); + Assert.Equal(summaryPath, artifactStore.CreatedArtifacts?.SummaryPath); Assert.Equal(noteStore.SavedNote?.Frontmatter.Title, artifactStore.ContextMeetingNote?.Frontmatter.Title); Assert.Equal(noteStore.SavedNote?.Frontmatter.StartTime, artifactStore.ContextMeetingNote?.Frontmatter.StartTime); @@ -2108,6 +2115,45 @@ public sealed class RecordingCoordinatorTests Assert.False(audioArchive.Deleted); } + [Fact] + public async Task StopQueuesAzureBacklogAndRetainsAudioWhenTranscriptionFails() + { + var audioSource = new ControlledAudioSource(); + var provider = new FailingAfterFirstAudioProvider(); + var audioArchive = new InMemoryRecordedAudioStore(); + var backlog = new InMemoryOfflineTranscriptionBacklog(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions + { + TranscriptionProvider = "azure-speech" + } + }), + NullLogger.Instance, + offlineTranscriptionBacklog: backlog); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await provider.WaitUntilFailureObservedAsync(); + + var stopped = await coordinator.StopAsync(CancellationToken.None); + + Assert.False(stopped.IsRecording); + var item = Assert.Single(backlog.Items); + Assert.Equal("memory-recording.wav", item.AudioPath); + Assert.True(audioArchive.Completed); + Assert.False(audioArchive.Deleted); + } + [Fact] public async Task OfflineBacklogReplaysQueuedRecordingAndCompletesMeetingArtifacts() { @@ -6141,6 +6187,31 @@ public sealed class RecordingCoordinatorTests } } + private sealed class FailingAfterFirstAudioProvider : IStreamingTranscriptionProvider + { + private readonly TaskCompletionSource failureObserved = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilFailureObservedAsync() + { + return failureObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + failureObserved.TrySetResult(); + throw new InvalidOperationException("Configured transcription backend is unavailable."); + } + + yield break; + } + } + private static byte[] Samples(params short[] samples) { var bytes = new byte[samples.Length * sizeof(short)]; @@ -6148,6 +6219,3 @@ public sealed class RecordingCoordinatorTests return bytes; } } - - - diff --git a/MeetingAssistant.Tests/TaskbarIconTests.cs b/MeetingAssistant.Tests/TaskbarIconTests.cs index ec2eea3..18c3528 100644 --- a/MeetingAssistant.Tests/TaskbarIconTests.cs +++ b/MeetingAssistant.Tests/TaskbarIconTests.cs @@ -181,6 +181,11 @@ public sealed class TaskbarIconTests [SupportedOSPlatform("windows")] public void TaskbarIconGlyphsAreVisuallyCentered() { + if (!OperatingSystem.IsWindows()) + { + return; + } + foreach (var state in new[] { RecordingProcessState.Idle, diff --git a/MeetingAssistant.Tests/VaultPathTests.cs b/MeetingAssistant.Tests/VaultPathTests.cs index 91e694e..f828ab1 100644 --- a/MeetingAssistant.Tests/VaultPathTests.cs +++ b/MeetingAssistant.Tests/VaultPathTests.cs @@ -7,16 +7,16 @@ public sealed class VaultPathTests [Fact] public void ResolveUsesVaultBaseFolderForRelativePaths() { + var baseFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-vault"); var vault = new VaultOptions { - BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta" + BaseFolder = baseFolder }; - var resolved = VaultPath.Resolve(vault, @"Meetings\Transcripts"); + var relativePath = Path.Combine("Meetings", "Transcripts"); + var resolved = VaultPath.Resolve(vault, relativePath); - Assert.Equal( - @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta\Meetings\Transcripts", - resolved); + Assert.Equal(Path.Combine(baseFolder, relativePath), resolved); } [Fact] @@ -24,11 +24,12 @@ public sealed class VaultPathTests { var vault = new VaultOptions { - BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta" + BaseFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-vault") }; + var absolutePath = Path.Combine(Path.GetTempPath(), "meeting-assistant-other"); - var resolved = VaultPath.Resolve(vault, @"C:\Other\Path"); + var resolved = VaultPath.Resolve(vault, absolutePath); - Assert.Equal(@"C:\Other\Path", resolved); + Assert.Equal(absolutePath, resolved); } } diff --git a/MeetingAssistant.Tests/VenvResemblyzerVoiceEncoderTests.cs b/MeetingAssistant.Tests/VenvResemblyzerVoiceEncoderTests.cs index ecdac99..00a771c 100644 --- a/MeetingAssistant.Tests/VenvResemblyzerVoiceEncoderTests.cs +++ b/MeetingAssistant.Tests/VenvResemblyzerVoiceEncoderTests.cs @@ -71,7 +71,9 @@ public sealed class VenvResemblyzerVoiceEncoderTests command => command.Arguments.Any( argument => argument.EndsWith("encode.py", StringComparison.Ordinal))); Assert.EndsWith( - Path.Combine("Scripts", "python.exe"), + OperatingSystem.IsWindows() + ? Path.Combine("Scripts", "python.exe") + : Path.Combine("bin", "python"), encodingCommand.FileName, StringComparison.OrdinalIgnoreCase); Assert.Equal(2, encodingCommand.Arguments.Count); diff --git a/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs b/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs index 14814e2..032c57f 100644 --- a/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs +++ b/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs @@ -732,6 +732,7 @@ public sealed class WorkflowRulesEditorTests var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); var rulesPath = Path.Combine(root, "rules.yaml"); var builder = new WorkflowRulesEditorInstructionBuilder( + HostOperatingSystem.DetectCurrent(), NullLogger.Instance); var instructions = await builder.BuildAsync(new MeetingAssistantOptions @@ -749,10 +750,61 @@ public sealed class WorkflowRulesEditorTests Assert.Contains("list_recent_summaries", instructions); } + [Fact] + public async Task InstructionBuilderUsesWindowsCommandsAndConceptsOnWindows() + { + var builder = new WorkflowRulesEditorInstructionBuilder( + new HostOperatingSystem(HostOperatingSystemKind.Windows), + NullLogger.Instance); + + var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None); + + Assert.Contains("Detected host operating system: Windows", instructions); + Assert.Contains("PowerShell", instructions); + Assert.Contains("Windows-style paths", instructions); + Assert.Contains("Windows services", instructions); + Assert.Contains("Task Manager", instructions); + Assert.Contains("Do not use macOS-only commands or concepts", instructions); + Assert.Contains("does not make unavailable integrations available", instructions); + } + + [Fact] + public async Task InstructionBuilderUsesMacOSCommandsAndConceptsOnMacOS() + { + var builder = new WorkflowRulesEditorInstructionBuilder( + new HostOperatingSystem(HostOperatingSystemKind.MacOS), + NullLogger.Instance); + + var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None); + + Assert.Contains("Detected host operating system: macOS", instructions); + Assert.Contains("zsh", instructions); + Assert.Contains("POSIX-style paths", instructions); + Assert.Contains("launchd and LaunchAgents", instructions); + Assert.Contains("Activity Monitor", instructions); + Assert.Contains("Do not use Windows-only commands or concepts", instructions); + Assert.Contains("does not make unavailable integrations available", instructions); + } + + [Fact] + public async Task InstructionBuilderUsesPortableConceptsOnUnsupportedHosts() + { + var builder = new WorkflowRulesEditorInstructionBuilder( + new HostOperatingSystem(HostOperatingSystemKind.Unsupported), + NullLogger.Instance); + + var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None); + + Assert.Contains("Detected host operating system: unsupported", instructions); + Assert.Contains("portable .NET concepts", instructions); + Assert.Contains("Do not assume Windows or macOS", instructions); + } + [Fact] public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes() { var builder = new WorkflowRulesEditorInstructionBuilder( + HostOperatingSystem.DetectCurrent(), NullLogger.Instance); var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None); @@ -767,6 +819,7 @@ public sealed class WorkflowRulesEditorTests public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs() { var builder = new WorkflowRulesEditorInstructionBuilder( + new HostOperatingSystem(HostOperatingSystemKind.MacOS), NullLogger.Instance); var options = new MeetingAssistantOptions { @@ -784,6 +837,8 @@ public sealed class WorkflowRulesEditorTests Assert.Contains("read the matching assistant context", instructions); Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions); Assert.Contains("append a concise record of your fixes and conclusions", instructions); + Assert.Contains("Detected host operating system: macOS", instructions); + Assert.Contains("zsh", instructions); } [Fact] diff --git a/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs b/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs index 9ba393e..ff5e18d 100644 --- a/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs +++ b/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs @@ -52,14 +52,14 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService cachedMeetingDay = today; nextSyncAt = now.Add(GetSyncInterval(promptOptions)); logger.LogInformation( - "Synced {MeetingCount} Outlook Teams calendar meetings for {Day}; next sync at {NextSyncAt}", + "Synced {MeetingCount} Teams calendar meetings for {Day}; next sync at {NextSyncAt}", cachedMeetings.Count, today, nextSyncAt); } catch (Exception exception) when (exception is not OperationCanceledException) { - logger.LogWarning(exception, "Could not read today's Outlook Teams meetings for recording prompts"); + logger.LogWarning(exception, "Could not read today's Teams calendar meetings for recording prompts"); nextSyncAt = now.Add(GetSyncInterval(promptOptions)); return; } diff --git a/MeetingAssistant/HostOperatingSystem.cs b/MeetingAssistant/HostOperatingSystem.cs new file mode 100644 index 0000000..41f4920 --- /dev/null +++ b/MeetingAssistant/HostOperatingSystem.cs @@ -0,0 +1,26 @@ +namespace MeetingAssistant; + +public enum HostOperatingSystemKind +{ + Windows, + MacOS, + Unsupported +} + +public sealed record HostOperatingSystem(HostOperatingSystemKind Kind) +{ + public static HostOperatingSystem DetectCurrent() + { + if (OperatingSystem.IsWindows()) + { + return new HostOperatingSystem(HostOperatingSystemKind.Windows); + } + + if (OperatingSystem.IsMacOS()) + { + return new HostOperatingSystem(HostOperatingSystemKind.MacOS); + } + + return new HostOperatingSystem(HostOperatingSystemKind.Unsupported); + } +} diff --git a/MeetingAssistant/MacOs/MacOsDesktopControlManifest.cs b/MeetingAssistant/MacOs/MacOsDesktopControlManifest.cs new file mode 100644 index 0000000..171d58d --- /dev/null +++ b/MeetingAssistant/MacOs/MacOsDesktopControlManifest.cs @@ -0,0 +1,47 @@ +using MeetingAssistant.LaunchProfiles; + +namespace MeetingAssistant.MacOs; + +public sealed record MacOsDesktopControlManifest( + string BaseUrl, + string StatusUrl, + string OpenAgentUrl, + IReadOnlyList Profiles, + IReadOnlyList Hotkeys) +{ + public static MacOsDesktopControlManifest Create( + string baseUrl, + IReadOnlyList profiles, + IReadOnlyList hotkeys) + { + var normalizedBaseUrl = baseUrl.TrimEnd('/'); + return new MacOsDesktopControlManifest( + normalizedBaseUrl, + $"{normalizedBaseUrl}/recording/status", + $"{normalizedBaseUrl}/diagnostics/settings-and-logs", + profiles + .Select(profile => new MacOsDesktopProfile( + profile.Name, + profile.Options.Hotkey.Toggle)) + .ToArray(), + hotkeys.Select(CreateHotkey).ToArray()); + } + + private static MacOsDesktopHotkey CreateHotkey(LaunchProfileHotkey hotkey) + { + var endpoint = hotkey.Action switch + { + LaunchProfileHotkeyAction.ToggleRecording => "recording/toggle", + LaunchProfileHotkeyAction.AbortRecording => "recording/abort", + LaunchProfileHotkeyAction.CaptureScreenshot => "meetings/screenshot/capture", + _ => throw new ArgumentOutOfRangeException(nameof(hotkey)) + }; + return new MacOsDesktopHotkey( + hotkey.Hotkey, + $"/profiles/{Uri.EscapeDataString(hotkey.ProfileName)}/{endpoint}"); + } +} + +public sealed record MacOsDesktopProfile(string Name, string ToggleShortcut); + +public sealed record MacOsDesktopHotkey(string Shortcut, string Endpoint); diff --git a/MeetingAssistant/MacOs/MacOsDesktopControlService.cs b/MeetingAssistant/MacOs/MacOsDesktopControlService.cs new file mode 100644 index 0000000..7bcc12b --- /dev/null +++ b/MeetingAssistant/MacOs/MacOsDesktopControlService.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.Text.Json; +using MeetingAssistant.LaunchProfiles; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.MacOs; + +public sealed class MacOsDesktopControlService : IHostedService, IDisposable +{ + private readonly ILaunchProfileOptionsProvider launchProfiles; + private readonly IOptions options; + private readonly IHostApplicationLifetime applicationLifetime; + private readonly ILogger logger; + private Process? helper; + private string? manifestPath; + + public MacOsDesktopControlService( + ILaunchProfileOptionsProvider launchProfiles, + IOptions options, + IHostApplicationLifetime applicationLifetime, + ILogger logger) + { + this.launchProfiles = launchProfiles; + this.options = options; + this.applicationLifetime = applicationLifetime; + this.logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + if (!OperatingSystem.IsMacOS()) + { + return Task.CompletedTask; + } + + var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-desktop-controls"); + if (!File.Exists(helperPath)) + { + logger.LogWarning("macOS desktop controls helper was not found at {HelperPath}", helperPath); + return Task.CompletedTask; + } + + var manifest = MacOsDesktopControlManifest.Create( + options.Value.Api.PublicBaseUrl, + launchProfiles.GetProfiles(), + launchProfiles.GetHotkeys()); + manifestPath = Path.Combine( + Path.GetTempPath(), + $"meeting-assistant-desktop-{Environment.ProcessId}.json"); + File.WriteAllText( + manifestPath, + JsonSerializer.Serialize( + manifest, + new JsonSerializerOptions(JsonSerializerDefaults.Web))); + + helper = Process.Start(new ProcessStartInfo + { + FileName = helperPath, + ArgumentList = { manifestPath }, + UseShellExecute = false, + CreateNoWindow = true + }); + + if (helper is null) + { + throw new InvalidOperationException("Could not start the macOS desktop controls helper."); + } + + helper.EnableRaisingEvents = true; + helper.Exited += (_, _) => + { + if (!applicationLifetime.ApplicationStopping.IsCancellationRequested) + { + logger.LogWarning("macOS desktop controls helper exited unexpectedly"); + } + }; + logger.LogInformation("Started macOS menu-bar controls with {HotkeyCount} global hotkeys", manifest.Hotkeys.Count); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + if (helper is { HasExited: false }) + { + helper.Kill(); + helper.WaitForExit(TimeSpan.FromSeconds(2)); + } + + DeleteManifest(); + return Task.CompletedTask; + } + + public void Dispose() + { + helper?.Dispose(); + DeleteManifest(); + } + + private void DeleteManifest() + { + if (manifestPath is not null && File.Exists(manifestPath)) + { + File.Delete(manifestPath); + manifestPath = null; + } + } +} diff --git a/MeetingAssistant/MacOs/MacOsMeetingIntegrations.cs b/MeetingAssistant/MacOs/MacOsMeetingIntegrations.cs new file mode 100644 index 0000000..73983d6 --- /dev/null +++ b/MeetingAssistant/MacOs/MacOsMeetingIntegrations.cs @@ -0,0 +1,447 @@ +using System.Text.Json; +using MeetingAssistant.Calendar; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Screenshots; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeetingAssistant.MacOs; + +internal sealed record MacOsCalendarEvent( + string Id, + string Subject, + DateTimeOffset Start, + DateTimeOffset End, + string Location, + string Notes, + string Url, + IReadOnlyList Attendees, + bool IsCanceled); + +internal interface IMacOsCalendarClient +{ + Task> GetCalendarEventsAsync( + DateTimeOffset from, + DateTimeOffset to, + CancellationToken cancellationToken); +} + +internal interface IMacOsMeetingPromptClient +{ + Task ShowMeetingStartPromptAsync( + MeetingStartPromptRequest request, + CancellationToken cancellationToken); +} + +internal sealed class MacOsDesktopIntegrationClient : + IMacOsCalendarClient, + IMacOsMeetingPromptClient, + IActiveWindowScreenshotCapture, + IScreenshotImageCropper +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private readonly ICommandRunner commandRunner; + private readonly string nativeDirectory; + + public MacOsDesktopIntegrationClient(ICommandRunner commandRunner) + : this(commandRunner, Path.Combine(AppContext.BaseDirectory, "Native")) + { + } + + internal MacOsDesktopIntegrationClient( + ICommandRunner commandRunner, + string nativeDirectory) + { + this.commandRunner = commandRunner; + this.nativeDirectory = nativeDirectory; + } + + public async Task> GetCalendarEventsAsync( + DateTimeOffset from, + DateTimeOffset to, + CancellationToken cancellationToken) + { + var fromValue = from.ToString("O", System.Globalization.CultureInfo.InvariantCulture); + var toValue = to.ToString("O", System.Globalization.CultureInfo.InvariantCulture); + CommandResult result; + try + { + result = await RunHelperAsync( + ["calendar", fromValue, toValue], + cancellationToken); + } + catch (MacOsIntegrationException exception) + when (exception.ExitCode == MacOsIntegrationException.CalendarPermissionDeniedExitCode) + { + result = await RunCalendarAutomationFallbackAsync( + fromValue, + toValue, + cancellationToken); + } + + return JsonSerializer.Deserialize(result.StandardOutput, JsonOptions) ?? []; + } + + public async Task ShowMeetingStartPromptAsync( + MeetingStartPromptRequest request, + CancellationToken cancellationToken) + { + var requestPath = Path.Combine( + Path.GetTempPath(), + $"meeting-assistant-prompt-{Guid.NewGuid():N}.json"); + try + { + await File.WriteAllTextAsync( + requestPath, + JsonSerializer.Serialize(new + { + subject = request.Meeting.Subject, + start = request.Meeting.Start + }, JsonOptions), + cancellationToken); + var result = await RunHelperAsync(["prompt", requestPath], cancellationToken); + var response = JsonSerializer.Deserialize( + result.StandardOutput, + JsonOptions); + return string.Equals(response?.Response, "record", StringComparison.OrdinalIgnoreCase) + ? MeetingStartPromptResponse.Record + : MeetingStartPromptResponse.Skip; + } + finally + { + File.Delete(requestPath); + } + } + + public async Task CaptureAsync( + CancellationToken cancellationToken) + { + var outputPath = Path.Combine( + Path.GetTempPath(), + $"meeting-assistant-screenshot-{Guid.NewGuid():N}.png"); + try + { + var result = await RunHelperAsync(["screenshot", outputPath], cancellationToken); + var response = JsonSerializer.Deserialize( + result.StandardOutput, + JsonOptions); + return new ActiveWindowScreenshot( + await File.ReadAllBytesAsync(outputPath, cancellationToken), + response?.WindowTitle); + } + finally + { + File.Delete(outputPath); + } + } + + public async Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + var croppedPath = Path.Combine( + Path.GetDirectoryName(screenshotPath)!, + $"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png"); + try + { + await RunHelperAsync( + [ + "crop", + screenshotPath, + croppedPath, + crop.X.ToString(System.Globalization.CultureInfo.InvariantCulture), + crop.Y.ToString(System.Globalization.CultureInfo.InvariantCulture), + crop.Width.ToString(System.Globalization.CultureInfo.InvariantCulture), + crop.Height.ToString(System.Globalization.CultureInfo.InvariantCulture) + ], + cancellationToken); + } + catch (MacOsIntegrationException exception) + { + throw new InvalidDataException( + $"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.", + exception); + } + return croppedPath; + } + + private async Task RunHelperAsync( + IReadOnlyList arguments, + CancellationToken cancellationToken) + { + if (!OperatingSystem.IsMacOS()) + { + throw new PlatformNotSupportedException( + "The native macOS desktop integrations can only run on macOS."); + } + + var helperPath = Path.Combine(nativeDirectory, "macos-meeting-integrations"); + if (!File.Exists(helperPath)) + { + throw new FileNotFoundException( + $"The macOS meeting integrations helper was not found at '{helperPath}'. Build or publish Meeting Assistant on macOS.", + helperPath); + } + + var result = await commandRunner.RunAsync(helperPath, arguments, cancellationToken); + if (result.ExitCode != 0) + { + var detail = string.IsNullOrWhiteSpace(result.StandardError) + ? "The helper did not provide an error message." + : result.StandardError.Trim(); + throw new MacOsIntegrationException( + result.ExitCode, + $"macOS meeting integration '{arguments[0]}' failed with exit code {result.ExitCode}. {detail}"); + } + + return result; + } + + private async Task RunCalendarAutomationFallbackAsync( + string from, + string to, + CancellationToken cancellationToken) + { + var scriptPath = Path.Combine(nativeDirectory, "macos-calendar-events.js"); + if (!File.Exists(scriptPath)) + { + throw new FileNotFoundException( + $"The macOS Calendar automation fallback was not found at '{scriptPath}'.", + scriptPath); + } + + var result = await commandRunner.RunAsync( + "/usr/bin/osascript", + ["-l", "JavaScript", scriptPath, from, to], + cancellationToken); + if (result.ExitCode != 0) + { + var detail = string.IsNullOrWhiteSpace(result.StandardError) + ? "Calendar automation did not provide an error message." + : result.StandardError.Trim(); + throw new InvalidOperationException( + $"macOS calendar access failed through both EventKit and Calendar automation. {detail}"); + } + + return result; + } + + private sealed record MacOsPromptResult(string Response); + + private sealed record MacOsScreenshotResult(string? WindowTitle); +} + +internal sealed class MacOsIntegrationException : InvalidOperationException +{ + public const int CalendarPermissionDeniedExitCode = 77; + + public MacOsIntegrationException(int exitCode, string message) + : base(message) + { + ExitCode = exitCode; + } + + public int ExitCode { get; } +} + +internal sealed class MacOsMeetingMetadataProvider : + IMeetingMetadataProvider, + IMeetingMetadataDiagnosticProvider +{ + private readonly IMacOsCalendarClient client; + private readonly ILogger logger; + + public MacOsMeetingMetadataProvider( + IMacOsCalendarClient client, + ILogger logger) + { + this.client = client; + this.logger = logger; + } + + public async Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + try + { + return await GetCurrentMeetingCoreAsync(startedAt, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger.LogDebug(exception, "macOS calendar meeting metadata was not available"); + return null; + } + } + + public async Task DiagnoseCurrentMeetingAsync( + DateTimeOffset startedAt, + TimeSpan timeout, + CancellationToken cancellationToken) + { + return await MeetingMetadataDiagnosticRunner.RunAsync( + nameof(MacOsMeetingMetadataProvider), + startedAt, + timeout, + token => GetCurrentMeetingCoreAsync(startedAt, token), + "No matching Teams calendar event was found.", + cancellationToken); + } + + private async Task GetCurrentMeetingCoreAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + var events = await client.GetCalendarEventsAsync( + startedAt.AddHours(-4), + startedAt.AddMinutes(5), + cancellationToken); + var candidates = events + .Where(MacOsCalendarEventMapper.IsTeamsEvent) + .Where(calendarEvent => !calendarEvent.IsCanceled) + .ToList(); + var selected = OutlookMeetingCandidateSelector.Select( + candidates, + startedAt.LocalDateTime, + calendarEvent => calendarEvent.Start.LocalDateTime, + calendarEvent => calendarEvent.End.LocalDateTime); + return selected is null ? null : MacOsCalendarEventMapper.CreateMetadata(selected); + } +} + +internal sealed class MacOsCalendarMeetingProvider : ICalendarMeetingProvider +{ + private readonly IMacOsCalendarClient client; + private readonly ILogger logger; + + public MacOsCalendarMeetingProvider( + IMacOsCalendarClient client, + ILogger logger) + { + this.client = client; + this.logger = logger; + } + + public async Task> GetRecordingPromptCandidatesForDayAsync( + DateOnly day, + CancellationToken cancellationToken) + { + var dayStart = AtLocalMidnight(day); + var dayEnd = AtLocalMidnight(day.AddDays(1)); + try + { + var events = await client.GetCalendarEventsAsync(dayStart, dayEnd, cancellationToken); + return events + .Where(calendarEvent => !calendarEvent.IsCanceled) + .Where(MacOsCalendarEventMapper.IsTeamsEvent) + .Where(calendarEvent => calendarEvent.Start >= dayStart && calendarEvent.Start < dayEnd) + .OrderBy(calendarEvent => calendarEvent.Start) + .Select(calendarEvent => new CalendarMeeting( + calendarEvent.Id, + CalendarAppointmentMetadata.ReadSubject(calendarEvent.Subject), + calendarEvent.Start, + calendarEvent.End, + MacOsCalendarEventMapper.CreateMetadata(calendarEvent), + calendarEvent.IsCanceled)) + .ToList(); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger.LogDebug(exception, "macOS calendar meetings were not available"); + return []; + } + } + + private static DateTimeOffset AtLocalMidnight(DateOnly day) + { + var localMidnight = day.ToDateTime(TimeOnly.MinValue); + return new DateTimeOffset( + localMidnight, + TimeZoneInfo.Local.GetUtcOffset(localMidnight)); + } +} + +internal sealed class MacOsMeetingStartPromptService : IMeetingStartPromptService +{ + private readonly IMacOsMeetingPromptClient client; + private readonly ILogger logger; + + public MacOsMeetingStartPromptService( + IMacOsMeetingPromptClient client, + ILogger logger) + { + this.client = client; + this.logger = logger; + } + + public async Task ShowPromptAsync( + MeetingStartPromptRequest request, + Func handleResponseAsync, + CancellationToken cancellationToken) + { + try + { + var response = await client.ShowMeetingStartPromptAsync(request, cancellationToken); + await handleResponseAsync(response, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger.LogError(exception, "Failed to display native macOS meeting-start prompt"); + } + } +} + +public static class MacOsMeetingIntegrationRegistration +{ + public static IServiceCollection Add( + IServiceCollection services, + bool isMacOs) + { + if (isMacOs) + { + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(services => + services.GetRequiredService()); + services.TryAddSingleton(services => + services.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(services => + services.GetRequiredService()); + services.AddSingleton(services => + services.GetRequiredService()); + return services; + } + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + return services; + } +} + +internal static class MacOsCalendarEventMapper +{ + public static bool IsTeamsEvent(MacOsCalendarEvent calendarEvent) + { + return CalendarAppointmentMetadata.IsTeamsAppointment( + calendarEvent.Subject, + calendarEvent.Location, + calendarEvent.Notes, + calendarEvent.Url); + } + + public static MeetingMetadata CreateMetadata(MacOsCalendarEvent calendarEvent) + { + return CalendarAppointmentMetadata.Create( + calendarEvent.Subject, + calendarEvent.Attendees, + calendarEvent.Notes, + calendarEvent.End); + } +} diff --git a/MeetingAssistant/MacOs/MacOsWorkflowRulesEditorWindowService.cs b/MeetingAssistant/MacOs/MacOsWorkflowRulesEditorWindowService.cs new file mode 100644 index 0000000..476b6f8 --- /dev/null +++ b/MeetingAssistant/MacOs/MacOsWorkflowRulesEditorWindowService.cs @@ -0,0 +1,161 @@ +using System.Diagnostics; +using MeetingAssistant.Workflow; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.MacOs; + +public sealed class MacOsWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService +{ + private readonly IOptions options; + + public MacOsWorkflowRulesEditorWindowService(IOptions options) + { + this.options = options; + } + + public void Show() + { + if (!OperatingSystem.IsMacOS()) + { + return; + } + + var url = $"{options.Value.Api.PublicBaseUrl.TrimEnd('/')}/diagnostics/settings-and-logs"; + Process.Start(new ProcessStartInfo + { + FileName = "/usr/bin/open", + ArgumentList = { url }, + UseShellExecute = false + }); + } +} + +public sealed class MacOsWorkflowRulesEditorSession +{ + private readonly WorkflowRulesEditorChatViewModel viewModel; + private readonly SemaphoreSlim sendLock = new(1, 1); + + public MacOsWorkflowRulesEditorSession(WorkflowRulesEditorChatViewModel viewModel) + { + this.viewModel = viewModel; + } + + public async Task SendAsync( + string? message, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(message)) + { + return Snapshot(); + } + + await sendLock.WaitAsync(cancellationToken); + try + { + viewModel.Draft = message; + await viewModel.SendAsync(cancellationToken); + return Snapshot(); + } + finally + { + sendLock.Release(); + } + } + + private MacOsWorkflowRulesEditorChatResponse Snapshot() + { + return new MacOsWorkflowRulesEditorChatResponse( + viewModel.Messages + .Where(item => item.Kind == WorkflowRulesEditorConversationItemKind.Message) + .Select(item => new MacOsWorkflowRulesEditorMessage( + item.Message!.Role.ToString().ToLowerInvariant(), + item.Message.Content)) + .ToArray()); + } +} + +public sealed record MacOsWorkflowRulesEditorChatRequest(string? Message); + +public sealed record MacOsWorkflowRulesEditorChatResponse( + IReadOnlyList Messages); + +public sealed record MacOsWorkflowRulesEditorMessage(string Role, string Content); + +public static class MacOsWorkflowRulesEditorPage +{ + public const string Html = """ + + + + + + Meeting Summary Agent + + + +
+

Meeting Summary Agent

+
+
+ + +
+
+ + + + """; +} diff --git a/MeetingAssistant/MeetingAssistant.csproj b/MeetingAssistant/MeetingAssistant.csproj index 416b61e..6250804 100644 --- a/MeetingAssistant/MeetingAssistant.csproj +++ b/MeetingAssistant/MeetingAssistant.csproj @@ -15,6 +15,27 @@ true + + true + $(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/main.swift + $(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/Info.plist + Native/MeetingAssistantAudioCapture.app + $(MacOsAudioCaptureBundlePath)/Contents/MacOS/macos-meeting-audio-capture + $(RuntimeIdentifier) + $(NETCoreSdkRuntimeIdentifier) + x86_64 + arm64 + $(MSBuildProjectDirectory)/Native/MacOsDesktopControls/main.swift + Native/macos-desktop-controls + $(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/main.swift + $(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/Info.plist + Native/macos-meeting-integrations + $(MSBuildProjectDirectory)/Native/MacOsLauncher/main.swift + $(MSBuildProjectDirectory)/Native/MacOsLauncher/Info.plist + Native/macos-meeting-assistant-launcher + MeetingAssistant.app + + @@ -47,6 +68,7 @@ + @@ -61,6 +83,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/MeetingAssistant/MeetingNotes/CalendarAppointmentMetadata.cs b/MeetingAssistant/MeetingNotes/CalendarAppointmentMetadata.cs new file mode 100644 index 0000000..727f0bd --- /dev/null +++ b/MeetingAssistant/MeetingNotes/CalendarAppointmentMetadata.cs @@ -0,0 +1,117 @@ +using MeetingAssistant.Calendar; + +namespace MeetingAssistant.MeetingNotes; + +internal static class CalendarAppointmentMetadata +{ + public static MeetingMetadata Create( + string? subject, + IEnumerable attendees, + string? body, + DateTimeOffset end) + { + return new MeetingMetadata( + ReadSubject(subject), + NormalizeAttendees(attendees), + ExtractAgenda(body ?? ""), + end); + } + + public static string ReadSubject(string? subject) + { + var trimmed = subject?.Trim(); + return string.IsNullOrWhiteSpace(trimmed) ? "Teams meeting" : trimmed; + } + + public static bool IsTeamsAppointment( + string? subject, + string? location, + string? body, + string? url = null) + { + return TeamsMeetingMarkerDetector.IsTeamsAppointment( + subject ?? "", + location ?? "", + string.Join(Environment.NewLine, body, url)); + } + + public static string ExtractAgenda(string body) + { + if (string.IsNullOrWhiteSpace(body)) + { + return ""; + } + + var lines = body + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n'); + var agendaLines = new List(); + foreach (var line in lines) + { + if (IsTeamsSeparator(line) || TeamsMeetingMarkerDetector.ContainsTeamsMarker(line)) + { + break; + } + + agendaLines.Add(line); + } + + return string.Join(Environment.NewLine, agendaLines).Trim(); + } + + public static IReadOnlyList NormalizeAttendees(IEnumerable attendees) + { + return attendees + .Select(NormalizeAttendee) + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase) + .Select(group => group + .OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal)) + .ThenBy(attendee => attendee.Length) + .First()) + .ToList(); + } + + private static bool IsTeamsSeparator(string line) + { + var trimmed = line.Trim(); + return trimmed.Length >= 8 && + trimmed.All(character => character is '_' or '-' or '*' or ' '); + } + + private static string NormalizeAttendee(string attendee) + { + var normalized = attendee.Trim() + .Replace("mailto:", "", StringComparison.OrdinalIgnoreCase); + while (normalized.Contains(" ", StringComparison.Ordinal)) + { + normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); + } + + var emailStart = normalized.IndexOf('<', StringComparison.Ordinal); + var emailEnd = normalized.LastIndexOf('>'); + if (emailStart > 0 && emailEnd > emailStart) + { + var name = normalized[..emailStart].Trim(); + var email = normalized[(emailStart + 1)..emailEnd].Trim(); + if (name.Equals(email, StringComparison.OrdinalIgnoreCase)) + { + return email; + } + + return $"{name} <{email}>"; + } + + return normalized; + } + + private static string GetAttendeeDeduplicationKey(string attendee) + { + var emailStart = attendee.IndexOf('<', StringComparison.Ordinal); + var emailEnd = attendee.LastIndexOf('>'); + return emailStart > 0 && emailEnd > emailStart + ? attendee[(emailStart + 1)..emailEnd].Trim() + : attendee.Trim(); + } +} diff --git a/MeetingAssistant/MeetingNotes/MeetingMetadataDiagnosticRunner.cs b/MeetingAssistant/MeetingNotes/MeetingMetadataDiagnosticRunner.cs new file mode 100644 index 0000000..eb79663 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingMetadataDiagnosticRunner.cs @@ -0,0 +1,58 @@ +using System.Diagnostics; + +namespace MeetingAssistant.MeetingNotes; + +internal static class MeetingMetadataDiagnosticRunner +{ + public static async Task RunAsync( + string provider, + DateTimeOffset startedAt, + TimeSpan timeout, + Func> lookupAsync, + string noMatchError, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + try + { + var lookup = lookupAsync(cancellationToken); + var metadata = timeout > TimeSpan.Zero + ? await lookup.WaitAsync(timeout, cancellationToken) + : await lookup.WaitAsync(cancellationToken); + return new MeetingMetadataDiagnosticResult( + provider, + startedAt, + metadata is not null, + false, + stopwatch.ElapsedMilliseconds, + metadata, + metadata is null ? noMatchError : null); + } + catch (TimeoutException exception) + { + return new MeetingMetadataDiagnosticResult( + provider, + startedAt, + false, + true, + stopwatch.ElapsedMilliseconds, + null, + exception.Message); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + return new MeetingMetadataDiagnosticResult( + provider, + startedAt, + false, + false, + stopwatch.ElapsedMilliseconds, + null, + exception.Message); + } + } +} diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs index 84f0b01..83cf59b 100644 --- a/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs +++ b/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs @@ -7,18 +7,17 @@ internal static class OutlookClassicAppointmentMetadata { public static MeetingMetadata CreateMetadata(object appointment, DateTime end) { - var title = ReadSubject(appointment); - return new MeetingMetadata( - title, + return CalendarAppointmentMetadata.Create( + Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")), ReadAttendees(appointment), - ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""), + Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")), OutlookClassicCom.ToLocalOffset(end)); } public static string ReadSubject(object appointment) { - var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim(); - return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject; + return CalendarAppointmentMetadata.ReadSubject( + Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))); } public static bool IsTeamsAppointment(object appointment) @@ -26,29 +25,12 @@ internal static class OutlookClassicAppointmentMetadata var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? ""; var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? ""; var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""; - return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body); + return CalendarAppointmentMetadata.IsTeamsAppointment(subject, location, body); } public static string ExtractAgenda(string body) { - if (string.IsNullOrWhiteSpace(body)) - { - return ""; - } - - var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'); - var agendaLines = new List(); - foreach (var line in lines) - { - if (IsTeamsSeparator(line) || IsTeamsJoinLine(line)) - { - break; - } - - agendaLines.Add(line); - } - - return string.Join(Environment.NewLine, agendaLines).Trim(); + return CalendarAppointmentMetadata.ExtractAgenda(body); } public static IReadOnlyList ReadAttendees(object appointment) @@ -93,66 +75,7 @@ internal static class OutlookClassicAppointmentMetadata public static IReadOnlyList NormalizeAttendees(IEnumerable attendees) { - return attendees - .Select(NormalizeAttendee) - .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) - .GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase) - .Select(group => group - .OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal)) - .ThenBy(attendee => attendee.Length) - .First()) - .ToList(); - } - - private static bool IsTeamsSeparator(string line) - { - var trimmed = line.Trim(); - return trimmed.Length >= 8 && - trimmed.All(character => character is '_' or '-' or '*' or ' '); - } - - private static bool IsTeamsJoinLine(string line) - { - return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line); - } - - private static string NormalizeAttendee(string attendee) - { - var normalized = attendee.Trim(); - var mailtoPrefix = "mailto:"; - normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase); - while (normalized.Contains(" ", StringComparison.Ordinal)) - { - normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); - } - - var emailStart = normalized.IndexOf('<', StringComparison.Ordinal); - var emailEnd = normalized.LastIndexOf('>'); - if (emailStart > 0 && emailEnd > emailStart) - { - var name = normalized[..emailStart].Trim(); - var email = normalized[(emailStart + 1)..emailEnd].Trim(); - if (name.Equals(email, StringComparison.OrdinalIgnoreCase)) - { - return email; - } - - return $"{name} <{email}>"; - } - - return normalized; - } - - private static string GetAttendeeDeduplicationKey(string attendee) - { - var emailStart = attendee.IndexOf('<', StringComparison.Ordinal); - var emailEnd = attendee.LastIndexOf('>'); - if (emailStart > 0 && emailEnd > emailStart) - { - return attendee[(emailStart + 1)..emailEnd].Trim(); - } - - return attendee.Trim(); + return CalendarAppointmentMetadata.NormalizeAttendees(attendees); } private static string FormatRecipient(object recipient) diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs index b261632..8e02563 100644 --- a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs +++ b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using MeetingAssistant.Calendar; namespace MeetingAssistant.MeetingNotes; @@ -36,54 +35,16 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv TimeSpan timeout, CancellationToken cancellationToken) { - var stopwatch = Stopwatch.StartNew(); - try - { - var lookup = OutlookClassicCom.RunStaAsync( + return await MeetingMetadataDiagnosticRunner.RunAsync( + nameof(OutlookClassicMeetingMetadataProvider), + startedAt, + timeout, + token => OutlookClassicCom.RunStaAsync( () => GetCurrentMeeting(startedAt), - cancellationToken, - "Meeting Assistant Outlook COM Lookup"); - var metadata = timeout > TimeSpan.Zero - ? await lookup.WaitAsync(timeout, cancellationToken) - : await lookup.WaitAsync(cancellationToken); - stopwatch.Stop(); - return new MeetingMetadataDiagnosticResult( - nameof(OutlookClassicMeetingMetadataProvider), - startedAt, - metadata is not null, - false, - stopwatch.ElapsedMilliseconds, - metadata, - metadata is null ? "No matching Teams appointment was found." : null); - } - catch (TimeoutException exception) - { - stopwatch.Stop(); - return new MeetingMetadataDiagnosticResult( - nameof(OutlookClassicMeetingMetadataProvider), - startedAt, - false, - true, - stopwatch.ElapsedMilliseconds, - null, - exception.Message); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception exception) - { - stopwatch.Stop(); - return new MeetingMetadataDiagnosticResult( - nameof(OutlookClassicMeetingMetadataProvider), - startedAt, - false, - false, - stopwatch.ElapsedMilliseconds, - null, - exception.Message); - } + token, + "Meeting Assistant Outlook COM Lookup"), + "No matching Teams appointment was found.", + cancellationToken); } private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt) diff --git a/MeetingAssistant/Native/MacOsDesktopControls/main.swift b/MeetingAssistant/Native/MacOsDesktopControls/main.swift new file mode 100644 index 0000000..5930dbc --- /dev/null +++ b/MeetingAssistant/Native/MacOsDesktopControls/main.swift @@ -0,0 +1,289 @@ +import AppKit +import Carbon +import Foundation +import WebKit + +struct Manifest: Decodable { + let baseUrl: String + let statusUrl: String + let openAgentUrl: String + let profiles: [Profile] + let hotkeys: [Hotkey] +} + +struct Profile: Decodable { + let name: String + let toggleShortcut: String +} + +struct Hotkey: Decodable { + let shortcut: String + let endpoint: String +} + +struct RecordingStatus: Decodable { + let isRecording: Bool + let state: String + let launchProfile: String? +} + +@main +final class DesktopControls: NSObject, NSApplicationDelegate, NSMenuDelegate { + private let manifest: Manifest + private var status = RecordingStatus(isRecording: false, state: "Idle", launchProfile: nil) + private var statusItem: NSStatusItem! + private var registeredHotkeys: [UInt32: Hotkey] = [:] + private var timer: Timer? + private var agentWindow: NSWindow? + + override init() { + guard CommandLine.arguments.count == 2, + let data = try? Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[1])), + let decoded = try? JSONDecoder().decode(Manifest.self, from: data) else { + fputs("Usage: macos-desktop-controls \n", stderr) + exit(2) + } + manifest = decoded + super.init() + } + + static func main() { + let application = NSApplication.shared + let delegate = DesktopControls() + application.delegate = delegate + application.setActivationPolicy(.accessory) + application.run() + } + + func applicationDidFinishLaunching(_ notification: Notification) { + statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.squareLength) + statusItem.button?.image = NSImage( + systemSymbolName: "waveform.circle", + accessibilityDescription: "Meeting Assistant") + let menu = NSMenu() + menu.delegate = self + statusItem.menu = menu + registerHotkeys() + refreshStatus() + timer = Timer.scheduledTimer(withTimeInterval: 2, repeats: true) { [weak self] _ in + self?.refreshStatus() + } + } + + func applicationWillTerminate(_ notification: Notification) { + timer?.invalidate() + } + + func menuWillOpen(_ menu: NSMenu) { + rebuildMenu(menu) + refreshStatus() + } + + private func rebuildMenu(_ menu: NSMenu) { + menu.removeAllItems() + let stateItem = NSMenuItem(title: tooltip, action: nil, keyEquivalent: "") + stateItem.isEnabled = false + menu.addItem(stateItem) + menu.addItem(.separator()) + menu.addItem(item("Open agent", #selector(openAgent))) + + if status.isRecording { + menu.addItem(item("Stop meeting recording and transcribe", #selector(stopRecording))) + menu.addItem(item("Cancel meeting recording and discard", #selector(abortRecording))) + } else { + for (index, profile) in manifest.profiles.enumerated() { + let entry = item( + "Start meeting recording (\(profile.name))", + #selector(startProfile(_:))) + entry.tag = index + entry.keyEquivalent = menuKey(from: profile.toggleShortcut) + entry.keyEquivalentModifierMask = menuModifiers(from: profile.toggleShortcut) + menu.addItem(entry) + } + } + menu.addItem(.separator()) + menu.addItem(item("Exit", #selector(exitApplication))) + } + + private var tooltip: String { + if status.isRecording { + return status.launchProfile.map { "Meeting Assistant — recording (\($0))" } + ?? "Meeting Assistant — recording" + } + return status.state.lowercased() == "summarizing" + ? "Meeting Assistant — summarizing" + : "Meeting Assistant — idle" + } + + private func item(_ title: String, _ action: Selector) -> NSMenuItem { + let result = NSMenuItem(title: title, action: action, keyEquivalent: "") + result.target = self + return result + } + + @objc private func openAgent() { + if agentWindow == nil { + let webView = WKWebView(frame: .zero) + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 840, height: 760), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, + defer: false) + window.title = "Meeting Summary Agent" + window.contentView = webView + window.center() + window.isReleasedWhenClosed = false + webView.load(URLRequest(url: URL(string: manifest.openAgentUrl)!)) + agentWindow = window + } + NSApplication.shared.activate(ignoringOtherApps: true) + agentWindow?.makeKeyAndOrderFront(nil) + } + + @objc private func startProfile(_ sender: NSMenuItem) { + let profile = manifest.profiles[sender.tag] + post(path: "/profiles/\(escaped(profile.name))/recording/start") + } + + @objc private func stopRecording() { + post(path: "/recording/stop") + } + + @objc private func abortRecording() { + post(path: "/recording/abort") + } + + @objc private func exitApplication() { + post(path: "/application/exit") + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + NSApplication.shared.terminate(nil) + } + } + + private func refreshStatus() { + guard let url = URL(string: manifest.statusUrl) else { return } + URLSession.shared.dataTask(with: url) { [weak self] data, _, _ in + guard let data, + let next = try? JSONDecoder().decode(RecordingStatus.self, from: data) else { return } + DispatchQueue.main.async { + self?.status = next + self?.statusItem.button?.image = NSImage( + systemSymbolName: next.isRecording ? "record.circle.fill" : "waveform.circle", + accessibilityDescription: self?.tooltip) + } + }.resume() + } + + private func post(path: String) { + post(url: manifest.baseUrl + path) + } + + private func post(url value: String) { + guard let url = URL(string: value) else { return } + var request = URLRequest(url: url) + request.httpMethod = "POST" + URLSession.shared.dataTask(with: request) { [weak self] _, _, _ in + DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { + self?.refreshStatus() + } + }.resume() + } + + private func registerHotkeys() { + var eventType = EventTypeSpec( + eventClass: OSType(kEventClassKeyboard), + eventKind: OSType(kEventHotKeyPressed)) + InstallEventHandler( + GetApplicationEventTarget(), + { _, event, context in + guard let event, let context else { return noErr } + var identifier = EventHotKeyID() + GetEventParameter( + event, + EventParamName(kEventParamDirectObject), + EventParamType(typeEventHotKeyID), + nil, + MemoryLayout.size, + nil, + &identifier) + let owner = Unmanaged.fromOpaque(context).takeUnretainedValue() + if let hotkey = owner.registeredHotkeys[identifier.id] { + owner.post(path: hotkey.endpoint) + } + return noErr + }, + 1, + &eventType, + Unmanaged.passUnretained(self).toOpaque(), + nil) + + for (index, hotkey) in manifest.hotkeys.enumerated() { + guard let keyCode = carbonKeyCode(from: hotkey.shortcut) else { continue } + let identifier = EventHotKeyID(signature: fourCC("MTGA"), id: UInt32(index + 1)) + var reference: EventHotKeyRef? + let result = RegisterEventHotKey( + keyCode, + carbonModifiers(from: hotkey.shortcut), + identifier, + GetApplicationEventTarget(), + 0, + &reference) + if result == noErr { + registeredHotkeys[identifier.id] = hotkey + } + } + } + + private func escaped(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? value + } +} + +private func tokens(_ shortcut: String) -> [String] { + shortcut.uppercased().split(separator: "+").map(String.init) +} + +private func carbonModifiers(from shortcut: String) -> UInt32 { + let parts = tokens(shortcut) + var result: UInt32 = 0 + if parts.contains("CTRL") || parts.contains("CONTROL") { result |= UInt32(controlKey) } + if parts.contains("ALT") || parts.contains("OPTION") { result |= UInt32(optionKey) } + if parts.contains("SHIFT") { result |= UInt32(shiftKey) } + if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") { + result |= UInt32(cmdKey) + } + return result +} + +private func menuModifiers(from shortcut: String) -> NSEvent.ModifierFlags { + let parts = tokens(shortcut) + var result: NSEvent.ModifierFlags = [] + if parts.contains("CTRL") || parts.contains("CONTROL") { result.insert(.control) } + if parts.contains("ALT") || parts.contains("OPTION") { result.insert(.option) } + if parts.contains("SHIFT") { result.insert(.shift) } + if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") { + result.insert(.command) + } + return result +} + +private func menuKey(from shortcut: String) -> String { + tokens(shortcut).last?.lowercased() ?? "" +} + +private func carbonKeyCode(from shortcut: String) -> UInt32? { + guard let key = tokens(shortcut).last, key.count == 1, let scalar = key.unicodeScalars.first else { + return nil + } + let codes: [Character: UInt32] = [ + "A": 0, "S": 1, "D": 2, "F": 3, "H": 4, "G": 5, "Z": 6, "X": 7, + "C": 8, "V": 9, "B": 11, "Q": 12, "W": 13, "E": 14, "R": 15, + "Y": 16, "T": 17, "M": 46, "L": 37, "N": 45, "P": 35, "O": 31, + "I": 34, "U": 32, "J": 38, "K": 40 + ] + return codes[Character(String(scalar))] +} + +private func fourCC(_ value: String) -> OSType { + value.utf8.reduce(0) { ($0 << 8) + OSType($1) } +} diff --git a/MeetingAssistant/Native/MacOsLauncher/Info.plist b/MeetingAssistant/Native/MacOsLauncher/Info.plist new file mode 100644 index 0000000..31e6bcd --- /dev/null +++ b/MeetingAssistant/Native/MacOsLauncher/Info.plist @@ -0,0 +1,24 @@ + + + + + CFBundleExecutable + MeetingAssistant + CFBundleIdentifier + cloud.schweigert.meeting-assistant + CFBundleName + Meeting Assistant + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSUIElement + + NSCalendarsFullAccessUsageDescription + Meeting Assistant reads calendar metadata for meeting notes and recording prompts. + NSMicrophoneUsageDescription + Meeting Assistant records microphone audio for live meeting transcription. + + diff --git a/MeetingAssistant/Native/MacOsLauncher/main.swift b/MeetingAssistant/Native/MacOsLauncher/main.swift new file mode 100644 index 0000000..51f5476 --- /dev/null +++ b/MeetingAssistant/Native/MacOsLauncher/main.swift @@ -0,0 +1,70 @@ +import Darwin +import Dispatch +import Foundation + +@main +private struct MeetingAssistantLauncher { + private static let serviceDirectoryName = "MeetingAssistant" + + static func main() { + let runtimeDirectory = Bundle.main.bundleURL + .appendingPathComponent("Contents/Resources/app", isDirectory: true) + let assemblyPath = runtimeDirectory.appendingPathComponent("MeetingAssistant.dll").path + let environmentFile = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support") + .appendingPathComponent(serviceDirectoryName) + .appendingPathComponent("config/meeting-assistant.env") + + guard FileManager.default.fileExists(atPath: assemblyPath) else { + fail("Meeting Assistant runtime not found at \(assemblyPath)") + } + + guard FileManager.default.fileExists(atPath: environmentFile.path) else { + fail("Meeting Assistant environment file not found at \(environmentFile.path)") + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/zsh") + process.arguments = [ + "-c", + "set -a; source \"$1\"; set +a; exec /usr/local/share/dotnet/dotnet \"$2\" --contentRoot \"$3\"", + "meeting-assistant-launcher", + environmentFile.path, + assemblyPath, + runtimeDirectory.path + ] + process.currentDirectoryURL = runtimeDirectory + + signal(SIGTERM, SIG_IGN) + signal(SIGINT, SIG_IGN) + + let terminationSignal = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .global()) + terminationSignal.setEventHandler { + if process.isRunning { + process.terminate() + } + } + terminationSignal.resume() + + let interruptSignal = DispatchSource.makeSignalSource(signal: SIGINT, queue: .global()) + interruptSignal.setEventHandler { + if process.isRunning { + kill(process.processIdentifier, SIGINT) + } + } + interruptSignal.resume() + + do { + try process.run() + process.waitUntilExit() + Darwin.exit(process.terminationStatus) + } catch { + fail("Meeting Assistant launcher failed: \(error)") + } + } + + private static func fail(_ message: String) -> Never { + FileHandle.standardError.write(Data("\(message)\n".utf8)) + Darwin.exit(1) + } +} diff --git a/MeetingAssistant/Native/MacOsMeetingAudioCapture/Info.plist b/MeetingAssistant/Native/MacOsMeetingAudioCapture/Info.plist new file mode 100644 index 0000000..bfaccee --- /dev/null +++ b/MeetingAssistant/Native/MacOsMeetingAudioCapture/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleExecutable + macos-meeting-audio-capture + CFBundleIdentifier + cloud.schweigert.meeting-assistant.audio-capture + CFBundleName + Meeting Assistant Audio Capture + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSUIElement + + NSMicrophoneUsageDescription + Meeting Assistant records microphone audio for live meeting transcription. + + diff --git a/MeetingAssistant/Native/MacOsMeetingAudioCapture/main.swift b/MeetingAssistant/Native/MacOsMeetingAudioCapture/main.swift new file mode 100644 index 0000000..71f46a7 --- /dev/null +++ b/MeetingAssistant/Native/MacOsMeetingAudioCapture/main.swift @@ -0,0 +1,328 @@ +import AVFoundation +import CoreMedia +import Darwin +import Foundation +import ScreenCaptureKit + +private enum CaptureFailure: Error, CustomStringConvertible { + case usage + case invalidFormat + case microphonePermission + case noDisplay + case audioConverter(String) + case systemAudio(String) + + var description: String { + switch self { + case .usage: + return "Usage: macos-meeting-audio-capture " + case .invalidFormat: + return "Sample rate must be positive and channels must be 1 or 2." + case .microphonePermission: + return "Microphone permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Microphone." + case .noDisplay: + return "ScreenCaptureKit could not find a display for system-audio capture." + case .audioConverter(let detail): + return "Native PCM conversion failed: \(detail)" + case .systemAudio(let detail): + return "Screen Recording/System Audio permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Screen & System Audio Recording. \(detail)" + } + } +} + +private final class ErrorReporter: @unchecked Sendable { + private let lock = NSLock() + + func write(_ message: String) { + lock.lock() + defer { lock.unlock() } + let line = message.hasSuffix("\n") ? message : message + "\n" + if let data = line.data(using: .utf8) { + try? FileHandle.standardError.write(contentsOf: data) + } + } + + func exit(with failure: CaptureFailure, code: Int32) -> Never { + write(failure.description) + Darwin.exit(code) + } +} + +private final class PcmWriter: @unchecked Sendable { + private let targetFormat: AVAudioFormat + private let output = FileHandle.standardOutput + private let reporter: ErrorReporter + private let lock = NSLock() + private var converter: AVAudioConverter? + + init(sampleRate: Int, channels: Int, reporter: ErrorReporter) throws { + guard let format = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: Double(sampleRate), + channels: AVAudioChannelCount(channels), + interleaved: true + ) else { + throw CaptureFailure.invalidFormat + } + + targetFormat = format + self.reporter = reporter + } + + func write(_ input: AVAudioPCMBuffer) { + lock.lock() + defer { lock.unlock() } + + do { + let outputBuffer = try convert(input) + let buffers = UnsafeMutableAudioBufferListPointer(outputBuffer.mutableAudioBufferList) + guard let audioBuffer = buffers.first, let bytes = audioBuffer.mData else { + return + } + + let data = Data(bytes: bytes, count: Int(audioBuffer.mDataByteSize)) + try output.write(contentsOf: data) + } catch { + reporter.exit(with: .audioConverter(error.localizedDescription), code: 70) + } + } + + private func convert(_ input: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer { + let sourceFormat = input.format + let activeConverter: AVAudioConverter + if let converter { + activeConverter = converter + } else { + guard let created = AVAudioConverter(from: sourceFormat, to: targetFormat) else { + throw CaptureFailure.audioConverter("AVAudioConverter could not convert \(sourceFormat) to \(targetFormat).") + } + converter = created + activeConverter = created + } + + let ratio = targetFormat.sampleRate / sourceFormat.sampleRate + let capacity = max(1, AVAudioFrameCount(ceil(Double(input.frameLength) * ratio)) + 32) + guard let converted = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else { + throw CaptureFailure.audioConverter("Could not allocate the target PCM buffer.") + } + + var suppliedInput = false + var conversionError: NSError? + let status = activeConverter.convert(to: converted, error: &conversionError) { _, inputStatus in + if suppliedInput { + inputStatus.pointee = .noDataNow + return nil + } + + suppliedInput = true + inputStatus.pointee = .haveData + return input + } + + if status == .error { + throw conversionError ?? CaptureFailure.audioConverter("AVAudioConverter returned an unknown error.") + } + + return converted + } +} + +private final class MicrophoneCapture: @unchecked Sendable { + private let engine = AVAudioEngine() + private let writer: PcmWriter + + init(writer: PcmWriter) { + self.writer = writer + } + + func run() async throws -> Never { + let authorized = await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { granted in + continuation.resume(returning: granted) + } + } + guard authorized else { + throw CaptureFailure.microphonePermission + } + + let input = engine.inputNode + let inputFormat = input.outputFormat(forBus: 0) + input.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [writer] buffer, _ in + writer.write(buffer) + } + engine.prepare() + try engine.start() + + while true { + try await Task.sleep(for: .seconds(3_600)) + } + } +} + +private final class SystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable { + private let writer: PcmWriter + private let reporter: ErrorReporter + private let sampleQueue = DispatchQueue(label: "meeting-assistant.system-audio") + private var stream: SCStream? + + init(writer: PcmWriter, reporter: ErrorReporter) { + self.writer = writer + self.reporter = reporter + } + + func run(sampleRate: Int, channels: Int) async throws -> Never { + let content: SCShareableContent + do { + content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: false + ) + } catch { + throw CaptureFailure.systemAudio(error.localizedDescription) + } + + guard let display = content.displays.first else { + throw CaptureFailure.noDisplay + } + + let filter = SCContentFilter( + display: display, + excludingApplications: [], + exceptingWindows: [] + ) + let configuration = SCStreamConfiguration() + configuration.capturesAudio = true + configuration.excludesCurrentProcessAudio = false + configuration.sampleRate = sampleRate + configuration.channelCount = channels + configuration.width = 2 + configuration.height = 2 + configuration.showsCursor = false + configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1) + + let captureStream = SCStream(filter: filter, configuration: configuration, delegate: self) + try captureStream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue) + do { + try await captureStream.startCapture() + } catch { + throw CaptureFailure.systemAudio(error.localizedDescription) + } + stream = captureStream + + while true { + try await Task.sleep(for: .seconds(3_600)) + } + } + + func stream( + _ stream: SCStream, + didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of outputType: SCStreamOutputType + ) { + guard outputType == .audio, + sampleBuffer.isValid, + CMSampleBufferDataIsReady(sampleBuffer), + let description = CMSampleBufferGetFormatDescription(sampleBuffer), + let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description) + else { + return + } + + let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer)) + guard frameCount > 0, + let sourceFormat = AVAudioFormat(streamDescription: streamDescription) + else { + return + } + + var bufferListSize = 0 + var retainedBlockBuffer: CMBlockBuffer? + var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: &bufferListSize, + bufferListOut: nil, + bufferListSize: 0, + blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, + flags: 0, + blockBufferOut: &retainedBlockBuffer + ) + guard status == noErr else { + reporter.exit( + with: .audioConverter("CoreMedia could not describe system audio (OSStatus \(status))."), + code: 71 + ) + } + + let rawBufferList = UnsafeMutableRawPointer.allocate( + byteCount: bufferListSize, + alignment: MemoryLayout.alignment + ) + defer { rawBufferList.deallocate() } + let audioBufferList = rawBufferList.bindMemory(to: AudioBufferList.self, capacity: 1) + status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: nil, + bufferListOut: audioBufferList, + bufferListSize: bufferListSize, + blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, + flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment), + blockBufferOut: &retainedBlockBuffer + ) + guard status == noErr, + let buffer = AVAudioPCMBuffer( + pcmFormat: sourceFormat, + bufferListNoCopy: audioBufferList, + deallocator: nil + ) + else { + reporter.exit( + with: .audioConverter("CoreMedia could not expose system audio (OSStatus \(status))."), + code: 71 + ) + } + + buffer.frameLength = frameCount + writer.write(buffer) + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + reporter.exit(with: .systemAudio(error.localizedDescription), code: 72) + } +} + +@main +private struct MacOsMeetingAudioCapture { + static func main() async { + let reporter = ErrorReporter() + do { + let arguments = CommandLine.arguments + guard arguments.count == 4, + let sampleRate = Int(arguments[2]), + let channels = Int(arguments[3]) + else { + throw CaptureFailure.usage + } + guard sampleRate > 0, channels == 1 || channels == 2 else { + throw CaptureFailure.invalidFormat + } + + let writer = try PcmWriter(sampleRate: sampleRate, channels: channels, reporter: reporter) + switch arguments[1] { + case "microphone": + try await MicrophoneCapture(writer: writer).run() + case "system": + try await SystemAudioCapture(writer: writer, reporter: reporter) + .run(sampleRate: sampleRate, channels: channels) + default: + throw CaptureFailure.usage + } + } catch let failure as CaptureFailure { + reporter.exit(with: failure, code: 64) + } catch { + reporter.write(error.localizedDescription) + Darwin.exit(1) + } + } +} diff --git a/MeetingAssistant/Native/MacOsMeetingIntegrations/Info.plist b/MeetingAssistant/Native/MacOsMeetingIntegrations/Info.plist new file mode 100644 index 0000000..898041d --- /dev/null +++ b/MeetingAssistant/Native/MacOsMeetingIntegrations/Info.plist @@ -0,0 +1,14 @@ + + + + + CFBundleIdentifier + cloud.schweigert.meeting-assistant.integrations + CFBundleName + Meeting Assistant Integrations + NSCalendarsUsageDescription + Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts. + NSCalendarsFullAccessUsageDescription + Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts. + + diff --git a/MeetingAssistant/Native/MacOsMeetingIntegrations/calendar-events.js b/MeetingAssistant/Native/MacOsMeetingIntegrations/calendar-events.js new file mode 100644 index 0000000..34c4e6e --- /dev/null +++ b/MeetingAssistant/Native/MacOsMeetingIntegrations/calendar-events.js @@ -0,0 +1,68 @@ +function safe(read, fallback) { + try { + const value = read(); + return value === null || value === undefined ? fallback : value; + } catch (_) { + return fallback; + } +} + +function participantText(participant) { + const name = String(safe(() => participant.displayName(), "")).trim(); + const email = String(safe(() => participant.email(), "")).trim(); + if (!name) { + return email; + } + if (!email || name.toLowerCase() === email.toLowerCase()) { + return name; + } + return `${name} <${email}>`; +} + +function run(argv) { + if (argv.length !== 2) { + throw new Error("Usage: calendar-events.js "); + } + + const from = new Date(argv[0]); + const to = new Date(argv[1]); + const calendar = Application("Calendar"); + const result = []; + + for (const sourceCalendar of calendar.calendars()) { + const events = sourceCalendar.events.whose({ + _and: [ + { startDate: { _lessThan: to } }, + { endDate: { _greaterThan: from } } + ] + })(); + for (const event of events) { + const start = event.startDate(); + const end = event.endDate(); + const attendees = safe(() => event.attendees(), []) + .map(participantText) + .filter(value => value.length > 0); + const organizer = String(safe(() => event.organizer(), "")).trim(); + if (organizer && !attendees.includes(organizer)) { + attendees.unshift(organizer); + } + + result.push({ + id: String(safe(() => event.uid(), `${start.toISOString()}|${event.summary()}`)), + subject: String(safe(() => event.summary(), "")), + start: start.toISOString(), + end: end.toISOString(), + location: String(safe(() => event.location(), "")), + notes: String(safe(() => event.description(), "")), + url: String(safe(() => event.url(), "")), + attendees: attendees, + isCanceled: String(safe(() => event.status(), "")) + .toLowerCase() + .includes("cancel") + }); + } + } + + result.sort((left, right) => left.start.localeCompare(right.start)); + return JSON.stringify(result); +} diff --git a/MeetingAssistant/Native/MacOsMeetingIntegrations/main.swift b/MeetingAssistant/Native/MacOsMeetingIntegrations/main.swift new file mode 100644 index 0000000..a9378dc --- /dev/null +++ b/MeetingAssistant/Native/MacOsMeetingIntegrations/main.swift @@ -0,0 +1,330 @@ +import AppKit +import CoreGraphics +import EventKit +import Foundation +import ImageIO +import UniformTypeIdentifiers + +private struct CalendarEventOutput: Encodable { + let id: String + let subject: String + let start: Date + let end: Date + let location: String + let notes: String + let url: String + let attendees: [String] + let isCanceled: Bool +} + +private struct PromptRequest: Decodable { + let subject: String + let start: Date +} + +private struct PromptOutput: Encodable { + let response: String +} + +private struct ScreenshotOutput: Encodable { + let windowTitle: String? +} + +private struct SelfTestOutput: Encodable { + let status: String + let features: [String] +} + +private enum IntegrationError: LocalizedError { + case usage + case invalidDate(String) + case calendarPermission + case screenRecordingPermission + case noFrontmostApplication + case noActiveWindow + case screenshotCapture + case screenshotWrite + + var errorDescription: String? { + switch self { + case .usage: + return "Usage: macos-meeting-integrations self-test | calendar | prompt | screenshot | crop " + case .invalidDate(let value): + return "Invalid ISO-8601 date: \(value)" + case .calendarPermission: + return "Calendar Full Access permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Calendars." + case .screenRecordingPermission: + return "Screen & System Audio Recording permission is required. Enable Meeting Assistant under System Settings > Privacy & Security." + case .noFrontmostApplication: + return "Could not determine the frontmost macOS application." + case .noActiveWindow: + return "Could not find an on-screen window for the frontmost macOS application." + case .screenshotCapture: + return "macOS could not capture the active application window." + case .screenshotWrite: + return "macOS could not write the active-window screenshot as PNG." + } + } +} + +@main +private enum MacOsMeetingIntegrations { + @MainActor + static func main() async { + do { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard let command = arguments.first else { + throw IntegrationError.usage + } + + switch command { + case "self-test": + try writeJson(SelfTestOutput( + status: "ok", + features: ["calendar", "prompt", "screenshot", "crop"])) + case "calendar": + guard arguments.count == 3 else { + throw IntegrationError.usage + } + let from = try parseDate(arguments[1]) + let to = try parseDate(arguments[2]) + try await writeCalendar(from: from, to: to) + case "prompt": + guard arguments.count == 2 else { + throw IntegrationError.usage + } + try showPrompt(requestPath: arguments[1]) + case "screenshot": + guard arguments.count == 2 else { + throw IntegrationError.usage + } + try captureScreenshot(outputPath: arguments[1]) + case "crop": + guard arguments.count == 7, + let x = Int(arguments[3]), + let y = Int(arguments[4]), + let width = Int(arguments[5]), + let height = Int(arguments[6]) else { + throw IntegrationError.usage + } + try cropScreenshot( + inputPath: arguments[1], + outputPath: arguments[2], + x: x, + y: y, + width: width, + height: height) + default: + throw IntegrationError.usage + } + } catch { + fputs("\(error.localizedDescription)\n", stderr) + if let integrationError = error as? IntegrationError, + case .calendarPermission = integrationError { + exit(77) + } + exit(1) + } + } + + private static func parseDate(_ value: String) throws -> Date { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: value) { + return date + } + + formatter.formatOptions = [.withInternetDateTime] + guard let date = formatter.date(from: value) else { + throw IntegrationError.invalidDate(value) + } + return date + } + + private static func writeCalendar(from: Date, to: Date) async throws { + let store = EKEventStore() + guard try await requestCalendarAccess(store) else { + throw IntegrationError.calendarPermission + } + + let predicate = store.predicateForEvents(withStart: from, end: to, calendars: nil) + let events = store.events(matching: predicate) + .map { event in + CalendarEventOutput( + id: event.eventIdentifier ?? "\(event.startDate.timeIntervalSince1970)|\(event.title ?? "")", + subject: event.title ?? "", + start: event.startDate, + end: event.endDate, + location: event.location ?? "", + notes: event.notes ?? "", + url: event.url?.absoluteString ?? "", + attendees: attendees(from: event), + isCanceled: event.status == .canceled) + } + .sorted { $0.start < $1.start } + try writeJson(events) + } + + private static func requestCalendarAccess(_ store: EKEventStore) async throws -> Bool { + if #available(macOS 14.0, *) { + return try await store.requestFullAccessToEvents() + } + + return try await withCheckedThrowingContinuation { continuation in + store.requestAccess(to: .event) { granted, error in + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: granted) + } + } + } + } + + private static func attendees(from event: EKEvent) -> [String] { + var values: [String] = [] + if let organizer = event.organizer, + let value = participantText(organizer) { + values.append(value) + } + for participant in event.attendees ?? [] { + if let value = participantText(participant) { + values.append(value) + } + } + return Array(NSOrderedSet(array: values)) as? [String] ?? values + } + + private static func participantText(_ participant: EKParticipant) -> String? { + let name = participant.name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let email = participant.url + .absoluteString + .replacingOccurrences(of: "mailto:", with: "", options: .caseInsensitive) + if name.isEmpty { + return email.isEmpty ? nil : email + } + if email.isEmpty || name.caseInsensitiveCompare(email) == .orderedSame { + return name + } + return "\(name) <\(email)>" + } + + @MainActor + private static func showPrompt(requestPath: String) throws { + let data = try Data(contentsOf: URL(fileURLWithPath: requestPath)) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let request = try decoder.decode(PromptRequest.self, from: data) + + let application = NSApplication.shared + application.setActivationPolicy(.accessory) + application.activate(ignoringOtherApps: true) + + let alert = NSAlert() + alert.messageText = "Record meeting?" + alert.informativeText = "\(request.subject) starts at \(request.start.formatted(date: .omitted, time: .shortened))." + alert.alertStyle = .informational + alert.addButton(withTitle: "Record") + alert.addButton(withTitle: "Skip") + let response = alert.runModal() == .alertFirstButtonReturn ? "record" : "skip" + try writeJson(PromptOutput(response: response)) + } + + private static func captureScreenshot(outputPath: String) throws { + guard CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() else { + throw IntegrationError.screenRecordingPermission + } + guard let frontmost = NSWorkspace.shared.frontmostApplication else { + throw IntegrationError.noFrontmostApplication + } + guard let windows = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID) as? [[String: Any]] else { + throw IntegrationError.noActiveWindow + } + + let candidate = windows.first { window in + let ownerPid = window[kCGWindowOwnerPID as String] as? Int32 + let layer = window[kCGWindowLayer as String] as? Int + let alpha = window[kCGWindowAlpha as String] as? Double + guard ownerPid == frontmost.processIdentifier, + layer == 0, + (alpha ?? 1) > 0, + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let width = bounds["Width"] as? Double, + let height = bounds["Height"] as? Double else { + return false + } + return width > 1 && height > 1 + } + guard let candidate, + let windowNumber = candidate[kCGWindowNumber as String] as? UInt32 else { + throw IntegrationError.noActiveWindow + } + guard let image = CGWindowListCreateImage( + .null, + .optionIncludingWindow, + CGWindowID(windowNumber), + [.boundsIgnoreFraming, .bestResolution]) else { + throw IntegrationError.screenshotCapture + } + + try writePng(image, to: outputPath) + + try writeJson(ScreenshotOutput( + windowTitle: candidate[kCGWindowName as String] as? String)) + } + + private static func cropScreenshot( + inputPath: String, + outputPath: String, + x: Int, + y: Int, + width: Int, + height: Int + ) throws { + guard x >= 0, + y >= 0, + width > 0, + height > 0, + let source = CGImageSourceCreateWithURL( + URL(fileURLWithPath: inputPath) as CFURL, + nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil), + x + width <= image.width, + y + height <= image.height, + let cropped = image.cropping(to: CGRect( + x: x, + y: y, + width: width, + height: height)) else { + throw IntegrationError.screenshotCapture + } + + try writePng(cropped, to: outputPath) + try writeJson(["status": "ok"]) + } + + private static func writePng(_ image: CGImage, to outputPath: String) throws { + let outputUrl = URL(fileURLWithPath: outputPath) as CFURL + guard let destination = CGImageDestinationCreateWithURL( + outputUrl, + UTType.png.identifier as CFString, + 1, + nil) else { + throw IntegrationError.screenshotWrite + } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { + throw IntegrationError.screenshotWrite + } + } + + private static func writeJson(_ value: T) throws { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.sortedKeys] + FileHandle.standardOutput.write(try encoder.encode(value)) + FileHandle.standardOutput.write(Data([0x0A])) + } +} diff --git a/MeetingAssistant/PlatformCapabilityReport.cs b/MeetingAssistant/PlatformCapabilityReport.cs new file mode 100644 index 0000000..cd9f572 --- /dev/null +++ b/MeetingAssistant/PlatformCapabilityReport.cs @@ -0,0 +1,52 @@ +using MeetingAssistant.Calendar; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Screenshots; +using MeetingAssistant.Workflow; + +namespace MeetingAssistant; + +public sealed record PlatformCapabilityReport( + string Host, + string MeetingMetadataProvider, + string CalendarMeetingProvider, + string MeetingStartPromptProvider, + string ScreenshotProvider, + string WorkflowEditorProvider, + bool WorkflowEditorAvailable, + bool CalendarRecordingPromptsEnabled, + bool SpeakerIdentificationEnabled, + bool ScreenshotOcrEnabled, + bool FunAsrBackendEnabled, + bool FunAsrDiarizationEnabled, + bool WhisperLocalDiarizationEnabled) +{ + public static PlatformCapabilityReport Create( + HostOperatingSystem host, + MeetingAssistantOptions options, + IMeetingMetadataProvider meetingMetadataProvider, + ICalendarMeetingProvider calendarMeetingProvider, + IMeetingStartPromptService meetingStartPromptProvider, + IActiveWindowScreenshotCapture screenshotProvider, + IWorkflowRulesEditorWindowService workflowEditorProvider) + { + return new PlatformCapabilityReport( + host.Kind switch + { + HostOperatingSystemKind.Windows => "Windows", + HostOperatingSystemKind.MacOS => "macOS", + _ => "Unsupported" + }, + meetingMetadataProvider.GetType().Name, + calendarMeetingProvider.GetType().Name, + meetingStartPromptProvider.GetType().Name, + screenshotProvider.GetType().Name, + workflowEditorProvider.GetType().Name, + workflowEditorProvider is not NoopWorkflowRulesEditorWindowService, + options.CalendarRecordingPrompts.Enabled, + options.SpeakerIdentification.Enabled, + options.Screenshots.Ocr.Enabled, + options.FunAsr.Backend.Enabled, + options.FunAsr.Diarization.Enabled, + options.WhisperLocal.Diarization.Enabled); + } +} diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index 055e2ef..d3a7b73 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -3,6 +3,7 @@ using MeetingAssistant.Calendar; using MeetingAssistant.Hotkeys; using MeetingAssistant.LaunchProfiles; using MeetingAssistant.Logging; +using MeetingAssistant.MacOs; using MeetingAssistant.MeetingNotes; using MeetingAssistant.Recording; using MeetingAssistant.Screenshots; @@ -20,6 +21,7 @@ builder.Services.AddSingleton, Meeting builder.Services.AddOptions() .Bind(builder.Configuration.GetSection("MeetingAssistant")) .ValidateOnStart(); +builder.Services.AddSingleton(HostOperatingSystem.DetectCurrent()); builder.Services.AddSingleton(); #if WINDOWS builder.Services.AddSingleton(); @@ -38,8 +40,9 @@ builder.Services.AddSingleton(services => new CompositeMeet services.GetRequiredService>())); builder.Services.AddSingleton(); #else -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +PortableMeetingAudioRegistration.Add(builder.Services); +MacOsMeetingIntegrationRegistration.Add(builder.Services, OperatingSystem.IsMacOS()); +builder.Services.AddHostedService(); #endif builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -69,14 +72,8 @@ builder.Services.AddSingleton(services => services.GetRequiredService()); builder.Services.AddHostedService(services => services.GetRequiredService()); -#else -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -#endif -#if WINDOWS builder.Services.AddSingleton(); -#else -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); #endif builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -144,7 +141,8 @@ builder.Services.AddSingleton(services => WorkflowRulesEditorMarkdownLinkResolve services.GetRequiredService>().Value)); builder.Services.AddSingleton(); #else -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); #endif builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -181,11 +179,32 @@ app.MapGet("/health", () => Results.Ok(new service = "meeting-assistant", status = "ok" })); +app.MapGet("/diagnostics/platform-capabilities", ( + HostOperatingSystem host, + IOptions options, + IMeetingMetadataProvider meetingMetadataProvider, + ICalendarMeetingProvider calendarMeetingProvider, + IMeetingStartPromptService meetingStartPromptProvider, + IActiveWindowScreenshotCapture screenshotProvider, + IWorkflowRulesEditorWindowService workflowEditorProvider) => + Results.Ok(PlatformCapabilityReport.Create( + host, + options.Value, + meetingMetadataProvider, + calendarMeetingProvider, + meetingStartPromptProvider, + screenshotProvider, + workflowEditorProvider))); app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus)); app.MapGet("/profiles/{launchProfile}/recording/status", ( string launchProfile, MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus)); +app.MapPost("/application/exit", (IHostApplicationLifetime lifetime) => +{ + lifetime.StopApplication(); + return Results.Accepted(); +}); app.MapGet("/diagnostics/outlook/current-meeting", async ( IMeetingMetadataProvider metadataProvider, IOptions options, @@ -315,6 +334,15 @@ app.MapPost("/diagnostics/settings-and-logs/show", ( rulesEditorWindow.Show(); return Results.Accepted(); }); +#if !WINDOWS +app.MapGet("/diagnostics/settings-and-logs", () => + Results.Content(MacOsWorkflowRulesEditorPage.Html, "text/html")); +app.MapPost("/diagnostics/settings-and-logs/chat", async ( + MacOsWorkflowRulesEditorChatRequest request, + MacOsWorkflowRulesEditorSession session, + CancellationToken cancellationToken) => + Results.Ok(await session.SendAsync(request.Message, cancellationToken))); +#endif static WorkflowConfigurationReloadResponse ReloadWorkflowConfiguration(IConfiguration configuration) { @@ -357,6 +385,11 @@ app.MapPost("/profiles/{launchProfile}/recording/abort", async ( MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => Results.Ok(await coordinator.AbortAsync(cancellationToken))); +app.MapPost("/profiles/{launchProfile}/meetings/screenshot/capture", async ( + string launchProfile, + MeetingRecordingCoordinator coordinator, + CancellationToken cancellationToken) => + Results.Ok(await coordinator.CaptureScreenshotAsync(launchProfile, cancellationToken))); app.MapPost("/meetings/current/summary/run", async ( MeetingRecordingCoordinator coordinator, IMeetingSummaryPipeline summaryPipeline, diff --git a/MeetingAssistant/Recording/MacOsMeetingAudioSource.cs b/MeetingAssistant/Recording/MacOsMeetingAudioSource.cs new file mode 100644 index 0000000..a1aac17 --- /dev/null +++ b/MeetingAssistant/Recording/MacOsMeetingAudioSource.cs @@ -0,0 +1,274 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace MeetingAssistant.Recording; + +internal enum MacOsAudioCaptureKind +{ + Microphone, + System +} + +internal sealed record MacOsAudioCaptureRequest( + MacOsAudioCaptureKind Kind, + int SampleRate, + int Channels); + +internal interface IMacOsAudioCaptureProcessFactory +{ + IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request); +} + +internal interface IMacOsAudioCaptureProcess : IAsyncDisposable +{ + Stream StandardOutput { get; } + + int ExitCode { get; } + + bool HasExited { get; } + + Task ReadStandardErrorAsync(CancellationToken cancellationToken); + + Task WaitForExitAsync(CancellationToken cancellationToken); + + void Terminate(); +} + +internal abstract class MacOsMeetingAudioSource : IMeetingAudioSource +{ + private const int ReadBufferSize = 8 * 1024; + private readonly MacOsAudioCaptureKind kind; + private readonly IMacOsAudioCaptureProcessFactory processFactory; + private readonly ILogger logger; + + protected MacOsMeetingAudioSource( + MacOsAudioCaptureKind kind, + IMacOsAudioCaptureProcessFactory processFactory, + ILogger logger) + { + this.kind = kind; + this.processFactory = processFactory; + this.logger = logger; + } + + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + return CaptureAsync(new MeetingAssistantOptions(), cancellationToken); + } + + public async IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var request = new MacOsAudioCaptureRequest( + kind, + options.Recording.SampleRate, + options.Recording.Channels); + await using var process = processFactory.Start(request); + var standardErrorTask = process.ReadStandardErrorAsync(CancellationToken.None); + var buffer = new byte[ReadBufferSize]; + byte? pendingByte = null; + var hasLoggedFirstChunk = false; + + logger.LogInformation( + "Starting macOS {CaptureKind} audio capture at {SampleRate} Hz/{Channels} channel(s)", + kind, + request.SampleRate, + request.Channels); + + try + { + while (true) + { + var bytesRead = await process.StandardOutput.ReadAsync(buffer, cancellationToken); + if (bytesRead == 0) + { + break; + } + + var pcm = CopyCompleteSamples(buffer, bytesRead, ref pendingByte); + if (pcm.Length == 0) + { + continue; + } + + if (!hasLoggedFirstChunk) + { + hasLoggedFirstChunk = true; + logger.LogInformation( + "macOS {CaptureKind} audio produced its first PCM chunk: {ByteCount} bytes", + kind, + pcm.Length); + } + + yield return new AudioChunk(pcm, request.SampleRate, request.Channels); + } + + await process.WaitForExitAsync(cancellationToken); + var standardError = await standardErrorTask; + if (process.ExitCode != 0) + { + throw CreateCaptureException(kind, process.ExitCode, standardError); + } + } + finally + { + if (!process.HasExited) + { + process.Terminate(); + await process.WaitForExitAsync(CancellationToken.None); + } + } + } + + private static byte[] CopyCompleteSamples(byte[] buffer, int bytesRead, ref byte? pendingByte) + { + var totalBytes = bytesRead + (pendingByte.HasValue ? 1 : 0); + var completeBytes = totalBytes - totalBytes % sizeof(short); + var pcm = new byte[completeBytes]; + var targetOffset = 0; + + if (pendingByte is { } firstByte) + { + pcm[0] = firstByte; + targetOffset = 1; + pendingByte = null; + } + + var copyCount = completeBytes - targetOffset; + if (copyCount > 0) + { + Buffer.BlockCopy(buffer, 0, pcm, targetOffset, copyCount); + } + + var consumedFromBuffer = copyCount; + if (consumedFromBuffer < bytesRead) + { + pendingByte = buffer[consumedFromBuffer]; + } + + return pcm; + } + + private static InvalidOperationException CreateCaptureException( + MacOsAudioCaptureKind kind, + int exitCode, + string standardError) + { + var detail = string.IsNullOrWhiteSpace(standardError) + ? "The native capture helper did not provide an error message." + : standardError.Trim(); + return new InvalidOperationException( + $"macOS {kind.ToString().ToLowerInvariant()} audio capture exited with code {exitCode}. {detail}"); + } +} + +internal sealed class MacOsMicrophoneAudioSource : MacOsMeetingAudioSource +{ + public MacOsMicrophoneAudioSource( + IMacOsAudioCaptureProcessFactory processFactory, + ILogger logger) + : base(MacOsAudioCaptureKind.Microphone, processFactory, logger) + { + } +} + +internal sealed class MacOsSystemAudioSource : MacOsMeetingAudioSource +{ + public MacOsSystemAudioSource( + IMacOsAudioCaptureProcessFactory processFactory, + ILogger logger) + : base(MacOsAudioCaptureKind.System, processFactory, logger) + { + } +} + +internal sealed class MacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory +{ + private readonly ILogger logger; + + public MacOsAudioCaptureProcessFactory(ILogger logger) + { + this.logger = logger; + } + + public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request) + { + if (!OperatingSystem.IsMacOS()) + { + throw new PlatformNotSupportedException("The macOS audio capture helper can only run on macOS."); + } + + var helperPath = Path.Combine( + AppContext.BaseDirectory, + "Native", + "MeetingAssistantAudioCapture.app", + "Contents", + "MacOS", + "macos-meeting-audio-capture"); + if (!File.Exists(helperPath)) + { + throw new FileNotFoundException( + $"The macOS audio capture helper was not found at '{helperPath}'. Build or publish the application on macOS so the Swift helper is compiled into the Native folder.", + helperPath); + } + + var startInfo = new ProcessStartInfo + { + FileName = helperPath, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + startInfo.ArgumentList.Add(request.Kind == MacOsAudioCaptureKind.Microphone ? "microphone" : "system"); + startInfo.ArgumentList.Add(request.SampleRate.ToString(System.Globalization.CultureInfo.InvariantCulture)); + startInfo.ArgumentList.Add(request.Channels.ToString(System.Globalization.CultureInfo.InvariantCulture)); + + logger.LogDebug("Starting native macOS audio helper {HelperPath} for {CaptureKind}", helperPath, request.Kind); + var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start the macOS audio capture helper '{helperPath}'."); + return new MacOsAudioCaptureProcess(process); + } + +} + +internal sealed class MacOsAudioCaptureProcess : IMacOsAudioCaptureProcess +{ + private readonly Process process; + + public MacOsAudioCaptureProcess(Process process) + { + this.process = process; + } + + public Stream StandardOutput => process.StandardOutput.BaseStream; + + public int ExitCode => process.ExitCode; + + public bool HasExited => process.HasExited; + + public Task ReadStandardErrorAsync(CancellationToken cancellationToken) + { + return process.StandardError.ReadToEndAsync(cancellationToken); + } + + public Task WaitForExitAsync(CancellationToken cancellationToken) + { + return process.WaitForExitAsync(cancellationToken); + } + + public void Terminate() + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + + public ValueTask DisposeAsync() + { + process.Dispose(); + return ValueTask.CompletedTask; + } +} diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs index 03c57ed..788717b 100644 --- a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -710,6 +710,24 @@ public sealed class MeetingRecordingCoordinator catch (Exception exception) { logger.LogError(exception, "Meeting recording failed"); + if (IsAzureSpeechRun(run) && !run.IsQueuedForOfflineTranscription) + { + logger.LogWarning( + "Azure Speech transcription failed; retaining recorded audio and queueing offline transcription backlog item"); + run.MarkQueuedForOfflineTranscription(); + try + { + await offlineTranscriptionBacklog.EnqueueAsync( + CreateOfflineBacklogItem(run), + CancellationToken.None); + } + catch (Exception backlogException) + { + logger.LogError( + backlogException, + "Could not queue offline transcription backlog item; retaining recorded audio for manual recovery"); + } + } } finally { diff --git a/MeetingAssistant/Recording/PortableMeetingAudioRegistration.cs b/MeetingAssistant/Recording/PortableMeetingAudioRegistration.cs new file mode 100644 index 0000000..7a9017a --- /dev/null +++ b/MeetingAssistant/Recording/PortableMeetingAudioRegistration.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace MeetingAssistant.Recording; + +internal static class PortableMeetingAudioRegistration +{ + public static void Add(IServiceCollection services) + { + Add(services, OperatingSystem.IsMacOS()); + } + + internal static void Add(IServiceCollection services, bool isMacOs) + { + if (!isMacOs) + { + services.AddSingleton(); + return; + } + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(serviceProvider => new CompositeMeetingAudioSource( + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService>())); + } +} diff --git a/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs b/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs index 91f722d..fb7c6a2 100644 --- a/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs +++ b/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs @@ -5,6 +5,6 @@ public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) { throw new PlatformNotSupportedException( - "Meeting audio capture is only implemented for the Windows target in this version."); + "Meeting audio capture is supported on Windows and macOS, but is unavailable on this host."); } } diff --git a/MeetingAssistant/Screenshots/MeetingScreenshotService.cs b/MeetingAssistant/Screenshots/MeetingScreenshotService.cs index 8b9b4ea..efc54dc 100644 --- a/MeetingAssistant/Screenshots/MeetingScreenshotService.cs +++ b/MeetingAssistant/Screenshots/MeetingScreenshotService.cs @@ -1,7 +1,4 @@ using System.Collections.Concurrent; -using System.Drawing; -using System.Drawing.Imaging; -using System.Runtime.InteropServices; using System.Text.RegularExpressions; using MeetingAssistant.MeetingNotes; using MeetingAssistant.Speakers; @@ -46,6 +43,25 @@ public sealed record ScreenshotOcrResult public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height); +public interface IScreenshotImageCropper +{ + Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken); +} + +internal sealed class UnavailableScreenshotImageCropper : IScreenshotImageCropper +{ + public Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + return Task.FromResult(null); + } +} + public interface IMeetingScreenshotService { Task CaptureAsync( @@ -147,6 +163,7 @@ public sealed partial class MeetingScreenshotService : IMeetingNoteImageOcrService { private readonly IActiveWindowScreenshotCapture screenshotCapture; + private readonly IScreenshotImageCropper screenshotImageCropper; private readonly IMeetingArtifactStore artifactStore; private readonly IMeetingNoteStore meetingNoteStore; private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer; @@ -162,6 +179,7 @@ public sealed partial class MeetingScreenshotService : IMeetingNoteStore meetingNoteStore, ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer, IScreenshotOcrClient ocrClient, + IScreenshotImageCropper screenshotImageCropper, ILogger logger, IMeetingWorkflowEngine? meetingWorkflowEngine = null) { @@ -171,6 +189,7 @@ public sealed partial class MeetingScreenshotService : this.attendeeCanonicalizer = attendeeCanonicalizer; this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance; this.ocrClient = ocrClient; + this.screenshotImageCropper = screenshotImageCropper; this.logger = logger; } @@ -639,17 +658,20 @@ public sealed partial class MeetingScreenshotService : return ""; } - if (!OperatingSystem.IsWindows()) - { - logger.LogWarning( - "Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows", - screenshotPath); - return ""; - } - try { - var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken); + var croppedPath = await screenshotImageCropper.SaveCroppedScreenshotAsync( + screenshotPath, + crop, + cancellationToken); + if (croppedPath is null) + { + logger.LogWarning( + "Ignoring screenshot crop coordinates for {ScreenshotPath} because no image cropper is available on this host", + screenshotPath); + return ""; + } + var relativePath = ToMarkdownPath(Path.GetRelativePath( Path.GetDirectoryName(assistantContextPath)!, croppedPath)); @@ -657,7 +679,7 @@ public sealed partial class MeetingScreenshotService : Environment.NewLine + Environment.NewLine; } - catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException) + catch (Exception exception) when (exception is InvalidDataException or ArgumentException) { logger.LogWarning( exception, @@ -667,39 +689,6 @@ public sealed partial class MeetingScreenshotService : } } -#pragma warning disable CA1416 - private static Task SaveCroppedScreenshotAsync( - string screenshotPath, - ScreenshotCropCoordinates crop, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - using var original = new Bitmap(screenshotPath); - ValidateCrop(crop, original.Width, original.Height); - var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height); - using var cropped = original.Clone(cropRectangle, original.PixelFormat); - var croppedPath = Path.Combine( - Path.GetDirectoryName(screenshotPath)!, - $"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png"); - cropped.Save(croppedPath, ImageFormat.Png); - return Task.FromResult(croppedPath); - } -#pragma warning restore CA1416 - - private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight) - { - if (crop.X < 0 || - crop.Y < 0 || - crop.Width <= 0 || - crop.Height <= 0 || - crop.X + crop.Width > imageWidth || - crop.Y + crop.Height > imageHeight) - { - throw new InvalidDataException( - $"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}."); - } - } - private void TrackOcr( MeetingSessionArtifacts artifacts, CancellationTokenSource cancellation, diff --git a/MeetingAssistant/Screenshots/WindowsScreenshotImageCropper.Windows.cs b/MeetingAssistant/Screenshots/WindowsScreenshotImageCropper.Windows.cs new file mode 100644 index 0000000..b8af752 --- /dev/null +++ b/MeetingAssistant/Screenshots/WindowsScreenshotImageCropper.Windows.cs @@ -0,0 +1,55 @@ +#if WINDOWS +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; + +namespace MeetingAssistant.Screenshots; + +internal sealed class WindowsScreenshotImageCropper : IScreenshotImageCropper +{ +#pragma warning disable CA1416 + public Task SaveCroppedScreenshotAsync( + string screenshotPath, + ScreenshotCropCoordinates crop, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + using var original = new Bitmap(screenshotPath); + ValidateCrop(crop, original.Width, original.Height); + var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height); + using var cropped = original.Clone(cropRectangle, original.PixelFormat); + var croppedPath = Path.Combine( + Path.GetDirectoryName(screenshotPath)!, + $"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png"); + cropped.Save(croppedPath, ImageFormat.Png); + return Task.FromResult(croppedPath); + } + catch (ExternalException exception) + { + throw new InvalidDataException( + $"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.", + exception); + } + } +#pragma warning restore CA1416 + + private static void ValidateCrop( + ScreenshotCropCoordinates crop, + int imageWidth, + int imageHeight) + { + if (crop.X < 0 || + crop.Y < 0 || + crop.Width <= 0 || + crop.Height <= 0 || + crop.X + crop.Width > imageWidth || + crop.Y + crop.Height > imageHeight) + { + throw new InvalidDataException( + $"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}."); + } + } +} +#endif diff --git a/MeetingAssistant/Workflow/WorkflowRulesEditorInstructionBuilder.cs b/MeetingAssistant/Workflow/WorkflowRulesEditorInstructionBuilder.cs index 7921c83..fa10e37 100644 --- a/MeetingAssistant/Workflow/WorkflowRulesEditorInstructionBuilder.cs +++ b/MeetingAssistant/Workflow/WorkflowRulesEditorInstructionBuilder.cs @@ -38,10 +38,14 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor Explain the final change briefly after the tools finish. """; + private readonly HostOperatingSystem hostOperatingSystem; private readonly ILogger logger; - public WorkflowRulesEditorInstructionBuilder(ILogger logger) + public WorkflowRulesEditorInstructionBuilder( + HostOperatingSystem hostOperatingSystem, + ILogger logger) { + this.hostOperatingSystem = hostOperatingSystem; this.logger = logger; } @@ -53,8 +57,10 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor : editorOptions.InitialPrompt!; var rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath) ?? ""; var docs = await ReadWorkflowDocsAsync(cancellationToken); + var platformGuidance = BuildPlatformGuidance(hostOperatingSystem); return configuredPrompt.Trim() + Environment.NewLine + Environment.NewLine + + platformGuidance + Environment.NewLine + Environment.NewLine + $"Configured workflow rules file: {rulesPath}" + Environment.NewLine + Environment.NewLine + "Configuration tools can read and replace the local appsettings JSON file. Use read_config_docs for the configuration reference." + Environment.NewLine + Environment.NewLine + "Log tools can read and search the current application-owned log file and four rotated older files under the temp log folder." + Environment.NewLine + Environment.NewLine + @@ -69,6 +75,28 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor "```"; } + private static string BuildPlatformGuidance(HostOperatingSystem operatingSystem) + { + var platformGuidance = operatingSystem.Kind switch + { + HostOperatingSystemKind.Windows => """ + Detected host operating system: Windows. + Use Windows-native operational guidance: PowerShell and native Windows commands, Windows-style paths, Windows environment-variable conventions, Windows services and processes, and Task Manager. Do not use macOS-only commands or concepts. + """, + HostOperatingSystemKind.MacOS => """ + Detected host operating system: macOS. + Use macOS-native operational guidance: zsh and native macOS commands, POSIX-style paths, macOS environment-variable conventions, launchd and LaunchAgents, application processes, and Activity Monitor. Do not use Windows-only commands or concepts. + """, + _ => """ + Detected host operating system: unsupported. + Use portable .NET concepts. Do not assume Windows or macOS commands, paths, services, environment conventions, or desktop features. + """ + }; + + return platformGuidance + Environment.NewLine + + "Prefer the provided Meeting Assistant tools over shell commands when a matching tool exists. Operating-system detection selects appropriate guidance; it does not make unavailable integrations available."; + } + private async Task ReadWorkflowDocsAsync(CancellationToken cancellationToken) { foreach (var path in RuntimeContentLocator.CandidateDocumentationPaths("meeting-workflow-engine.md")) diff --git a/README.md b/README.md index e5c0573..b968d99 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ # Meeting Assistant -Meeting Assistant is Manuel's local Windows meeting capture and knowledge service. It records microphone plus system audio, maintains meeting artifacts in an Obsidian vault, enriches them from local context, and runs transcription, speaker attribution, screenshot OCR, and agentic summarization without depending on a meeting-platform API for the primary flow. +Meeting Assistant is Manuel's local .NET meeting capture and knowledge service. It runs on Windows and macOS, records microphone plus system audio, maintains meeting artifacts in an Obsidian vault, and runs transcription, speaker attribution, screenshot OCR, and agentic summarization without depending on a meeting-platform API for the primary flow. ## Scope And Runtime Boundary This repository owns the application source, tests, OpenSpec requirements, checked-in configuration, and CI validation. It does not own workstation startup automation, a homelab deployment stack, Traefik routing, Docker Compose, or a deployed image tag. -The Windows target provides audio capture, the tray icon, global hotkeys, Outlook Classic COM access, active-window screenshots, notifications, and the workflow-agent window. The cross-platform target keeps the HTTP and testable service surface but substitutes unavailable/no-op implementations for Windows integrations. - +- Windows builds provide the tray icon, global hotkeys, NAudio capture, Outlook Classic COM enrichment, active-window screenshots, and notifications. +- Portable builds on macOS provide a native menu-bar icon and global hotkeys, and capture the default microphone through AVFoundation plus computer output through ScreenCaptureKit. +- Portable builds on other hosts keep the server/testable service surface but do not provide meeting audio capture. +- The normal runtime endpoint is local HTTP on port `5090`, with `/health` and `/recording/status` as the safe first checks. Accepted requirements live under `openspec/specs`. Relevant active changes under `openspec/changes` can describe implemented behavior that has not yet been folded into the accepted specs. ## Quick Start @@ -27,6 +29,15 @@ Invoke-RestMethod http://127.0.0.1:5090/health Invoke-RestMethod http://127.0.0.1:5090/recording/status ``` +On macOS, build the portable target on the Mac that will run it: + +```zsh +dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 +dotnet run --project MeetingAssistant/MeetingAssistant.csproj -f net10.0 +``` + +The macOS build compiles bundled Swift helpers into `Native/macos-meeting-audio-capture`, `Native/macos-desktop-controls`, and `Native/macos-meeting-integrations`. On first use, allow **Microphone**, **Screen & System Audio Recording**, and **Calendar Full Access** under System Settings > Privacy & Security. The menu-bar icon and global hotkeys use the same configured bindings as Windows and call the local service surface. + For a foreground Windows development run when port `5090` is free: ```powershell @@ -43,16 +54,31 @@ The helper publishes before stopping the old process, starts the timestamped Win ## Recording And Control -The default global controls are: +Recording can be controlled through global hotkeys, the Windows tray icon, the macOS menu-bar icon, or local HTTP endpoints. The default hotkeys are: - `Ctrl+Alt+M`: toggle the default recording profile. - `Ctrl+Alt+L`: start or switch to the configured `english` profile. - `Ctrl+Alt+Z`: abort the active run and delete its artifacts. - `Ctrl+Alt+S`: capture the active window into the meeting context. -The tray presents `Finish meeting` as the primary action during capture; pause/unpause transcription, microphone selection, cancel/discard, and profile switching remain separate controls. A paused meeting stays active and can still be finished or canceled normally. `Exit` is always available and requires confirmation while any meeting is recording or processing. +The Windows tray presents `Finish meeting` as the primary action during capture; microphone selection, cancel/discard, profile switching, and pause/unpause transcription remain separate controls. A paused meeting stays active and can still be finished or canceled normally. Windows `Exit` requires confirmation while any meeting is recording or processing. The macOS menu offers profile start actions while idle and `Stop meeting recording and transcribe` plus cancel/discard during capture; its `Exit` action currently exits directly. -The loopback HTTP surface exposes health and recording status, recording start/stop/toggle/abort operations, profile-specific equivalents, and diagnostics/retry operations. It has no application authentication. Some generated retry links use `GET` while starting work, so port `5090` must remain a trusted loopback-only control surface. +The loopback HTTP surface has no application authentication, so port `5090` must remain a trusted loopback-only control surface. The main endpoints are: + +- `GET /health` +- `GET /recording/status` +- `POST /recording/start`, `/recording/stop`, `/recording/toggle`, `/recording/abort` +- `POST /profiles/{launchProfile}/recording/start`, `/stop`, `/toggle`, `/abort` +- `POST /asr/transcribe-file` and `/asr/diarize-file` for diagnostic WAV checks +- `POST /diagnostics/workflow/reload` +- `POST /diagnostics/settings-and-logs/show` +- `POST /diagnostics/workflow/rules-editor/show` +- `GET /diagnostics/platform-capabilities` +- `POST /meetings/current/summary/run` +- `POST` or `GET /meetings/summary/retry` +- `POST` or `GET /meetings/screenshot-ocr/retry` + +Some generated retry links use `GET` while starting work. Stopping capture lets buffered transcription, speaker work, meeting-note image OCR, screenshot OCR, and summary generation finish. Another meeting can start while an older stopped run finalizes; each run retains isolated options and artifact paths. @@ -84,7 +110,7 @@ Abort is destructive: it removes the active run's note, transcript, context, sum - The default `azure-speech` provider sends mixed meeting audio and dictation phrase hints to Azure AI Speech. Azure-backed speaker matching also sends selected voice audio. - Summary, screenshot OCR, and interactive-agent requests go to the configured OpenAI-compatible Responses endpoint. They can include meeting/transcript/project text, screenshots, configuration, logs, and speaker samples when corresponding tools are used. The checked-in endpoint is a loopback proxy; its ultimate provider, data path, and retention policy are outside this repository. -- Outlook Classic access is local COM and reads appointment metadata; it does not provide the primary capture path. +- Outlook Classic access on Windows is local COM. EventKit access on macOS reads calendars synchronized into the Calendar app. Neither provides the primary capture path. - A managed FunASR run pulls its configured image, removes any same-named container, starts a disposable container privileged by default, publishes the configured host port, and mounts the model/hotword cache. Pyannote may build a local image and start disposable containers with audio inputs mounted read-only; those paths require Docker Desktop or a compatible Docker CLI. The opt-in Resemblyzer speaker matcher instead provisions an isolated local Python venv with CPU-only dependencies. First use may download images, Python packages, or models from external registries. ## Configuration @@ -95,21 +121,51 @@ The settings with the largest operational effect are: - `Vault`: selects the durable vault and artifact/project locations. - `Recording:TranscriptionProvider`: selects `azure-speech`, `funasr`, or `whisper-local`; the latter requires a local Whisper model file. -- `Recording:MicrophoneDeviceId`, mix gains, stop timeout, minimum duration, and temporary folder: control capture selection, audio, cleanup, and Azure backlog behavior. +- `Recording:MicrophoneDeviceId`, mix gains, stop timeout, minimum duration, and temporary folder: control capture selection, audio, cleanup, and Azure backlog behavior. Microphone-device selection is Windows-only; macOS follows the system default input device. - `Recording:InactivitySafeguard`: prompts and can auto-finish a run after no new transcript text; it is not an audio-silence detector. - `LaunchProfiles`: overlay named recording/ASR/agent settings and require distinct hotkeys. - `SpeakerIdentification:Resemblyzer:Enabled`: selects the local, managed-Python-venv vector matcher for the whole application; when disabled, the existing WAV/Azure path stays active. Five vectors unlock matching by default without capping retained evidence, and mature profiles use configurable fail-safe density clustering to remove likely mixed-speaker outliers. -- `Automation:RulesPath`: points to the local YAML rules file, normally ignored `meeting-rules.local.yaml`. -- `CalendarRecordingPrompts` and `Screenshots`: control Outlook prompts, capture, attachments, and configured OCR. +- `Automation:RulesPath`: points to the local YAML workflow-rules file, normally ignored `meeting-rules.local.yaml`. +- `CalendarRecordingPrompts` and `Screenshots`: control Outlook prompts on Windows, EventKit prompts on macOS, capture, attachments, and configured OCR. - `Agent` and `WorkflowRulesEditor`: select the Responses endpoint/model, streaming or non-streaming transport, reasoning, retry, output, and compaction behavior; the available tools are defined by the application. -Common secret environment variables are `AZURE_SPEECH_KEY`, `LITELLM_API_KEY`, and `HF_TOKEN`. See `docs/meeting-assistant-configuration.md` for the full setting reference and `docs/meeting-workflow-engine.md` for rule syntax and safety behavior. +Required or commonly used secrets: + +- `AZURE_SPEECH_KEY` for Azure Speech live transcription and speaker matching. +- `LITELLM_API_KEY` for summary, OCR, and workflow editor agents when their effective endpoint requires an API key. +- `HF_TOKEN` for pyannote model access when pyannote diarization or validation is enabled. + +See `docs/meeting-assistant-configuration.md` for the full configuration reference. + +## Integrations + +- **Obsidian vault**: primary durable store for notes, transcripts, summaries, assistant context, project files, and generated links. +- **Outlook Classic on Windows**: optional COM metadata lookup and scheduled Teams-meeting start prompts. +- **EventKit on macOS**: metadata lookup and native recording prompts for Teams events in calendars available to macOS, including Outlook-synced calendars. +- **Native active-window capture**: Windows foreground-window capture or macOS frontmost-application window capture, feeding the same screenshot/OCR pipeline. +- **Azure AI Speech**: default checked-in ASR path, live diarized conversation transcription, and speaker identity matching. +- **FunASR**: optional WebSocket streaming ASR. When managed backend startup is enabled, the app pulls and runs the configured Docker image as `meeting-assistant-funasr` on port `10095`. +- **Whisper.NET plus pyannote**: optional local Whisper fallback and Docker-backed final diarization. +- **LiteLLM/OpenAI-compatible Responses endpoint**: summary generation, screenshot OCR fallback, project tools, the tray-launched assistant, and retry flows. +- **Docker Desktop or compatible Docker CLI**: required only for managed FunASR and pyannote paths. + +## Workflow Rules And Agents + +Meeting-specific automation lives in a local YAML file, not in committed personal rules. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, transcript-line writes, and added attendees. They can add/remove attendees, set supported properties, add context, add projects, rewrite a transcript line, and transform an attendee name before it is stored. + +The Windows tray and macOS menu-bar menus expose `Open agent`, which opens the `Meeting Summary Agent` window. It can edit workflow rules with validation, inspect logs and health/status, manage speaker identities and samples, run ASR diagnostics, and read/write scoped meeting/project artifacts through explicit tools. Windows renders the chat with WPF; macOS renders the same agent pipeline in a native AppKit window backed by WebKit. + +At startup, Meeting Assistant detects whether the host is Windows or macOS and includes that immutable runtime context in the interactive settings/logs assistant instructions. On Windows the assistant uses Windows commands and concepts such as PowerShell, Windows paths, services, and Task Manager. On macOS it uses zsh, POSIX paths, launchd/LaunchAgents, and Activity Monitor. This guidance is also appended when a custom interactive-agent prompt is configured. macOS audio capture, menu-bar controls, global hotkeys, EventKit calendar enrichment/prompts, active-window screenshots, screenshot OCR, speaker identification, FunASR, local diarization, and the AppKit/WebKit workflow editor are available in the portable build. Outlook Classic COM and Windows toast notifications remain Windows-specific implementations. + +The macOS publish output includes a signed `MeetingAssistant.app` bundle. Install that bundle in `/Applications` and launch the background service through its native executable so macOS microphone, calendar, and Screen/System Audio privacy grants are attributed to the stable Meeting Assistant application identity. + +Detailed workflow syntax and extension guidance live in `docs/meeting-workflow-engine.md`. ## Development And CI Behavior changes are OpenSpec-driven and test-first: update the relevant requirement/scenario, add a failing public behavior test, implement the smallest passing change, run focused tests and then the justified broader suite, and validate the active change with `openspec validate --strict`. Documentation-only maintenance does not need a new OpenSpec change. -The Gitea workflow runs for pull requests, pushes, and manual dispatch. It builds the Windows target on an Ubuntu runner, installs Wine plus a matching Windows .NET SDK, and runs the test project through the Windows host under Wine. It validates source; it does not publish or deploy the workstation application. +The Gitea workflow runs for pull requests, pushes, and manual dispatch. Its Linux job builds the Windows target, installs Wine plus a matching Windows .NET SDK, and runs the test project through the Windows host under Wine. A separate `runs-on: macos` job builds the portable target and bundled Swift helpers, verifies the native helper executables, and runs the complete portable unit-test suite on macOS. The Gitea runner serving that job must run natively on a Mac and expose the `macos:host` label; Docker-OSX is not used because it requires nested KVM that is normally unavailable on CI runners. CI validates source; it does not publish or deploy the workstation application. ## Operations And Limitations diff --git a/docs/evidence/macos-meeting-audio-proof.wav b/docs/evidence/macos-meeting-audio-proof.wav new file mode 100644 index 0000000..076ae23 Binary files /dev/null and b/docs/evidence/macos-meeting-audio-proof.wav differ diff --git a/docs/meeting-assistant-configuration.md b/docs/meeting-assistant-configuration.md index 3d9d4ce..983c7aa 100644 --- a/docs/meeting-assistant-configuration.md +++ b/docs/meeting-assistant-configuration.md @@ -132,13 +132,15 @@ The default profile is always named `default`. Non-default profile hotkeys are r During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription. +On macOS, the portable target captures the default microphone through AVFoundation and computer output through ScreenCaptureKit. The build compiles the Swift audio and desktop-integration helpers into the application output and publish `Native` folder; build or publish on macOS with Xcode Command Line Tools installed. The first capture requires both **Microphone** and **Screen & System Audio Recording** permissions under System Settings > Privacy & Security. Restart the application after granting a newly requested permission. `Recording:MicrophoneDeviceId` and runtime microphone selection remain Windows-only; macOS follows the system default input device. + The tray's fine-grained controls expose `Pause transcription` while a meeting is active and `Unpause transcription` while it is paused. Pause does not stop audio devices, finish the meeting, replace the speech-recognition pipeline, or reset run-local speaker mappings and collected samples. Instead, each captured mixed chunk is replaced with equal-length PCM silence before it reaches the temporary WAV, live speaker buffer, or configured transcription provider. This discards real audio from the paused interval while preserving the provider session and meeting-relative timing. In particular, Azure keeps the same active `ConversationTranscriber` and push stream; pause therefore does not suspend Azure connection time or billing. `Finish meeting` and cancel/discard remain available while paused, and `/recording/status` reports the state in `isPaused`. On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits. `Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV. -`Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings. +`Recording:MaxMetadataAttendeeImportCount` limits how many attendees calendar metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings. Windows reads Outlook Classic through COM. macOS reads EventKit calendars, including Outlook accounts synchronized into macOS Calendar, and requires **Calendar Full Access**. `Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text is written. Writing new transcript text also dismisses every outstanding inactivity notification and invalidates its actions. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with Yes, No, and `Pause transcription` actions, requests reminder-style toast behavior, keeps each stop reminder actionable for 1 minute, and automatically stops normally after 30 minutes. While transcription is intentionally paused, those prompts and the ordinary transcript-inactivity stop are fully suspended. A separate `MaximumPauseDuration`, defaulting to 4 hours, normally stops a meeting that remains continuously paused without showing an inactivity notification; it remains active when the ordinary inactivity safeguard is disabled, while a non-positive value disables the paused-session cutoff. Unpausing clears the continuous-pause timer and restarts transcript-inactivity timing from that moment. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When transcript inactivity stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding. @@ -341,20 +343,22 @@ See `docs/meeting-workflow-engine.md` for the detailed YAML format, supported va ## Calendar Recording Prompts -`CalendarRecordingPrompts` controls optional Outlook Classic calendar prompts for starting recordings. It is enabled by default on Windows. +`CalendarRecordingPrompts` controls optional calendar prompts for starting recordings. It is enabled by default on Windows and macOS. -When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Classic calendar appointments through COM, filters all non-canceled Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. It does not query Outlook for every prompt. The notification asks `Record meeting?` with `Yes` and `No` actions when a cached Teams meeting reaches its start window, requests reminder-style toast behavior, and remains actionable for 5 minutes. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply. +Meeting Assistant periodically syncs today's calendar appointments, filters all non-canceled Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. Windows reads Outlook Classic through COM and displays a Windows app notification. macOS prefers EventKit and displays a native AppKit `Record`/`Skip` alert; the first read requests Calendar Full Access. When EventKit access is denied, it falls back to Calendar application automation so an already-authorized local Calendar setup remains usable. It does not query the calendar for every prompt. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply. | Setting | Purpose | | --- | --- | -| `Enabled` | Enables scheduled Outlook Teams recording-start prompts. | -| `SyncInterval` | How often Meeting Assistant refreshes today's Outlook calendar cache. Defaults to 30 minutes when unset or invalid. | +| `Enabled` | Enables scheduled Teams recording-start prompts from the platform calendar. | +| `SyncInterval` | How often Meeting Assistant refreshes today's calendar cache. Defaults to 30 minutes when unset or invalid. | | `PromptWindow` | How long after a Teams appointment's scheduled start it remains eligible for the one-time prompt. Defaults to 5 minutes when unset or invalid. | ## Screenshots `Screenshots:Hotkey` configures a global hotkey that captures the currently active window during an active meeting. Screenshots are written under `Screenshots:AttachmentsFolder`, which defaults to an `Attachments` folder beside the assistant context note, and each capture appends a timestamped markdown image link to the assistant context. +Windows captures the foreground window through the existing Win32 provider. macOS captures the foremost on-screen window belonging to the frontmost application and requires **Screen & System Audio Recording** permission. Both paths feed the same attachment, OCR, crop, attendee-enrichment, retry, and summary-wait pipeline. + `Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Screenshot OCR also inherits `Agent:UseStreaming`, using the Responses SSE transport when it is `true` and the non-streaming Responses transport when it is `false`. Automatic summarization scans the meeting note for user-added Obsidian image embeds such as `![[whiteboard.png]]` and Markdown image embeds such as `![Diagram](attachments/diagram.png)`, adds resolvable local images to the assistant context without copying them or changing the meeting note, runs OCR without crop or attendee updates, and waits for all pending OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. Failed or timed-out screenshot OCR writes a retry link that targets `/meetings/screenshot-ocr/retry` with the exact saved screenshot and OCR block id. | Setting | Purpose | @@ -418,7 +422,8 @@ The top-level `Logging` section is standard ASP.NET Core logging configuration a Independently from stdout and stderr redirection, Meeting Assistant writes an application-owned log under: ```text -%TEMP%\MeetingAssistant\Logs\meeting-assistant.log +Windows: %TEMP%\MeetingAssistant\Logs\meeting-assistant.log +macOS: $TMPDIR/MeetingAssistant/Logs/meeting-assistant.log ``` On startup it rotates the previous current file to `meeting-assistant.log.1` and keeps up to `.4`. The tray-launched assistant reads and searches these current and rotated files. If another app instance or test host still has the file open, rotation is skipped and logging appends to the current file so startup is not blocked. @@ -426,3 +431,5 @@ On startup it rotates the previous current file to `meeting-assistant.log.1` and ## API `Api:PublicBaseUrl` is used to build clickable local links, such as summary retry links in generated failure notes. + +`GET /diagnostics/platform-capabilities` reports the detected host, concrete metadata/calendar/prompt/screenshot/workflow-editor provider names, explicit workflow-editor availability, and effective enabled switches for calendar prompts, speaker identification, screenshot OCR, the FunASR backend, FunASR diarization, and Whisper-local diarization. It is read-only and is the quickest way to confirm that a platform is not using no-op providers. diff --git a/docs/meeting-workflow-engine.md b/docs/meeting-workflow-engine.md index d091405..27a9d14 100644 --- a/docs/meeting-workflow-engine.md +++ b/docs/meeting-workflow-engine.md @@ -26,6 +26,8 @@ If the configured path is empty, missing, or points to a blank file, the workflo The tray menu includes `Open agent`, which opens the `Meeting Summary Agent` chat window for this configured rules file, the local speaker identity database, appsettings configuration, and application logs. The assistant uses the summarizer agent configuration by default and can be overridden through `MeetingAssistant:WorkflowRulesEditor`. +Meeting Assistant detects the host operating system once during application startup and appends that context to the interactive agent prompt. Windows hosts receive PowerShell, Windows-path, Windows-service, and Task Manager guidance. macOS hosts receive zsh, POSIX-path, launchd/LaunchAgent, and Activity Monitor guidance. Other hosts receive neutral portable guidance. The platform context remains present when `WorkflowRulesEditor:InitialPrompt` is configured, and it does not imply that unavailable platform integrations have been implemented. + ```json { "MeetingAssistant": { diff --git a/global.json b/global.json new file mode 100644 index 0000000..512142d --- /dev/null +++ b/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature" + } +} diff --git a/openspec/changes/add-macos-desktop-controls/design.md b/openspec/changes/add-macos-desktop-controls/design.md new file mode 100644 index 0000000..6260f4d --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/design.md @@ -0,0 +1,23 @@ +## Context + +The application already exposes local HTTP endpoints for recording lifecycle actions. Windows desktop integrations invoke the coordinator in-process, while macOS currently has no desktop process capable of participating in AppKit or Carbon event loops. + +## Decisions + +### Native helper owns macOS desktop APIs + +A bundled Swift helper owns the AppKit status item and Carbon global-hotkey registrations. The .NET host starts the helper only on macOS and terminates it during shutdown. + +### Existing HTTP endpoints remain the control seam + +The helper invokes loopback endpoints for recording and diagnostic actions. It does not duplicate recording state or lifecycle policy. It polls the status endpoint to keep the menu-bar presentation current. + +### Configuration is passed as an explicit manifest + +The .NET host derives a manifest from configured launch profiles. Shortcut strings are passed unchanged so macOS uses the same bindings displayed and registered on Windows. + +## Verification + +- Behavior tests prove the manifest preserves configured shortcuts and maps actions to the existing endpoints. +- The Swift helper supports a describe/self-test mode for deterministic packaging validation. +- Runtime verification launches the application, starts a new recording, and captures the visible macOS menu-bar recording state. diff --git a/openspec/changes/add-macos-desktop-controls/implementation-evidence.md b/openspec/changes/add-macos-desktop-controls/implementation-evidence.md new file mode 100644 index 0000000..5ce33bb --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/implementation-evidence.md @@ -0,0 +1,98 @@ +# Implementation Evidence + +## Automated + +- `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --filter FullyQualifiedName~MacOsDesktopControlManifestTests --no-restore` + - Passed: 1, Failed: 0. +- `dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 --no-restore` + - Passed and produced `Native/macos-desktop-controls` as an arm64 Mach-O executable. +- `openspec validate add-macos-desktop-controls --strict` + - Passed. +- Full portable test project: + - Passed: 427. + - Failed: 9 existing macOS/environment-dependent tests: Windows path expectations, unavailable GDI+, and user-environment key resolution. +- Whole solution: + - Cannot build the Windows target on macOS without enabling cross-Windows targeting (`NETSDK1100`). + +## Runtime + +- Installed the rebuilt application assembly and both native helpers into the existing local LaunchAgent application directory. +- LaunchAgent restarted successfully. +- `/health` returned `ok`. +- `macos-desktop-controls` ran as a child companion process. +- The helper manifest contained six configured hotkeys, including the unchanged defaults `Ctrl+Alt+M`, `Ctrl+Alt+L`, `Ctrl+Alt+Z`, and `Ctrl+Alt+S`. +- A fresh default-profile recording request created the `20260724-2015` meeting note and linked artifacts. +- Screenshot evidence: `tmp/macos-desktop-recording-verification.png`. + +## Live desktop-control proof (2026-07-24) + +- The installed LaunchAgent ran: + - `.NET host` PID `23023` + - native `macos-desktop-controls` PID `23097` +- The native helper manifest exposed: + - `Ctrl+Alt+M` -> `/profiles/default/recording/toggle` + - `Ctrl+Alt+L` -> `/profiles/english/recording/toggle` + - `Ctrl+Alt+Z` -> `/profiles/default/recording/abort` + - `Ctrl+Alt+S` -> `/profiles/default/meetings/screenshot/capture` +- Accessibility inspection of the real AppKit menu returned: + - `Meeting Assistant — idle` + - `Open agent` + - `Start meeting recording (default)` + - `Start meeting recording (english)` + - `Exit` +- Visible menu screenshot: + - `tmp/proof-macos-menu-idle.png` + - The menu visibly displays the macOS equivalents `⌃⌥M` and `⌃⌥L`. +- Clicking `Start meeting recording (default)` through the real menu produced the sampled status transition `false,true,false`, with `launchProfile: default`, and created: + - `Meetings/Notes/20260724-2022-note.md` + - `Meetings/Transcripts/20260724-2022-transcript.md` + - `Meetings/Assistant Context/20260724-2022-context.md` +- Posting the HID-level `Ctrl+Alt+M` combination produced the sampled status transition `false,true,false`, with `launchProfile: default`. +- Posting the HID-level `Ctrl+Alt+L` combination produced an active sampled state with `launchProfile: english`. +- Raw sampled status evidence: + - `tmp/proof-menu-recording-status.jsonl` + - `tmp/proof-hotkey-recording-status.jsonl` + - `tmp/proof-english-hotkey-status.jsonl` + +## Runtime limitation + +The new recording did not remain active because the local audio capture process stopped shortly after artifact creation. The desktop-control helper, menu-bar item, endpoint routing, and meeting creation were verified; sustained audio capture was not. + +## Open agent regression and runtime proof + +- Original deterministic regression: + - `MacOsApplicationRegistersARealInteractiveAgentWindow` + - Red result: portable registration resolved exactly `NoopWorkflowRulesEditorWindowService`. +- Fix: + - macOS now registers `MacOsWorkflowRulesEditorWindowService`. + - `Open agent` in the native helper creates an AppKit `NSWindow` titled `Meeting Summary Agent`. + - The window hosts a WebKit chat page backed by the existing `WorkflowRulesEditorChatViewModel` and agent pipeline. +- Green regression: + - Passed: 1, Failed: 0. +- Live installed runtime: + - Clicking the actual menu item created an on-screen CoreGraphics window: + - owner: `macos-desktop-controls` + - title: `Meeting Summary Agent` + - layer: `0` + - bounds: `756x714` + - `GET /diagnostics/settings-and-logs` returned the interactive agent HTML page. + +## Open agent prompt execution proof (2026-07-25) + +- Original live failure: + - `Meeting Summary Agent failed: No workflow rules editor API key configured.` +- Root cause: + - The LaunchAgent did not inherit the interactive shell's model API key. + - Its configured `127.0.0.1:4021` model endpoint had no running service. +- Runtime configuration: + - The protected LaunchAgent environment file now supplies the local LiteLLM gateway key. + - `MeetingAssistant__WorkflowRulesEditor__Endpoint` targets the running local gateway on port `4000`. + - The local gateway exposes the Responses-compatible `chatgpt-gpt-5.5` model alias. +- Event-stream compatibility regression: + - `ClientReconstructsResponseFromServerSentEvents` failed before the fix with `JsonReaderException`. + - The client now reconstructs completed response items from LiteLLM server-sent events. + - All 11 `LiteLlmResponsesChatClientTests` pass. +- Live installed proof: + - `POST /diagnostics/settings-and-logs/chat` with `Reply with exactly: key-ok` + returned `{"role":"agent","content":"key-ok"}`. + - `/health` returned successfully after deployment. diff --git a/openspec/changes/add-macos-desktop-controls/proposal.md b/openspec/changes/add-macos-desktop-controls/proposal.md new file mode 100644 index 0000000..8d8593e --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/proposal.md @@ -0,0 +1,24 @@ +## Why + +Meeting Assistant can record audio on macOS, but the portable build does not register the configured global shortcuts or provide the menu-bar controls used for the normal Windows desktop workflow. macOS users must call HTTP endpoints manually. + +## What Changes + +- Add a native macOS menu-bar companion that starts and stops with Meeting Assistant. +- Register the same configured recording, abort, and screenshot shortcuts used on Windows. +- Expose recording/profile controls, `Open agent`, and exit from the macOS menu bar. +- Keep recording lifecycle behavior behind the existing local HTTP/service surface. + +## Capabilities + +### Modified Capabilities + +- `meeting-recording`: configured global recording controls work on macOS. +- `meeting-session`: the desktop control menu is available from the macOS menu bar. + +## Impact + +- `MeetingAssistant/Native/MacOsDesktopControls` +- portable macOS build/publish output +- startup service registration +- desktop-control behavior tests and macOS runtime documentation diff --git a/openspec/changes/add-macos-desktop-controls/specs/meeting-recording/spec.md b/openspec/changes/add-macos-desktop-controls/specs/meeting-recording/spec.md new file mode 100644 index 0000000..648af53 --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/specs/meeting-recording/spec.md @@ -0,0 +1,20 @@ +## MODIFIED Requirements + +### Requirement: Recording mode is controlled by a configurable hotkey +Meeting Assistant SHALL use normal .NET configuration to define global hotkeys that work on Windows and macOS and toggle recording/transcription mode. + +Meeting Assistant SHALL preserve the configured shortcut strings across Windows and macOS, including the defaults `Ctrl+Alt+M`, `Ctrl+Alt+L`, `Ctrl+Alt+Z`, and `Ctrl+Alt+S`. + +Meeting Assistant SHALL expose a configurable abort/discard hotkey for an active recording on Windows and macOS. + +#### Scenario: macOS uses the configured Windows-equivalent shortcuts +- **GIVEN** the default and English profiles use `Ctrl+Alt+M` and `Ctrl+Alt+L` +- **AND** abort and screenshot use `Ctrl+Alt+Z` and `Ctrl+Alt+S` +- **WHEN** Meeting Assistant starts on macOS +- **THEN** it registers those same global shortcut combinations +- **AND** routes them to toggle, abort, and screenshot actions through the local service surface + +#### Scenario: macOS recording hotkey starts recording +- **GIVEN** Meeting Assistant is idle on macOS +- **WHEN** the user presses the configured recording hotkey +- **THEN** Meeting Assistant starts recording/transcription mode diff --git a/openspec/changes/add-macos-desktop-controls/specs/meeting-session/spec.md b/openspec/changes/add-macos-desktop-controls/specs/meeting-session/spec.md new file mode 100644 index 0000000..7190b39 --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/specs/meeting-session/spec.md @@ -0,0 +1,28 @@ +## MODIFIED Requirements + +### Requirement: Workflow rules and speaker identities can be edited through a tray-launched assistant +Meeting Assistant SHALL expose desktop controls from the Windows tray icon and the macOS menu bar. + +The macOS menu-bar control SHALL start with Meeting Assistant, show current recording state, expose each configured launch profile with its configured hotkey, expose stop and abort controls while recording, expose `Open agent`, and expose exit. + +Selecting `Open agent` on macOS SHALL open a native window titled `Meeting Summary Agent` containing the interactive settings-and-logs chat surface. + +#### Scenario: macOS menu bar shows idle recording controls +- **GIVEN** Meeting Assistant is idle on macOS +- **WHEN** the user opens the menu-bar control +- **THEN** it shows `Open agent` +- **AND** shows a start action for every configured launch profile +- **AND** displays each profile's configured toggle hotkey +- **AND** shows an exit action + +#### Scenario: macOS menu bar shows active recording controls +- **GIVEN** a meeting is recording on macOS +- **WHEN** the user opens the menu-bar control +- **THEN** it shows the active recording state +- **AND** exposes stop and abort actions + +#### Scenario: macOS menu opens the interactive agent +- **GIVEN** Meeting Assistant is running on macOS +- **WHEN** the user selects `Open agent` from the menu-bar control +- **THEN** a `Meeting Summary Agent` window opens +- **AND** the user can submit a chat message and receive the interactive agent response diff --git a/openspec/changes/add-macos-desktop-controls/tasks.md b/openspec/changes/add-macos-desktop-controls/tasks.md new file mode 100644 index 0000000..c125925 --- /dev/null +++ b/openspec/changes/add-macos-desktop-controls/tasks.md @@ -0,0 +1,18 @@ +## 1. Contract and behavior + +- [x] 1.1 Define macOS desktop-control requirements and design. +- [x] 1.2 Add a failing behavior test for configured shortcut and endpoint preservation. +- [x] 1.3 Implement the desktop-control manifest and make the behavior test pass. + +## 2. Native macOS integration + +- [x] 2.1 Add and package the native macOS menu-bar/global-hotkey helper. +- [x] 2.2 Start and stop the helper with the .NET application on macOS. +- [x] 2.3 Add deterministic helper packaging/self-test coverage. + +## 3. Verification and closeout + +- [x] 3.1 Run narrow and full tests. +- [x] 3.2 Run strict OpenSpec validation. +- [x] 3.3 Launch Meeting Assistant, start a new recording, and save screenshot evidence. +- [x] 3.4 Perform required DRY, SOLID, and KISS refactoring reviews and re-run verification. diff --git a/openspec/changes/add-macos-meeting-integrations/design.md b/openspec/changes/add-macos-meeting-integrations/design.md new file mode 100644 index 0000000..f0a923e --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/design.md @@ -0,0 +1,37 @@ +## Context + +The macOS build already bundles Swift helpers for audio and menu-bar controls. Calendar and screenshot integrations are currently replaced with no-op providers at compile time, while the ASR, diarization, speaker, OCR, and workflow-agent implementations themselves are portable. + +## Decisions + +### EventKit is the preferred macOS calendar source + +A bundled Swift helper reads events through EventKit. This covers calendars exposed to macOS, including Outlook accounts synchronized into Calendar, without linking Windows COM into the portable target. If Calendar Full Access is denied, the managed client falls back to the already-authorized Calendar application's automation dictionary. The helper uses a stable permission-denied exit code rather than coupling the fallback to human-readable error text. The managed providers retain the existing Teams-marker and current-meeting selection rules. + +### Native AppKit owns the recording prompt + +The same helper displays a record/skip AppKit alert and returns the selected response to the existing `IMeetingStartPromptService` callback. Recording policy remains in the existing scheduler/controller. + +### CoreGraphics captures the active app window + +The helper resolves the frontmost application and captures its foremost on-screen window as PNG. The managed screenshot service continues to own attachment naming, note updates, OCR, cropping, attendee enrichment, retries, and waits. + +### Portable managed features stay managed + +Screenshot OCR, speaker identity, Docker-backed FunASR, FunASR diarization, Whisper-local pyannote diarization, and the AppKit/WebKit workflow editor use the same managed implementations on macOS and Windows. Platform branching is limited to the native input providers. + +### Diagnostics report effective behavior + +A read-only endpoint returns the host platform, concrete provider names, and effective enabled switches. This supplies deterministic proof without starting a recording or mutating meeting artifacts. + +## Windows compatibility + +The Windows target keeps its existing conditional registrations and does not compile or invoke the macOS helper. Verification must build the Windows target in addition to the portable target and tests. + +## Verification + +- Regression tests prove macOS resolves real providers and reports enabled capabilities. +- Provider behavior tests prove EventKit JSON maps to calendar metadata, native prompt responses reach the callback, and PNG screenshot output reaches the screenshot interface. +- The Swift helper provides deterministic self-test output. +- A live macOS instance proves health, provider diagnostics, native helper packaging, calendar access, and screenshot capture. +- A Windows-target build proves the existing Windows client still compiles with its native providers. diff --git a/openspec/changes/add-macos-meeting-integrations/implementation-evidence.md b/openspec/changes/add-macos-meeting-integrations/implementation-evidence.md new file mode 100644 index 0000000..2412c0f --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/implementation-evidence.md @@ -0,0 +1,47 @@ +## Verification evidence + +Verified on macOS 26 on 2026-07-25. + +### Automated checks + +- Focused macOS/calendar/screenshot/OCR tests: 35 passed, 0 failed. +- Full portable test project: 441 passed, 6 failed, 447 total. The six failures are existing macOS-incompatible tests for Windows path assumptions, GDI+ taskbar rendering, and user-environment fallback; none cover this change. +- Portable `net10.0` build: succeeded. +- Windows `net10.0-windows10.0.19041.0` target build with Windows targeting enabled: succeeded with 0 warnings and 0 errors. +- `openspec validate add-macos-meeting-integrations --strict`: valid. +- Sequential DRY, SOLID, and KISS reviews were completed and their findings implemented. +- Final independent standards and spec reviews reported no findings. + +### Live macOS installation + +The recording status was idle before deployment. The repository updater published the portable build, installed it into `~/Library/Application Support/MeetingAssistant/app`, restarted the LaunchAgent, and passed its health check. + +`GET /diagnostics/platform-capabilities` returned: + +```json +{ + "host": "macOS", + "meetingMetadataProvider": "MacOsMeetingMetadataProvider", + "calendarMeetingProvider": "MacOsCalendarMeetingProvider", + "meetingStartPromptProvider": "MacOsMeetingStartPromptService", + "screenshotProvider": "MacOsDesktopIntegrationClient", + "workflowEditorProvider": "MacOsWorkflowRulesEditorWindowService", + "workflowEditorAvailable": true, + "calendarRecordingPromptsEnabled": true, + "speakerIdentificationEnabled": true, + "screenshotOcrEnabled": true, + "funAsrBackendEnabled": true, + "funAsrDiarizationEnabled": true, + "whisperLocalDiarizationEnabled": true +} +``` + +The installed arm64 Mach-O helper returned: + +```json +{"features":["calendar","prompt","screenshot","crop"],"status":"ok"} +``` + +Direct native screenshot capture selected the foreground ChatGPT window and produced a valid 3258 by 1920 PNG. The asymmetric native crop regression test also proves OCR top-left pixel coordinates are preserved. + +The live calendar diagnostic returned `provider=MacOsMeetingMetadataProvider`, `timedOut=false`, and a normal no-matching-event result. EventKit reported missing Full Calendar Access with stable exit code 77, after which the Calendar automation fallback completed; the scheduler logged `Synced 0 Teams calendar meetings`. diff --git a/openspec/changes/add-macos-meeting-integrations/proposal.md b/openspec/changes/add-macos-meeting-integrations/proposal.md new file mode 100644 index 0000000..3cef2cb --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/proposal.md @@ -0,0 +1,29 @@ +## Why + +The portable macOS application can capture meeting audio and expose menu-bar controls, but it still registers unavailable/no-op implementations for calendar metadata, recording prompts, and active-window screenshots. The local macOS environment also disables screenshot OCR, speaker identification, the managed FunASR backend, FunASR diarization, and Whisper-local diarization even though their managed implementations are portable. + +## What Changes + +- Read Teams meeting metadata and prompt candidates from calendars available through macOS EventKit, including Outlook-synced calendars. +- Show a native macOS record/skip prompt for due calendar meetings. +- Capture the foreground macOS application window through a bundled native helper so screenshot OCR can use the existing portable OCR pipeline. +- Register the real macOS integrations instead of no-op/unavailable providers. +- Enable the portable OCR, speaker identification, FunASR, and diarization configuration in the macOS runtime environment. +- Expose a diagnostic capability surface that proves the effective platform providers and enabled feature switches. +- Preserve all Windows-specific COM, notification, screenshot, tray, hotkey, and audio registrations. + +## Capabilities + +### Modified Capabilities + +- `meeting-session`: calendar enrichment, recording prompts, workflow editor access, and capability diagnostics work on macOS. +- `meeting-summary`: active-window screenshots and screenshot OCR work on macOS. +- `meeting-transcription`: portable speaker identification, FunASR, and local diarization can be enabled on macOS. + +## Impact + +- portable service registration in `MeetingAssistant/Program.cs` +- a bundled native macOS desktop-integration helper +- calendar, meeting-metadata, screenshot, and diagnostic providers +- macOS runtime configuration and operational documentation +- macOS behavior tests plus Windows-target build verification diff --git a/openspec/changes/add-macos-meeting-integrations/specs/meeting-session/spec.md b/openspec/changes/add-macos-meeting-integrations/specs/meeting-session/spec.md new file mode 100644 index 0000000..446c3fd --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/specs/meeting-session/spec.md @@ -0,0 +1,41 @@ +## MODIFIED Requirements + +### Requirement: Meeting metadata is captured when available +Meeting Assistant SHALL enrich a meeting from a matching calendar appointment when platform calendar metadata is available. + +On Windows, Meeting Assistant SHALL retain Outlook Classic COM metadata lookup. + +On macOS, Meeting Assistant SHALL read calendars available through EventKit, including Outlook-synced calendars, and SHALL apply the existing Teams-marker and current-meeting selection rules. + +#### Scenario: macOS meeting uses calendar metadata +- **GIVEN** an EventKit calendar contains a Teams appointment spanning the meeting start time +- **WHEN** Meeting Assistant enriches the meeting on macOS +- **THEN** it applies the appointment title, attendees, agenda, and scheduled end time + +#### Scenario: Windows metadata provider remains unchanged +- **GIVEN** Meeting Assistant is compiled for the Windows target +- **WHEN** meeting metadata is registered +- **THEN** it uses the Outlook Classic COM provider +- **AND** does not invoke or require the macOS native helper + +### Requirement: Meeting Assistant can prompt to record scheduled Teams meetings +Meeting Assistant SHALL optionally inspect the host calendar for scheduled Teams meetings and show a native record-or-skip prompt when a meeting enters the configured prompt window. + +On Windows, Meeting Assistant SHALL retain Outlook Classic COM lookup and Windows app notifications. + +On macOS, Meeting Assistant SHALL use EventKit calendar events and a native AppKit record-or-skip prompt. + +#### Scenario: macOS calendar meeting becomes due +- **GIVEN** calendar recording prompts are enabled on macOS +- **AND** an EventKit Teams appointment enters the prompt window +- **WHEN** the scheduler checks due meetings +- **THEN** Meeting Assistant shows the native macOS prompt +- **AND** starts recording with the appointment metadata when the user chooses record + +### Requirement: Platform capabilities are diagnosable +Meeting Assistant SHALL expose a read-only diagnostic response containing the detected host, effective integration provider names, and enabled state of calendar prompts, speaker identification, screenshot OCR, FunASR backend, FunASR diarization, Whisper-local diarization, and workflow editor availability. + +#### Scenario: macOS reports real integration providers +- **WHEN** the capability diagnostic is requested on macOS +- **THEN** it reports real macOS calendar metadata, calendar prompt, screenshot, and workflow editor providers +- **AND** does not report the no-op or unavailable provider names diff --git a/openspec/changes/add-macos-meeting-integrations/specs/meeting-summary/spec.md b/openspec/changes/add-macos-meeting-integrations/specs/meeting-summary/spec.md new file mode 100644 index 0000000..52a3c03 --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/specs/meeting-summary/spec.md @@ -0,0 +1,23 @@ +## MODIFIED Requirements + +### Requirement: Meeting screenshots are captured into assistant context +Meeting Assistant SHALL capture the active application window on supported Windows and macOS hosts and append it to the active meeting assistant context. + +On Windows, Meeting Assistant SHALL retain the existing foreground-window capture implementation. + +On macOS, Meeting Assistant SHALL capture the foremost on-screen window belonging to the frontmost application through native macOS APIs. + +The existing screenshot OCR, crop, attendee enrichment, retry, and summary-wait behavior SHALL apply unchanged to macOS screenshots. + +#### Scenario: macOS captures the active window for OCR +- **GIVEN** an active meeting on macOS and Screen Recording permission +- **WHEN** the screenshot hotkey or endpoint is invoked +- **THEN** Meeting Assistant saves a PNG of the frontmost application window +- **AND** appends it to assistant context +- **AND** starts OCR when screenshot OCR is enabled + +#### Scenario: Windows screenshot provider remains unchanged +- **GIVEN** Meeting Assistant is compiled for the Windows target +- **WHEN** active-window screenshot capture is registered +- **THEN** it uses the existing Windows foreground-window provider +- **AND** does not invoke or require the macOS native helper diff --git a/openspec/changes/add-macos-meeting-integrations/specs/meeting-transcription/spec.md b/openspec/changes/add-macos-meeting-integrations/specs/meeting-transcription/spec.md new file mode 100644 index 0000000..715cf8a --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/specs/meeting-transcription/spec.md @@ -0,0 +1,18 @@ +## MODIFIED Requirements + +### Requirement: Portable transcription backends and speaker processing are available on macOS +Meeting Assistant SHALL allow speaker identification, managed FunASR startup, FunASR diarization, and Whisper-local pyannote diarization to run on macOS using the same managed implementations and configuration contracts as Windows. + +These capabilities SHALL remain independently configurable and SHALL report actionable dependency errors when Docker, model files, API credentials, or model-access credentials are missing. + +#### Scenario: macOS enables configured portable processing +- **GIVEN** the portable build is running on macOS +- **AND** speaker identification, managed FunASR, FunASR diarization, or Whisper-local diarization is enabled +- **WHEN** the corresponding transcription path runs +- **THEN** Meeting Assistant invokes the existing managed implementation +- **AND** does not disable the capability solely because the host is macOS + +#### Scenario: Windows processing behavior remains unchanged +- **GIVEN** Meeting Assistant is compiled for the Windows target +- **WHEN** portable transcription and speaker-processing services are registered +- **THEN** it retains the existing managed implementations and configuration behavior diff --git a/openspec/changes/add-macos-meeting-integrations/tasks.md b/openspec/changes/add-macos-meeting-integrations/tasks.md new file mode 100644 index 0000000..e2e6393 --- /dev/null +++ b/openspec/changes/add-macos-meeting-integrations/tasks.md @@ -0,0 +1,26 @@ +## 1. Contract and regression coverage + +- [x] 1.1 Define macOS integration requirements and design. +- [x] 1.2 Add failing registration and capability-diagnostic tests. +- [x] 1.3 Add failing calendar, prompt, screenshot, and helper-contract tests. + +## 2. Native and managed implementation + +- [x] 2.1 Add and package the native macOS calendar/prompt/screenshot helper. +- [x] 2.2 Add managed macOS calendar metadata and prompt providers. +- [x] 2.3 Add managed macOS active-window screenshot provider. +- [x] 2.4 Register the real providers on macOS while preserving unsupported-host no-ops. +- [x] 2.5 Add the read-only platform-capability diagnostic endpoint. + +## 3. Configuration and documentation + +- [x] 3.1 Enable the portable OCR, speaker, FunASR, and diarization switches in the macOS runtime. +- [x] 3.2 Document macOS permissions, calendar source behavior, and diagnostics. + +## 4. Verification and closeout + +- [x] 4.1 Run narrow tests and portable build. +- [x] 4.2 Run the full test project and strict OpenSpec validation. +- [x] 4.3 Build the Windows target. +- [x] 4.4 Verify the live macOS application through direct operational surfaces. +- [x] 4.5 Perform required DRY, SOLID, KISS, and final code reviews. diff --git a/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/design.md b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/design.md new file mode 100644 index 0000000..e78966d --- /dev/null +++ b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/design.md @@ -0,0 +1,49 @@ +## Context + +The recording pipeline already depends on `IMeetingAudioSource` and composes independent microphone and system streams through `CompositeMeetingAudioSource`. Windows supplies those streams with NAudio behind the `WINDOWS` compilation boundary. The portable target currently supplies an unavailable source on every non-Windows host. + +.NET does not ship ScreenCaptureKit or AVFoundation bindings. A small native process is therefore the narrowest reliable adapter between Apple's capture frameworks and the existing managed PCM stream contract. + +## Goals / Non-Goals + +**Goals:** + +- Capture the default macOS microphone and computer output without a virtual audio device. +- Emit the configured sample rate/channel count as signed 16-bit PCM. +- Reuse the existing managed alignment, AEC, gain, mixing, WAV, and transcription path. +- Keep Windows source files, registrations, packages, and runtime behavior unchanged. +- Fail with actionable privacy-permission or helper-packaging diagnostics. + +**Non-Goals:** + +- Add a macOS tray icon, global hotkeys, Outlook metadata, notifications, or screenshots. +- Add runtime macOS microphone selection in this change. +- Replace or refactor the Windows NAudio implementation. +- Commit long-lived user meeting audio; the proof WAV is a short implementation artifact only. + +## Decisions + +### Reuse the existing composite source + +macOS registers separate microphone and system `IMeetingAudioSource` adapters and feeds them to `CompositeMeetingAudioSource`. This preserves the established audio semantics and keeps platform code limited to capture and PCM conversion. + +### Use one native helper with two modes + +The bundled Swift helper accepts `microphone` or `system`, plus sample rate and channel count, and writes headerless signed 16-bit PCM to standard output. AVFoundation supplies microphone buffers. ScreenCaptureKit supplies system-audio buffers. AVAudioConverter normalizes both sources before managed code reads them. + +The managed adapter owns process lifetime, reads PCM chunks asynchronously, terminates the helper when capture is cancelled, and surfaces native diagnostics when the helper exits unexpectedly. + +### Build the helper only on macOS + +The Swift source is portable content, but compilation is conditioned on the build host being macOS. macOS build and publish output receives the executable under `Native`. Windows compilation remains within its existing `WINDOWS` branch and does not compile or invoke Swift code. + +### Keep unsupported-host behavior + +The portable target chooses macOS capture with a runtime OS check. Linux and other hosts continue resolving `UnavailableMeetingAudioSource` with an updated host-neutral error. + +## Risks / Trade-offs + +- [macOS privacy controls can initially deny or pause capture] → Request microphone authorization and report explicit Screen Recording/System Audio or Microphone guidance on stderr. +- [A helper process adds a packaging boundary] → Make the build fail on macOS if Swift compilation fails and verify the helper exists in build and publish output. +- [Native audio buffer formats vary] → Convert the CoreMedia/AVFoundation buffers with `AVAudioConverter` to the exact requested PCM format before writing. +- [Windows regressions from shared startup edits] → Leave the `#if WINDOWS` branch intact and build the Windows target in addition to portable tests. diff --git a/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/implementation-evidence.md b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/implementation-evidence.md new file mode 100644 index 0000000..39b5401 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/implementation-evidence.md @@ -0,0 +1,52 @@ +## Implementation Evidence + +Date: 2026-07-21 +Host: macOS arm64 + +### Live audio verification + +- The bundled Swift helper compiled as an arm64 Mach-O executable and was launched directly from the portable application build output. +- AVFoundation microphone capture emitted 546,920 bytes of signed 16-bit, 16 kHz mono PCM during the final 17-second proof window. The selected default input was `Jabra Link 390`; its captured samples were all zero, consistent with a muted/silent input rather than a capture-process failure. +- ScreenCaptureKit system capture emitted 551,680 bytes of signed 16-bit, 16 kHz mono PCM while the repository's known `sample-16khz-mono.wav` fixture was played 12 times through the default system output. +- The captured streams were aligned to their shared duration, summed with 16-bit clamping, and written to `docs/evidence/macos-meeting-audio-proof.wav`. +- `afinfo` reports the proof as 17.091250 seconds, mono, 16000 Hz, signed 16-bit PCM, with 546,920 audio bytes. +- Signal inspection found 48,133 non-zero samples and a peak amplitude of 12,000 in the proof WAV. +- SHA-256: `f38285a5386067f0c2637698f2e3c720ad30b65cebaaefe8d00454c78d8c2d05`. + +### Behavior and packaging verification + +- Public recording endpoint behavior: + - Test: `RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses` calls `/recording/start`, observes known microphone/system samples combined into one `12000` PCM sample at the speech-pipeline boundary, calls `/recording/stop`, and verifies both native capture processes were terminated. + - The test uses deterministic process-boundary audio because automated test hosts cannot depend on live devices or macOS privacy state. +- Focused macOS-source, public endpoint, and existing mixer tests: + - Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --filter 'FullyQualifiedName~MacOsMeetingAudioSourceTests|FullyQualifiedName~AudioMixingTests' --no-restore --nologo` + - Result: passed, 11/11. +- Portable application build: + - Command: `dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 --no-restore --nologo` + - Result: succeeded with 0 warnings and 0 errors; the Swift helper was compiled into the application output. +- macOS publish: + - Command: `dotnet publish MeetingAssistant/MeetingAssistant.csproj -f net10.0 -r osx-arm64 --self-contained false -p:EnableWindowsTargeting=true -o tmp/macos-publish-proof --nologo` + - Result: succeeded; `Native/macos-meeting-audio-capture` is an executable arm64 Mach-O file in publish output. +- OpenSpec validation: + - Command: `openspec validate add-macos-meeting-audio-capture --strict` + - Result: valid. + +### Windows isolation verification + +- The existing `#if WINDOWS` registration continues to use `MicrophoneAudioSource`, `SystemAudioSource`, `AdaptiveFilterAcousticEchoCancellerFactory`, and `CompositeMeetingAudioSource`. macOS selection exists only inside the portable `#else` branch. +- Windows C# compilation: + - Command: `dotnet msbuild MeetingAssistant/MeetingAssistant.csproj -t:Compile -p:TargetFramework=net10.0-windows10.0.19041.0 -p:EnableWindowsTargeting=true -nologo -v:minimal` + - Result: succeeded. +- A full Windows-target build cannot finish on macOS because the Windows App SDK attempts to execute `MakePri.exe` and returns `Exec format error`. This occurs after managed compilation and is the repository's known cross-host limitation; Windows CI or a Windows host remains the full binary verification environment. + +### Full portable test result + +- Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --no-restore --nologo` +- Result: 425 passed and 10 failed in the combined run. One unrelated inactivity-timer test was then rerun in isolation and passed (1/1), leaving the effective result at 426 passing tests and the same 9 unrelated baseline failures. +- The 9 failures are the same pre-existing macOS baseline recorded by the active `detect-host-operating-system` change: Windows/GDI+ image rendering, Windows-path expectations, and Windows user-environment behavior. None exercises macOS audio capture or the shared mixer. + +### Operational verification boundary + +The live helpers were verified while the macOS display session was awake and produced the committed proof signal. The public endpoint path was verified deterministically in-process through the real registration, macOS source adapters, shared mixer, coordinator, and start/stop endpoints. A later attempt to repeat live ScreenCaptureKit capture under the transient `vstest`/console proof host could not enumerate a display after the built-in display entered the asleep state; ScreenCaptureKit returned `ScreenCaptureKit could not find a display for system-audio capture`. The production port-5090 service was deliberately not restarted or reconfigured with fake ASR dependencies. These are the reasons live native input and the public application path were verified in two complementary passes rather than one production meeting run. + +The live Meeting Assistant service was not restarted, and no active recording or meeting artifacts were modified during verification. diff --git a/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/proposal.md b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/proposal.md new file mode 100644 index 0000000..229a420 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/proposal.md @@ -0,0 +1,30 @@ +## Why + +The portable Meeting Assistant build runs on macOS, but starting a recording resolves `UnavailableMeetingAudioSource` and fails before any audio reaches transcription. macOS users need the same microphone-plus-computer-output capture contract without changing the established Windows NAudio implementation. + +## What Changes + +- Add native macOS microphone capture through AVFoundation. +- Add native macOS computer-output capture through ScreenCaptureKit. +- Stream both sources as 16-bit PCM into the existing alignment, echo-cancellation, gain, mixing, recording, and transcription pipeline. +- Select the macOS sources only when the portable build is running on macOS; retain the existing compile-time Windows registrations unchanged. +- Package a small Swift capture helper in macOS build and publish output. +- Produce a short mixed WAV from live microphone and system PCM emitted by the implemented native capture helper as operational evidence. +- Verify the public recording endpoints, managed macOS adapters, mixer, and stop behavior together with deterministic process-boundary audio. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `meeting-recording`: Extend microphone and computer-output recording to macOS while preserving Windows behavior. + +## Impact + +- Portable application dependency registration. +- A macOS-native helper executable built from Swift source. +- macOS Screen Recording/System Audio and Microphone privacy permissions. +- Recording behavior tests, build verification, and runtime documentation. diff --git a/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/specs/meeting-recording/spec.md new file mode 100644 index 0000000..ef74c79 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/specs/meeting-recording/spec.md @@ -0,0 +1,55 @@ +## MODIFIED Requirements + +### Requirement: Recording mode captures microphone and computer output +Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription on supported Windows and macOS hosts. + +Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline. + +Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks. + +On Windows, Meeting Assistant SHALL retain the existing NAudio microphone and WASAPI loopback implementation. + +On macOS, Meeting Assistant SHALL capture the default microphone through AVFoundation and computer output through ScreenCaptureKit without requiring a virtual audio device. + +The macOS capture adapter SHALL convert both native sources to signed 16-bit PCM using the active run's configured sample rate and channel count before passing chunks to the existing managed mixing pipeline. + +The macOS capture adapter SHALL stop its native capture process when the recording capture token is cancelled. + +When macOS privacy permission is missing or denied, Meeting Assistant SHALL fail capture with an actionable message identifying the required Microphone or Screen Recording/System Audio permission. + +Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference. + +Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples. + +Meeting Assistant SHALL align microphone and system samples through per-source buffers before producing final chunks and SHALL apply the existing configured gains and clamping behavior. + +Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization. + +#### Scenario: macOS records both meeting-audio sources +- **GIVEN** the portable build is running on macOS +- **AND** Microphone and Screen Recording/System Audio permissions are granted +- **WHEN** a meeting recording starts +- **THEN** Meeting Assistant captures microphone audio through AVFoundation +- **AND** captures computer output through ScreenCaptureKit +- **AND** passes both signed 16-bit PCM streams through the existing mixer + +#### Scenario: macOS capture uses run audio format +- **GIVEN** an active macOS recording configures a sample rate and channel count +- **WHEN** the native capture helpers start +- **THEN** both helpers emit signed 16-bit PCM with that sample rate and channel count + +#### Scenario: macOS recording stops native capture +- **GIVEN** macOS microphone and system capture are active +- **WHEN** Meeting Assistant stops capture +- **THEN** it terminates both native capture processes +- **AND** PCM already delivered to the managed pipeline before stop remains available to the existing recording pipeline + +#### Scenario: macOS privacy permission is unavailable +- **WHEN** macOS denies microphone or Screen Recording/System Audio capture permission +- **THEN** Meeting Assistant reports which macOS privacy permission is required + +#### Scenario: Windows audio capture remains isolated +- **GIVEN** Meeting Assistant is compiled for the Windows target +- **WHEN** the application registers and starts meeting audio capture +- **THEN** it uses the existing NAudio microphone and WASAPI loopback sources +- **AND** does not invoke or require the macOS native helper diff --git a/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/tasks.md b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/tasks.md new file mode 100644 index 0000000..1611c41 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-add-macos-meeting-audio-capture/tasks.md @@ -0,0 +1,13 @@ +## 1. macOS Capture Behavior + +- [x] 1.1 Specify macOS microphone and computer-output capture behavior and Windows isolation. +- [x] 1.2 Add failing behavior tests for macOS PCM streaming, platform registration, and the public recording endpoints. +- [x] 1.3 Implement the managed macOS audio-source and process-lifetime adapter. +- [x] 1.4 Implement and package the AVFoundation/ScreenCaptureKit Swift helper. +- [x] 1.5 Register the macOS composite source without changing the Windows registration path. + +## 2. Verification and Documentation + +- [x] 2.1 Document macOS permissions, build packaging, and recording behavior. +- [x] 2.2 Capture and inspect a short example mixed WAV through the new implementation. +- [x] 2.3 Run focused tests, portable tests/build, Windows-target regression compilation, strict OpenSpec validation, and refactoring/code review. diff --git a/openspec/changes/archive/2026-07-25-detect-host-operating-system/design.md b/openspec/changes/archive/2026-07-25-detect-host-operating-system/design.md new file mode 100644 index 0000000..4eacee6 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-detect-host-operating-system/design.md @@ -0,0 +1,55 @@ +## Context + +Meeting Assistant already separates many Windows-only implementations at compile time, while its portable service surface can run on macOS. The interactive settings/logs assistant is different: its model instructions are currently platform-neutral and do not tell the model which local operating system it is helping to operate. As a result, responses can mix PowerShell, Windows path and service concepts with macOS shells, POSIX paths, and launchd concepts. + +The host platform is stable for the lifetime of one application process, so it should be detected once during startup and passed to consumers as runtime context. + +## Goals / Non-Goals + +**Goals:** + +- Detect Windows and macOS explicitly when the process starts. +- Give the interactive settings/logs assistant unambiguous host-platform context. +- Require Windows-native operational guidance on Windows and macOS-native guidance on macOS. +- Prevent the assistant from presenting unavailable platform features as if they existed. +- Retain safe, neutral behavior on other hosts used for build or test execution. + +**Non-Goals:** + +- Implement macOS audio capture, global hotkeys, tray UI, Outlook integration, active-window screenshots, or notifications. +- Change automatic meeting-summary instructions, which do not operate the local application environment. +- Add a user-configurable OS override. +- Translate every internal process invocation into a shell script; existing argument-based process execution remains portable where it already is. + +## Decisions + +### Detect once and inject immutable runtime context + +Startup will create one immutable host-operating-system value using .NET runtime checks and register it as a singleton. The interactive instruction builder will consume that value instead of invoking static OS checks itself. + +This makes startup responsible for environment detection, keeps model instruction generation deterministic, and lets behavior tests supply explicit Windows and macOS values without depending on the test runner's host. + +Alternative considered: call `OperatingSystem.IsWindows()` or `OperatingSystem.IsMacOS()` directly inside the instruction builder. That would couple instruction tests to their host and repeat environment detection at the usage site. + +### Append platform guidance even when a custom prompt is configured + +The detected platform and platform constraints will be part of the always-appended runtime context, alongside configured paths and tool descriptions. A custom initial prompt may change the assistant persona, but it must not erase the factual host environment. + +Alternative considered: include platform text only in the built-in prompt. That would recreate the current ambiguity whenever `WorkflowRulesEditor:InitialPrompt` is configured. + +### Use explicit platform concepts and exclusions + +Windows guidance will name PowerShell, Windows paths, environment variables, processes, services, and Task Manager concepts while excluding macOS-only instructions. macOS guidance will name zsh/POSIX paths, launchd/LaunchAgents, Activity Monitor, and macOS environment conventions while excluding Windows-only instructions. + +Both branches will remind the assistant that it must respect the application's actual feature availability. Platform detection informs operational language; it does not create platform integrations. + +### Preserve a neutral unsupported-host fallback + +Linux and other hosts may be used for CI or the portable service surface. Those hosts will be identified as unsupported for platform-specific guidance and will receive portable .NET/tool guidance without being mislabeled as Windows or macOS. + +## Risks / Trade-offs + +- [Prompt guidance cannot guarantee every model response uses the correct command syntax] → Make the platform explicit, name positive concepts, and explicitly forbid cross-platform substitutions. +- [Platform examples can become too prescriptive] → Keep the guidance at the command/concept family level and prefer the assistant's built-in tools when available. +- [Users may interpret macOS detection as full macOS feature support] → State in both the prompt and documentation that feature availability remains separate from OS detection. + diff --git a/openspec/changes/archive/2026-07-25-detect-host-operating-system/implementation-evidence.md b/openspec/changes/archive/2026-07-25-detect-host-operating-system/implementation-evidence.md new file mode 100644 index 0000000..9ebfe6e --- /dev/null +++ b/openspec/changes/archive/2026-07-25-detect-host-operating-system/implementation-evidence.md @@ -0,0 +1,42 @@ +## Implementation Evidence + +Date: 2026-07-21 +Host: macOS arm64 + +### Behavior verification + +- Focused interactive-instruction and startup tests: + - Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --filter 'FullyQualifiedName~InstructionBuilder|FullyQualifiedName~ApplicationStartupRegistersDetectedHostOperatingSystemOnce' --no-restore --nologo` + - Result: passed, 11/11. +- The focused tests cover Windows guidance, macOS guidance, unsupported-host fallback, custom-prompt retention, and singleton startup detection. + +### Build and spec verification + +- Portable application build: + - Command: `dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 --no-restore --nologo` + - Result: succeeded with 0 warnings and 0 errors. +- OpenSpec validation: + - Command: `openspec validate detect-host-operating-system --strict` + - Result: valid. +- Diff check: + - Command: `git diff --check` + - Result: passed. + +### Broader test result and known limitations + +- Full portable test project: + - Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --no-restore --nologo` + - Result: 422 passed, 9 failed. + - The 9 failures are the pre-existing macOS failures identified before this change: Windows/GDI+ image rendering, Windows-path expectations, and Windows user-environment behavior. None exercise the new host detection or platform instruction output. +- Windows target: + - Restore with `EnableWindowsTargeting=true` succeeded. + - Native Windows build cannot complete on macOS because the Windows SDK attempts to execute `MakePri.exe` and returns `Exec format error`. Windows CI or a Windows host remains the appropriate verification environment for that target. + +### Refactoring review + +- DRY review consolidated shared platform-guidance policy text and reused the production detector in the startup test. +- SOLID review found no actionable issue. +- KISS review found no actionable simplification. + +The live Meeting Assistant process was not restarted or modified during verification. + diff --git a/openspec/changes/archive/2026-07-25-detect-host-operating-system/proposal.md b/openspec/changes/archive/2026-07-25-detect-host-operating-system/proposal.md new file mode 100644 index 0000000..ff928e6 --- /dev/null +++ b/openspec/changes/archive/2026-07-25-detect-host-operating-system/proposal.md @@ -0,0 +1,28 @@ +## Why + +The interactive Meeting Assistant settings/logs assistant can run against either a Windows or macOS installation, but its instructions do not currently identify the host operating system. Without that context, the agent can suggest Windows commands, paths, services, or desktop concepts on macOS, or suggest macOS concepts on Windows. + +## What Changes + +- Detect the host operating system when Meeting Assistant starts. +- Make the detected platform available to the interactive settings/logs assistant instruction builder. +- Tell the assistant to use Windows commands and concepts on Windows and macOS commands and concepts on macOS. +- Keep a platform-neutral fallback for unsupported hosts rather than incorrectly claiming Windows or macOS. +- Document that platform-aware agent guidance does not add missing platform integrations such as macOS audio capture or Windows-only desktop features. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `meeting-session`: Define how the interactive settings/logs assistant receives and applies detected host operating-system context. + +## Impact + +- Application startup dependency registration. +- Interactive settings/logs assistant instructions and behavior tests. +- Runtime documentation describing platform-aware operational guidance. + diff --git a/openspec/changes/archive/2026-07-25-detect-host-operating-system/specs/meeting-session/spec.md b/openspec/changes/archive/2026-07-25-detect-host-operating-system/specs/meeting-session/spec.md new file mode 100644 index 0000000..278084c --- /dev/null +++ b/openspec/changes/archive/2026-07-25-detect-host-operating-system/specs/meeting-session/spec.md @@ -0,0 +1,40 @@ +## ADDED Requirements + +### Requirement: Interactive agent uses detected host operating-system context +Meeting Assistant SHALL detect the host operating system when the application starts and SHALL make that immutable runtime context available to the interactive settings/logs assistant. + +When the detected host is Windows, the interactive assistant instructions SHALL identify Windows as the runtime platform and SHALL direct the assistant to use Windows-specific commands, paths, services, environment conventions, and desktop concepts instead of macOS concepts. + +When the detected host is macOS, the interactive assistant instructions SHALL identify macOS as the runtime platform and SHALL direct the assistant to use macOS-specific commands, POSIX paths, launchd services, environment conventions, and desktop concepts instead of Windows concepts. + +Platform-aware instructions SHALL remain present when the interactive assistant uses a configured custom initial prompt. + +The instructions SHALL distinguish platform-aware operational guidance from feature availability and SHALL NOT imply that detecting macOS provides Windows-only or otherwise unavailable integrations. + +For other detected hosts, Meeting Assistant SHALL identify the platform as unsupported for platform-specific guidance and SHALL direct the assistant to prefer portable tools and concepts rather than assuming Windows or macOS. + +#### Scenario: Windows host receives Windows operational guidance +- **GIVEN** Meeting Assistant detected Windows during startup +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions identify Windows as the runtime platform +- **AND** direct the assistant to use Windows commands and concepts +- **AND** direct the assistant not to substitute macOS commands or concepts + +#### Scenario: macOS host receives macOS operational guidance +- **GIVEN** Meeting Assistant detected macOS during startup +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions identify macOS as the runtime platform +- **AND** direct the assistant to use macOS commands and concepts +- **AND** direct the assistant not to substitute Windows commands or concepts + +#### Scenario: Custom prompt retains detected platform context +- **GIVEN** Meeting Assistant detected macOS during startup +- **AND** a custom interactive-assistant initial prompt is configured +- **WHEN** it builds the interactive assistant instructions +- **THEN** the instructions contain both the custom prompt and macOS runtime guidance + +#### Scenario: Unsupported host uses neutral guidance +- **GIVEN** Meeting Assistant runs on a host other than Windows or macOS +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions do not label the host as Windows or macOS +- **AND** direct the assistant to prefer portable tools and concepts diff --git a/openspec/changes/archive/2026-07-25-detect-host-operating-system/tasks.md b/openspec/changes/archive/2026-07-25-detect-host-operating-system/tasks.md new file mode 100644 index 0000000..a06456c --- /dev/null +++ b/openspec/changes/archive/2026-07-25-detect-host-operating-system/tasks.md @@ -0,0 +1,11 @@ +## 1. Runtime Platform Context + +- [x] 1.1 Add behavior tests for Windows and macOS interactive-assistant instructions. +- [x] 1.2 Detect the host operating system once at startup and register immutable runtime context. +- [x] 1.3 Append platform-specific command, path, service, and desktop concepts to interactive-assistant instructions. +- [x] 1.4 Preserve a platform-neutral fallback for unsupported hosts. + +## 2. Documentation and Validation + +- [x] 2.1 Document platform-aware interactive-assistant guidance and its feature-support boundary. +- [x] 2.2 Run focused behavior tests, the full test suite, strict OpenSpec validation, and the required refactoring review; record the known unrelated macOS failures and Windows-host build limitation in implementation evidence. diff --git a/openspec/specs/meeting-recording/spec.md b/openspec/specs/meeting-recording/spec.md index e1267fa..8492aa4 100644 --- a/openspec/specs/meeting-recording/spec.md +++ b/openspec/specs/meeting-recording/spec.md @@ -33,12 +33,26 @@ When no recording is active, aborting SHALL leave the current recording status u - **THEN** Meeting Assistant remains idle ### Requirement: Recording mode captures microphone and computer output -Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription. +Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription on supported Windows and macOS hosts. Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline. Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks. +On Windows, Meeting Assistant SHALL retain the existing NAudio microphone and WASAPI loopback implementation. + +On macOS, Meeting Assistant SHALL capture the default microphone through AVFoundation and computer output through ScreenCaptureKit without requiring a virtual audio device. + +The macOS native audio helper SHALL be packaged and launched from a signed application bundle with the stable bundle identifier `cloud.schweigert.meeting-assistant.audio-capture` and a microphone usage description so macOS privacy grants apply to background LaunchAgent capture and persist across deployments. + +The macOS background service SHALL be launched through a signed application bundle with the stable bundle identifier `cloud.schweigert.meeting-assistant` plus microphone and calendar usage descriptions, with the managed runtime nested under that bundle so macOS attributes privacy-sensitive child processes to the Meeting Assistant application identity. The bundle SHALL use a stable designated code requirement for that identifier so routine deployments remain compatible with its macOS privacy grant. + +The macOS capture adapter SHALL convert both native sources to signed 16-bit PCM using the active run's configured sample rate and channel count before passing chunks to the existing managed mixing pipeline. + +The macOS capture adapter SHALL stop its native capture process when the recording capture token is cancelled. + +When macOS privacy permission is missing or denied, Meeting Assistant SHALL fail capture with an actionable message identifying the required Microphone or Screen Recording/System Audio permission. + Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference. Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples. @@ -136,6 +150,41 @@ When no runtime microphone override is selected, the checked microphone SHALL be - **WHEN** the user selects `other microphone` from the taskbar microphone submenu - **THEN** later recording starts capture from `other microphone` +#### Scenario: macOS records both meeting-audio sources +- **GIVEN** the portable build is running on macOS +- **AND** Microphone and Screen Recording/System Audio permissions are granted +- **WHEN** a meeting recording starts +- **THEN** Meeting Assistant captures microphone audio through AVFoundation +- **AND** captures computer output through ScreenCaptureKit +- **AND** passes both signed 16-bit PCM streams through the existing mixer + +#### Scenario: macOS background capture uses a stable privacy identity +- **GIVEN** Meeting Assistant runs as a macOS LaunchAgent +- **WHEN** it starts the native audio helper +- **THEN** it launches the executable from the signed Meeting Assistant audio-capture application bundle +- **AND** macOS evaluates Microphone and Screen Recording/System Audio access against the bundle identifier `cloud.schweigert.meeting-assistant.audio-capture` + +#### Scenario: macOS capture uses run audio format +- **GIVEN** an active macOS recording configures a sample rate and channel count +- **WHEN** the native capture helpers start +- **THEN** both helpers emit signed 16-bit PCM with that sample rate and channel count + +#### Scenario: macOS recording stops native capture +- **GIVEN** macOS microphone and system capture are active +- **WHEN** Meeting Assistant stops capture +- **THEN** it terminates both native capture processes +- **AND** PCM already delivered to the managed pipeline before stop remains available to the existing recording pipeline + +#### Scenario: macOS privacy permission is unavailable +- **WHEN** macOS denies microphone or Screen Recording/System Audio capture permission +- **THEN** Meeting Assistant reports which macOS privacy permission is required + +#### Scenario: Windows audio capture remains isolated +- **GIVEN** Meeting Assistant is compiled for the Windows target +- **WHEN** the application registers and starts meeting audio capture +- **THEN** it uses the existing NAudio microphone and WASAPI loopback sources +- **AND** does not invoke or require the macOS native helper + ### Requirement: Stopping recording drains captured audio through transcription Meeting Assistant SHALL stop capturing new audio when recording mode is stopped, but it SHALL allow already captured audio to finish running through the configured speech recognition pipeline before the recording session completes. @@ -362,4 +411,3 @@ When a recording stops normally and its meeting note, transcript, and assistant - **WHEN** the recording stops normally - **THEN** Meeting Assistant deletes the run artifacts - **AND** does not run summary generation - diff --git a/openspec/specs/meeting-session/spec.md b/openspec/specs/meeting-session/spec.md index af180c1..4b49c49 100644 --- a/openspec/specs/meeting-session/spec.md +++ b/openspec/specs/meeting-session/spec.md @@ -383,3 +383,43 @@ After repairing a meeting or summary, the instructions SHALL direct the agent to - **WHEN** the user asks the interactive agent to fix that meeting or its summary - **THEN** the agent instructions direct it to inspect the matching assistant context for relevant meeting-specific memory - **AND** direct it to append its fixes and conclusions to that assistant context + +### Requirement: Interactive agent uses detected host operating-system context +Meeting Assistant SHALL detect the host operating system when the application starts and SHALL make that immutable runtime context available to the interactive settings/logs assistant. + +When the detected host is Windows, the interactive assistant instructions SHALL identify Windows as the runtime platform and SHALL direct the assistant to use Windows-specific commands, paths, services, environment conventions, and desktop concepts instead of macOS concepts. + +When the detected host is macOS, the interactive assistant instructions SHALL identify macOS as the runtime platform and SHALL direct the assistant to use macOS-specific commands, POSIX paths, launchd services, environment conventions, and desktop concepts instead of Windows concepts. + +Platform-aware instructions SHALL remain present when the interactive assistant uses a configured custom initial prompt. + +The instructions SHALL distinguish platform-aware operational guidance from feature availability and SHALL NOT imply that detecting macOS provides Windows-only or otherwise unavailable integrations. + +For other detected hosts, Meeting Assistant SHALL identify the platform as unsupported for platform-specific guidance and SHALL direct the assistant to prefer portable tools and concepts rather than assuming Windows or macOS. + +#### Scenario: Windows host receives Windows operational guidance +- **GIVEN** Meeting Assistant detected Windows during startup +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions identify Windows as the runtime platform +- **AND** direct the assistant to use Windows commands and concepts +- **AND** direct the assistant not to substitute macOS commands or concepts + +#### Scenario: macOS host receives macOS operational guidance +- **GIVEN** Meeting Assistant detected macOS during startup +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions identify macOS as the runtime platform +- **AND** direct the assistant to use macOS commands and concepts +- **AND** direct the assistant not to substitute Windows commands or concepts + +#### Scenario: Custom prompt retains detected platform context +- **GIVEN** Meeting Assistant detected macOS during startup +- **AND** a custom interactive-assistant initial prompt is configured +- **WHEN** it builds the interactive assistant instructions +- **THEN** the instructions contain both the custom prompt and macOS runtime guidance + +#### Scenario: Unsupported host uses neutral guidance +- **GIVEN** Meeting Assistant runs on a host other than Windows or macOS +- **WHEN** it builds the interactive settings/logs assistant instructions +- **THEN** the instructions do not label the host as Windows or macOS +- **AND** direct the assistant to prefer portable tools and concepts + diff --git a/openspec/specs/meeting-transcription/spec.md b/openspec/specs/meeting-transcription/spec.md index de2211b..cdc4788 100644 --- a/openspec/specs/meeting-transcription/spec.md +++ b/openspec/specs/meeting-transcription/spec.md @@ -137,6 +137,8 @@ After Azure Speech reconnects through a new SDK session, Meeting Assistant SHALL When Azure Speech is still unavailable after recording stops and transcription cannot drain before the configured stop-processing timeout, Meeting Assistant SHALL persist the stopped meeting as a durable transcription backlog item that references the completed mixed WAV and meeting artifacts. +When Azure Speech transcription fails after meeting audio has been captured, Meeting Assistant SHALL retain the completed mixed WAV and persist a durable transcription backlog item instead of deleting the only recoverable recording. + When a stopped meeting is persisted to the durable transcription backlog, Meeting Assistant SHALL release the active recording slot so another meeting can be recorded while the stopped meeting waits for Azure Speech to become available. When Azure Speech becomes available again, Meeting Assistant SHALL retry durable backlog items, rewrite the transcript from the recorded WAV, run the normal post-transcription meeting completion and summary flow, and remove the backlog item after successful completion. @@ -182,6 +184,12 @@ When Meeting Assistant starts, it SHALL preserve WAV files that are referenced b - **AND** keeps the completed mixed WAV referenced by that backlog item - **AND** returns to an idle recording state so another meeting can start +#### Scenario: Azure transcription failure retains captured meeting audio +- **GIVEN** Meeting Assistant captured meeting audio with the `azure-speech` provider +- **WHEN** Azure transcription fails before the transcript is completed +- **THEN** Meeting Assistant persists a durable backlog item for the meeting +- **AND** keeps the completed mixed WAV referenced by that backlog item for retry or manual recovery + #### Scenario: Durable Azure backlog resumes after connectivity returns - **GIVEN** a stopped Azure meeting exists in the durable transcription backlog - **WHEN** Azure Speech can transcribe the recorded WAV @@ -565,4 +573,3 @@ When pyannote secondary validation is enabled, Meeting Assistant SHALL start a n - **WHEN** Meeting Assistant starts - **THEN** it begins preparing the configured pyannote runtime image and model cache without waiting for the first validation request - **AND** application startup is not blocked by the warm-up task -