From aecef306272b415bb7c941f7ae880a293d2edf75 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Fri, 26 Jun 2026 13:15:32 +0200 Subject: [PATCH] Archive completed meeting assistant changes --- .../CalendarRecordingPromptSchedulerTests.cs | 170 +++++- .../LiteLlmScreenshotOcrClientTests.cs | 81 ++- .../MeetingScreenshotServiceTests.cs | 285 +++++++++- .../MicrophoneSelectionTests.cs | 54 ++ ...lookClassicMeetingMetadataProviderTests.cs | 13 +- .../RecordingCoordinatorTests.cs | 186 ++++++- .../ScreenshotOcrOptionsTests.cs | 1 + MeetingAssistant.Tests/TaskbarIconTests.cs | 30 +- .../WorkflowRulesEditorTests.cs | 13 +- .../CalendarRecordingPromptScheduler.cs | 25 +- ...kClassicCalendarMeetingProvider.Windows.cs | 19 +- MeetingAssistant/MeetingAssistantOptions.cs | 7 +- .../MeetingNotes/MeetingAttendeeNames.cs | 9 + .../MeetingNotes/MeetingNoteActionLinks.cs | 20 + ...tlookClassicAppointmentMetadata.Windows.cs | 176 ++++++ .../MeetingNotes/OutlookClassicCom.Windows.cs | 31 ++ ...kClassicMeetingMetadataProvider.Windows.cs | 166 +----- MeetingAssistant/Program.cs | 134 ++++- .../FileOfflineTranscriptionBacklog.cs | 22 +- .../Recording/IMicrophoneDeviceProvider.cs | 12 + .../Recording/MeetingRecordingCoordinator.cs | 107 +++- .../Recording/MicrophoneDevice.cs | 9 + .../Recording/MicrophoneDeviceSelection.cs | 54 ++ .../Recording/NaudioCaptureSource.cs | 8 +- .../Recording/OfflineTranscriptionBacklog.cs | 4 +- .../OfflineTranscriptionBacklogProcessor.cs | 29 +- .../Recording/PcmWavAudioChunkReader.cs | 48 ++ .../WindowsMicrophoneDeviceProvider.cs | 79 +++ .../Screenshots/LiteLlmScreenshotOcrClient.cs | 59 +- .../Screenshots/MeetingScreenshotService.cs | 518 +++++++++++++++++- .../SpeakerIdentityAttendeeCanonicalizer.cs | 6 +- .../Taskbar/MeetingTaskbarMenu.cs | 38 +- .../Taskbar/UnoTaskbarIconService.Windows.cs | 82 ++- .../Transcription/AsrDiagnosticService.cs | 37 +- MeetingAssistant/appsettings.json | 1 + README.md | 6 +- docs/meeting-assistant-configuration.md | 10 +- docs/meeting-workflow-engine.md | 2 + .../proposal.md | 16 + .../specs/meeting-recording/spec.md | 105 ++++ .../tasks.md | 9 + .../proposal.md | 12 + .../specs/meeting-summary/spec.md | 71 +++ .../tasks.md | 5 + .../proposal.md | 12 + .../specs/meeting-summary/spec.md | 65 +++ .../tasks.md | 5 + .../2026-06-26-add-tray-exit}/.openspec.yaml | 0 .../2026-06-26-add-tray-exit}/design.md | 0 .../2026-06-26-add-tray-exit}/proposal.md | 0 .../specs/meeting-recording/spec.md | 0 .../2026-06-26-add-tray-exit}/tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/meeting-transcription/spec.md | 0 .../tasks.md | 0 .../proposal.md | 8 + .../specs/meeting-recording/spec.md | 36 ++ .../tasks.md | 4 + .../proposal.md | 12 + .../specs/meeting-summary/spec.md | 70 +++ .../tasks.md | 5 + .../proposal.md | 0 .../specs/meeting-summary/spec.md | 0 .../tasks.md | 0 .../proposal.md | 14 + .../specs/meeting-session/spec.md | 120 ++++ .../tasks.md | 6 + openspec/specs/meeting-recording/spec.md | 62 ++- openspec/specs/meeting-session/spec.md | 34 ++ openspec/specs/meeting-summary/spec.md | 53 +- openspec/specs/meeting-transcription/spec.md | 45 ++ 72 files changed, 2914 insertions(+), 406 deletions(-) create mode 100644 MeetingAssistant.Tests/MicrophoneSelectionTests.cs create mode 100644 MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs create mode 100644 MeetingAssistant/Recording/IMicrophoneDeviceProvider.cs create mode 100644 MeetingAssistant/Recording/MicrophoneDevice.cs create mode 100644 MeetingAssistant/Recording/MicrophoneDeviceSelection.cs create mode 100644 MeetingAssistant/Recording/PcmWavAudioChunkReader.cs create mode 100644 MeetingAssistant/Recording/WindowsMicrophoneDeviceProvider.cs create mode 100644 openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/specs/meeting-recording/spec.md create mode 100644 openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/tasks.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/specs/meeting-summary/spec.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/tasks.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/specs/meeting-summary/spec.md create mode 100644 openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/tasks.md rename openspec/changes/{add-tray-exit => archive/2026-06-26-add-tray-exit}/.openspec.yaml (100%) rename openspec/changes/{add-tray-exit => archive/2026-06-26-add-tray-exit}/design.md (100%) rename openspec/changes/{add-tray-exit => archive/2026-06-26-add-tray-exit}/proposal.md (100%) rename openspec/changes/{add-tray-exit => archive/2026-06-26-add-tray-exit}/specs/meeting-recording/spec.md (100%) rename openspec/changes/{add-tray-exit => archive/2026-06-26-add-tray-exit}/tasks.md (100%) rename openspec/changes/{azure-speech-offline-resilience => archive/2026-06-26-azure-speech-offline-resilience}/design.md (100%) rename openspec/changes/{azure-speech-offline-resilience => archive/2026-06-26-azure-speech-offline-resilience}/proposal.md (100%) rename openspec/changes/{azure-speech-offline-resilience => archive/2026-06-26-azure-speech-offline-resilience}/specs/meeting-transcription/spec.md (100%) rename openspec/changes/{azure-speech-offline-resilience => archive/2026-06-26-azure-speech-offline-resilience}/tasks.md (100%) create mode 100644 openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/specs/meeting-recording/spec.md create mode 100644 openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/tasks.md create mode 100644 openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/specs/meeting-summary/spec.md create mode 100644 openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/tasks.md rename openspec/changes/{sync-projects-after-note-corrections => archive/2026-06-26-sync-projects-after-note-corrections}/proposal.md (100%) rename openspec/changes/{sync-projects-after-note-corrections => archive/2026-06-26-sync-projects-after-note-corrections}/specs/meeting-summary/spec.md (100%) rename openspec/changes/{sync-projects-after-note-corrections => archive/2026-06-26-sync-projects-after-note-corrections}/tasks.md (100%) create mode 100644 openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/proposal.md create mode 100644 openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/specs/meeting-session/spec.md create mode 100644 openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/tasks.md diff --git a/MeetingAssistant.Tests/CalendarRecordingPromptSchedulerTests.cs b/MeetingAssistant.Tests/CalendarRecordingPromptSchedulerTests.cs index 972f1b1..c3dc901 100644 --- a/MeetingAssistant.Tests/CalendarRecordingPromptSchedulerTests.cs +++ b/MeetingAssistant.Tests/CalendarRecordingPromptSchedulerTests.cs @@ -1,4 +1,5 @@ using MeetingAssistant.Calendar; +using MeetingAssistant.MeetingNotes; using MeetingAssistant.Recording; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -22,6 +23,94 @@ public sealed class CalendarRecordingPromptSchedulerTests Assert.Equal(0, harness.Recorder.StopCount); } + [Fact] + public async Task AcceptedPromptStartsRecordingWithPromptedMeetingMetadata() + { + var harness = CreateHarness(isRecording: false); + var selectedMetadata = new MeetingMetadata( + "Selected planning", + ["Ada"], + "Selected agenda", + DateTimeOffset.Parse("2026-06-03T10:30:00+00:00")); + var otherMetadata = new MeetingMetadata( + "Other planning", + ["Grace"], + "Other agenda", + DateTimeOffset.Parse("2026-06-03T10:40:00+00:00")); + var selectedMeeting = CreateMeeting( + harness.Clock, + id: "teams-selected", + subject: "Selected planning", + metadata: selectedMetadata); + var otherMeeting = CreateMeeting( + harness.Clock, + id: "teams-other", + subject: "Other planning", + startOffset: TimeSpan.FromMinutes(40), + metadata: otherMetadata); + harness.Provider.Meetings = [selectedMeeting, otherMeeting]; + + await SyncThenPromptAsync(harness, selectedMeeting); + + Assert.Equal(1, harness.Recorder.StartCount); + Assert.Same(selectedMetadata, harness.Recorder.StartMetadata.Single()); + Assert.Equal(1, harness.Recorder.PromptStartCount); + } + + [Fact] + public async Task AcceptedPromptWithoutMetadataStillUsesPromptStartPath() + { + var harness = CreateHarness(isRecording: false); + var meeting = CreateMeeting( + harness.Clock, + id: "teams-without-metadata", + subject: "No metadata", + metadata: null); + harness.Provider.Meetings = [meeting]; + + await SyncThenPromptAsync(harness, meeting); + + Assert.Equal(1, harness.Recorder.StartCount); + Assert.Null(harness.Recorder.StartMetadata.Single()); + Assert.Equal(1, harness.Recorder.PromptStartCount); + } + + [Fact] + public async Task AcceptingDisplayedPromptsOutOfOrderUsesAcceptedPromptMetadata() + { + var harness = CreateHarness(isRecording: false, autoAcceptPrompts: false); + var firstMetadata = new MeetingMetadata( + "First planning", + ["Ada"], + "First agenda", + DateTimeOffset.Parse("2026-06-03T10:30:00+00:00")); + var secondMetadata = new MeetingMetadata( + "Second planning", + ["Grace"], + "Second agenda", + DateTimeOffset.Parse("2026-06-03T10:30:00+00:00")); + var firstMeeting = CreateMeeting( + harness.Clock, + id: "teams-first", + subject: "First planning", + metadata: firstMetadata); + var secondMeeting = CreateMeeting( + harness.Clock, + id: "teams-second", + subject: "Second planning", + metadata: secondMetadata); + harness.Provider.Meetings = [firstMeeting, secondMeeting]; + + await harness.Scheduler.SyncOnceAsync(CancellationToken.None); + harness.Clock.Now = firstMeeting.Start; + await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None); + await harness.PromptService.RespondAsync(secondMeeting, MeetingStartPromptResponse.Record); + + Assert.Equal([firstMeeting, secondMeeting], harness.PromptService.PromptedMeetings); + Assert.Equal(1, harness.Recorder.StartCount); + Assert.Same(secondMetadata, harness.Recorder.StartMetadata.Single()); + } + [Fact] public async Task AcceptedPromptStopsActiveRecordingBeforeStartingNewRecording() { @@ -34,6 +123,25 @@ public sealed class CalendarRecordingPromptSchedulerTests Assert.Equal(["stop", "start"], harness.Recorder.Commands); } + [Fact] + public async Task CanceledCachedMeetingDoesNotPromptRecording() + { + var harness = CreateHarness(isRecording: false); + var canceledMeeting = CreateMeeting( + harness.Clock, + id: "teams-canceled", + subject: "Canceled project sync", + isCanceled: true); + harness.Provider.Meetings = [canceledMeeting]; + + await harness.Scheduler.SyncOnceAsync(CancellationToken.None); + harness.Clock.Now = canceledMeeting.Start; + await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None); + + Assert.Empty(harness.PromptService.PromptedMeetings); + Assert.Equal(0, harness.Recorder.StartCount); + } + [Fact] public async Task DisabledCalendarPromptsDoNotQueryCalendar() { @@ -115,23 +223,28 @@ public sealed class CalendarRecordingPromptSchedulerTests string id = "teams-1", string subject = "Project sync", TimeSpan? startOffset = null, - TimeSpan? duration = null) + TimeSpan? duration = null, + MeetingMetadata? metadata = null, + bool isCanceled = false) { var start = clock.Now.Add(startOffset ?? TimeSpan.FromMinutes(30)); return new CalendarMeeting( id, subject, start, - start.Add(duration ?? TimeSpan.FromMinutes(30))); + start.Add(duration ?? TimeSpan.FromMinutes(30)), + metadata, + isCanceled); } private static SchedulerHarness CreateHarness( bool isRecording = false, - bool enabled = true) + bool enabled = true, + bool autoAcceptPrompts = true) { var clock = new ManualCalendarPromptClock(new DateTimeOffset(2026, 6, 3, 9, 30, 0, TimeSpan.Zero)); var provider = new RecordingCalendarMeetingProvider([]); - var promptService = new AcceptingMeetingStartPromptService(); + var promptService = new CapturingMeetingStartPromptService(autoAcceptPrompts); var recorder = new RecordingPromptRecorder(isRecording); var scheduler = new CalendarRecordingPromptScheduler( Options.Create(new MeetingAssistantOptions @@ -155,7 +268,7 @@ public sealed class CalendarRecordingPromptSchedulerTests private sealed record SchedulerHarness( CalendarRecordingPromptScheduler Scheduler, RecordingCalendarMeetingProvider Provider, - AcceptingMeetingStartPromptService PromptService, + CapturingMeetingStartPromptService PromptService, RecordingPromptRecorder Recorder, ManualCalendarPromptClock Clock); @@ -179,8 +292,16 @@ public sealed class CalendarRecordingPromptSchedulerTests } } - private sealed class AcceptingMeetingStartPromptService : IMeetingStartPromptService + private sealed class CapturingMeetingStartPromptService : IMeetingStartPromptService { + private readonly bool autoAccept; + private readonly List pendingPrompts = []; + + public CapturingMeetingStartPromptService(bool autoAccept) + { + this.autoAccept = autoAccept; + } + public List PromptedMeetings { get; } = []; public async Task ShowPromptAsync( @@ -189,8 +310,24 @@ public sealed class CalendarRecordingPromptSchedulerTests CancellationToken cancellationToken) { PromptedMeetings.Add(request.Meeting); - await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken); + pendingPrompts.Add(new PendingPrompt(request.Meeting, handleResponseAsync)); + if (autoAccept) + { + await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken); + } } + + public Task RespondAsync( + CalendarMeeting meeting, + MeetingStartPromptResponse response) + { + var prompt = pendingPrompts.Single(pending => ReferenceEquals(pending.Meeting, meeting)); + return prompt.HandleResponseAsync(response, CancellationToken.None); + } + + private sealed record PendingPrompt( + CalendarMeeting Meeting, + Func HandleResponseAsync); } private sealed class RecordingPromptRecorder : IMeetingPromptRecordingController @@ -206,16 +343,35 @@ public sealed class CalendarRecordingPromptSchedulerTests public int StopCount { get; private set; } + public int PromptStartCount { get; private set; } + public List Commands { get; } = []; + public List StartMetadata { get; } = []; + public Task StartAsync(CancellationToken cancellationToken) + { + return StartRecordingAsync(null); + } + + private Task StartRecordingAsync( + MeetingMetadata? metadata) { StartCount++; Commands.Add("start"); + StartMetadata.Add(metadata); CurrentStatus = Status(isRecording: true); return Task.FromResult(CurrentStatus); } + public Task StartFromPromptAsync( + MeetingMetadata? metadata, + CancellationToken cancellationToken) + { + PromptStartCount++; + return StartRecordingAsync(metadata); + } + public Task StopAsync(CancellationToken cancellationToken) { StopCount++; diff --git a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs index 6d0ee68..f67fe81 100644 --- a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs +++ b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs @@ -16,9 +16,7 @@ public sealed class LiteLlmScreenshotOcrClientTests [Fact] public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank() { - var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); - Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); - await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]); + var screenshotPath = await CreateScreenshotAsync([1, 2, 3]); var handler = new RecordingHandler(""" { "output": [ @@ -75,9 +73,7 @@ public sealed class LiteLlmScreenshotOcrClientTests [Fact] public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured() { - var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); - Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); - await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]); + var screenshotPath = await CreateScreenshotAsync([4, 5, 6]); var handler = new RecordingHandler("""{ "output_text": "OCR result" }"""); var client = new LiteLlmScreenshotOcrClient( () => handler, @@ -116,9 +112,7 @@ public sealed class LiteLlmScreenshotOcrClientTests [Fact] public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText() { - var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); - Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); - await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6)); + var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6)); var handler = new RecordingHandler(""" { "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```" @@ -150,6 +144,67 @@ public sealed class LiteLlmScreenshotOcrClientTests StringComparison.Ordinal); } + [Fact] + public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText() + { + var screenshotPath = await CreateScreenshotAsync([1, 2, 3]); + var handler = new RecordingHandler(""" + { + "output_text": "Visible participant tiles: Ada and Grace.\n\n```json\n{ \"crop\": null, \"attendees\": [\"Ada Lovelace\", \"Grace Hopper\"] }\n```" + } + """); + var client = new LiteLlmScreenshotOcrClient( + () => handler, + NullLogger.Instance); + var options = new MeetingAssistantOptions + { + Agent = + { + Key = "agent-key" + } + }; + + var result = await client.ExtractAsync( + screenshotPath, + "Extract screenshot.", + options, + CancellationToken.None); + + Assert.Equal("Visible participant tiles: Ada and Grace.", result.Text); + Assert.Equal(["Ada Lovelace", "Grace Hopper"], result.Attendees); + } + + [Fact] + public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop() + { + var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6)); + var handler = new RecordingHandler(""" + { + "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 }, \"attendees\": \"Ada\" }\n```" + } + """); + var client = new LiteLlmScreenshotOcrClient( + () => handler, + NullLogger.Instance); + var options = new MeetingAssistantOptions + { + Agent = + { + Key = "agent-key" + } + }; + + var result = await client.ExtractAsync( + screenshotPath, + "Extract screenshot.", + options, + CancellationToken.None); + + Assert.Equal("Slide text", result.Text); + Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop); + Assert.Empty(result.Attendees); + } + private sealed class RecordingHandler : HttpMessageHandler { private readonly string responseBody; @@ -193,6 +248,14 @@ public sealed class LiteLlmScreenshotOcrClientTests bitmap.Save(stream, ImageFormat.Png); return stream.ToArray(); } + + private static async Task CreateScreenshotAsync(byte[] bytes) + { + var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); + Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); + await File.WriteAllBytesAsync(screenshotPath, bytes); + return screenshotPath; + } } #pragma warning restore CA1416 diff --git a/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs b/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs index 7a099cf..4a2c070 100644 --- a/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs +++ b/MeetingAssistant.Tests/MeetingScreenshotServiceTests.cs @@ -1,6 +1,8 @@ using MeetingAssistant.MeetingNotes; using MeetingAssistant.Screenshots; +using MeetingAssistant.Speakers; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using System.Drawing; using System.Drawing.Imaging; @@ -130,6 +132,165 @@ public sealed class MeetingScreenshotServiceTests Assert.Contains("Shared screen text", context); } + [Fact] + public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer() + { + var fixture = await ScreenshotFixture.CreateAsync( + options => + { + options.Screenshots.Ocr.Enabled = true; + }, + attendees: ["Ada Lovelace"]); + var ocr = new CapturingScreenshotOcrClient( + "Visible participant tiles: Ada, Grace, and Ada again.", + attendees: ["Ada L.", "Grace Hopper", "Ada Lovelace"]); + var canonicalizer = new MappingAttendeeCanonicalizer(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Ada L."] = "Ada Lovelace" + }); + var service = fixture.CreateService( + new FixedScreenshotCapture([1, 2, 3]), + ocr, + canonicalizer); + + await service.CaptureAsync( + fixture.Artifacts, + DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"), + DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"), + fixture.Options, + CancellationToken.None); + await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None); + + var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None); + Assert.Equal(["Ada Lovelace", "Grace Hopper"], meeting.Frontmatter.Attendees); + Assert.Contains(canonicalizer.Requests, request => + request.SequenceEqual(["Ada Lovelace", "Ada L.", "Grace Hopper", "Ada Lovelace"])); + } + + [Fact] + public async Task CaptureWritesRetryLinkWhenOcrFails() + { + var fixture = await ScreenshotFixture.CreateAsync(options => + { + options.Api.PublicBaseUrl = "http://localhost:5090"; + options.Screenshots.Ocr.Enabled = true; + }); + var service = fixture.CreateService( + new FixedScreenshotCapture([1, 2, 3]), + new ThrowingScreenshotOcrClient("vision offline")); + + var result = await service.CaptureAsync( + fixture.Artifacts, + DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"), + DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"), + fixture.Options, + CancellationToken.None); + await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None); + + var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath); + var screenshotId = ExtractScreenshotOcrId(context); + Assert.Contains("_OCR failed: vision offline_", context); + Assert.Contains("", start, StringComparison.Ordinal); + Assert.True(end > start); + return context[start..end]; + } } #pragma warning restore CA1416 diff --git a/MeetingAssistant.Tests/MicrophoneSelectionTests.cs b/MeetingAssistant.Tests/MicrophoneSelectionTests.cs new file mode 100644 index 0000000..1e364fe --- /dev/null +++ b/MeetingAssistant.Tests/MicrophoneSelectionTests.cs @@ -0,0 +1,54 @@ +using MeetingAssistant.Recording; + +namespace MeetingAssistant.Tests; + +public sealed class MicrophoneSelectionTests +{ + [Fact] + public void BlankConfiguredMicrophoneUsesDefaultDevice() + { + var selection = new MicrophoneDeviceSelection(); + + var selected = selection.Resolve( + configuredDeviceId: null, + new MicrophoneDevice("default-id", "default microphone"), + [ + new MicrophoneDevice("default-id", "default microphone"), + new MicrophoneDevice("other-id", "other microphone") + ]); + + Assert.Equal("default-id", selected?.Id); + } + + [Fact] + public void ConfiguredMicrophoneUsesMatchingDevice() + { + var selection = new MicrophoneDeviceSelection(); + var selected = selection.Resolve( + "other-id", + new MicrophoneDevice("default-id", "default microphone"), + [ + new MicrophoneDevice("default-id", "default microphone"), + new MicrophoneDevice("other-id", "other microphone") + ]); + + Assert.Equal("other-id", selected?.Id); + } + + [Fact] + public void RuntimeMicrophoneSelectionOverridesConfiguredDevice() + { + var selection = new MicrophoneDeviceSelection(); + selection.Select("runtime-id"); + + var selected = selection.Resolve( + "configured-id", + new MicrophoneDevice("default-id", "default microphone"), + [ + new MicrophoneDevice("configured-id", "configured microphone"), + new MicrophoneDevice("runtime-id", "runtime microphone") + ]); + + Assert.Equal("runtime-id", selected?.Id); + } +} diff --git a/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs index 5c9454b..6c3b618 100644 --- a/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs +++ b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs @@ -8,7 +8,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests [Fact] public void ExtractAgendaStopsBeforeTeamsJoinInformation() { - var agenda = OutlookClassicMeetingMetadataProvider.ExtractAgenda( + var agenda = OutlookClassicAppointmentMetadata.ExtractAgenda( """ Review current prototype Decide next backend @@ -25,7 +25,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests [Fact] public void NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail() { - var attendees = OutlookClassicMeetingMetadataProvider.NormalizeAttendees([ + var attendees = OutlookClassicAppointmentMetadata.NormalizeAttendees([ "Marcus.Altmann@sew-eurodrive.de", "Marcus.Altmann@sew-eurodrive.de ", "Schweigert, Manuel", @@ -37,5 +37,14 @@ public sealed class OutlookClassicMeetingMetadataProviderTests "Schweigert, Manuel" ], attendees); } + + [Theory] + [InlineData("Canceled: Project sync")] + [InlineData("Cancelled: Project sync")] + [InlineData("Abgesagt: Project sync")] + public void SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(string subject) + { + Assert.True(OutlookClassicCom.SubjectIndicatesCancellation(subject)); + } } #endif diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs index d9718bf..6b07ccd 100644 --- a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -784,6 +784,88 @@ public sealed class RecordingCoordinatorTests await coordinator.StopAsync(CancellationToken.None); } + [Fact] + public async Task StartWithPromptedMetadataBypassesLookupAndRunsWorkflowWithMetadata() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\prompted-metadata-meeting.md"); + var artifactStore = new InMemoryMeetingArtifactStore(); + var metadataProvider = new CountingMeetingMetadataProvider(); + var workflowEngine = new MetadataObservingWorkflowEngine(() => noteStore.SavedNote); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + meetingMetadataProvider: metadataProvider, + meetingWorkflowEngine: workflowEngine); + var promptedMetadata = new MeetingMetadata( + "Prompted Architecture Sync", + ["Ada "], + "Prompted agenda", + DateTimeOffset.Parse("2026-05-19T11:00:00+02:00")); + + await coordinator.StartFromPromptAsync(promptedMetadata, CancellationToken.None); + + await WaitUntilAsync(() => workflowEngine.ObservedEvents.Any(entry => + entry.Type == MeetingWorkflowEventType.StateTransition && + entry.Title == "Prompted Architecture Sync")); + Assert.Equal(0, metadataProvider.CallCount); + Assert.Equal("Prompted Architecture Sync", noteStore.SavedNote?.Frontmatter.Title); + Assert.Equal(["Ada "], noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Equal("Prompted agenda", artifactStore.Agenda); + Assert.Equal(DateTimeOffset.Parse("2026-05-19T11:00:00+02:00"), artifactStore.ScheduledEnd); + Assert.Collection( + workflowEngine.ObservedEvents.Where(entry => + entry.Type is MeetingWorkflowEventType.Created or MeetingWorkflowEventType.StateTransition), + created => + { + Assert.Equal(MeetingWorkflowEventType.Created, created.Type); + Assert.StartsWith("Meeting ", created.Title, StringComparison.Ordinal); + }, + transition => + { + Assert.Equal(MeetingWorkflowEventType.StateTransition, transition.Type); + Assert.Equal("Prompted Architecture Sync", transition.Title); + Assert.Equal(AssistantContextState.CollectingMetadata, transition.FromState); + Assert.Equal(AssistantContextState.Transcribing, transition.ToState); + }); + + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartFromPromptWithoutMetadataBypassesLookup() + { + var audioSource = new ControlledAudioSource(); + var metadataProvider = new CountingMeetingMetadataProvider(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + meetingMetadataProvider: metadataProvider); + + await coordinator.StartFromPromptAsync(null, CancellationToken.None); + + await Task.Delay(100); + Assert.Equal(0, metadataProvider.CallCount); + + await coordinator.StopAsync(CancellationToken.None); + } + [Fact] public async Task StartCanonicalizesOutlookMeetingAttendeesBeforeWritingNote() { @@ -1353,7 +1435,9 @@ public sealed class RecordingCoordinatorTests Assert.Equal([null, "english"], pipelineFactory.ProfileNames); Assert.Contains(transcriptStore.Segments, segment => segment.Speaker == "System" && - segment.Text.Contains("Transcription profile changed to english", StringComparison.Ordinal)); + segment.Text.Contains( + "Transcription profile changed to english. Speaker recognition and identities reset.", + StringComparison.Ordinal)); Assert.Null(summaryPipeline.Artifacts); await WaitUntilAsync(() => metadataProvider.CallCount == 1); @@ -2121,6 +2205,44 @@ public sealed class RecordingCoordinatorTests Assert.NotNull(summaryPipeline.Artifacts); } + [Fact] + public async Task StopProcessesMeetingNoteImageEmbedsBeforeSummarizing() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var artifactStore = new InMemoryMeetingArtifactStore(); + var summaryPipeline = new CapturingMeetingSummaryPipeline(); + var screenshotService = new BlockingMeetingScreenshotService(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + summaryPipeline, + Options.Create(CreateOptionsWithoutFinalizer()), + NullLogger.Instance, + screenshotService: screenshotService, + meetingNoteImageOcrService: screenshotService); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + var stop = coordinator.StopAsync(CancellationToken.None); + + await screenshotService.WaitUntilMeetingNoteImageProcessingStartedAsync(); + await screenshotService.WaitUntilOcrWaitStartedAsync(); + + Assert.DoesNotContain(AssistantContextState.Summarizing, artifactStore.States); + Assert.Null(summaryPipeline.Artifacts); + + screenshotService.ReleaseOcrWait(); + await stop; + + Assert.Contains(AssistantContextState.Summarizing, artifactStore.States); + Assert.NotNull(summaryPipeline.Artifacts); + } + [Fact] public async Task StopMarksAssistantContextErrorWhenSummaryFails() { @@ -2866,6 +2988,46 @@ public sealed class RecordingCoordinatorTests } } + private sealed class MetadataObservingWorkflowEngine : IMeetingWorkflowEngine + { + private readonly Func getMeetingNote; + + public MetadataObservingWorkflowEngine(Func getMeetingNote) + { + this.getMeetingNote = getMeetingNote; + } + + public List ObservedEvents { get; } = []; + + public Task RunAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + var note = getMeetingNote(); + ObservedEvents.Add(new ObservedWorkflowEvent( + workflowEvent.Type, + note?.Frontmatter.Title, + workflowEvent.FromState, + workflowEvent.ToState)); + return Task.CompletedTask; + } + + public Task TransformTranscriptLineAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult(workflowEvent.TranscriptLineText ?? ""); + } + + public sealed record ObservedWorkflowEvent( + MeetingWorkflowEventType Type, + string? Title, + AssistantContextState? FromState, + AssistantContextState? ToState); + } + private sealed class FailingFirstTranscriptLineWorkflowEngine : IMeetingWorkflowEngine { public Task RunAsync( @@ -3003,12 +3165,14 @@ public sealed class RecordingCoordinatorTests } } - private sealed class BlockingMeetingScreenshotService : IMeetingScreenshotService + private sealed class BlockingMeetingScreenshotService : IMeetingScreenshotService, IMeetingNoteImageOcrService { private readonly TaskCompletionSource waitStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource releaseWait = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource meetingNoteImageProcessingStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); public Task CaptureAsync( MeetingSessionArtifacts artifacts, @@ -3041,6 +3205,20 @@ public sealed class RecordingCoordinatorTests return Task.CompletedTask; } + public Task ProcessMeetingNoteImageEmbedsAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + meetingNoteImageProcessingStarted.TrySetResult(); + return Task.FromResult(new MeetingNoteImageOcrQueueResult(0)); + } + + public Task WaitUntilMeetingNoteImageProcessingStartedAsync() + { + return meetingNoteImageProcessingStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + public Task WaitUntilOcrWaitStartedAsync() { return waitStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); @@ -3683,9 +3861,9 @@ public sealed class RecordingCoordinatorTests return Task.FromResult>(Items.ToList()); } - public Task CompleteAsync(string id, CancellationToken cancellationToken) + public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) { - Items.RemoveAll(item => string.Equals(item.Id, id, StringComparison.Ordinal)); + Items.RemoveAll(existing => string.Equals(existing.Id, item.Id, StringComparison.Ordinal)); return Task.CompletedTask; } } diff --git a/MeetingAssistant.Tests/ScreenshotOcrOptionsTests.cs b/MeetingAssistant.Tests/ScreenshotOcrOptionsTests.cs index f07ccf7..292303d 100644 --- a/MeetingAssistant.Tests/ScreenshotOcrOptionsTests.cs +++ b/MeetingAssistant.Tests/ScreenshotOcrOptionsTests.cs @@ -9,5 +9,6 @@ public sealed class ScreenshotOcrOptionsTests { Assert.Contains("exactly who is in the meeting", ScreenshotOcrOptions.DefaultPrompt); Assert.Contains("partial", ScreenshotOcrOptions.DefaultPrompt); + Assert.Contains("\"attendees\"", ScreenshotOcrOptions.DefaultPrompt); } } diff --git a/MeetingAssistant.Tests/TaskbarIconTests.cs b/MeetingAssistant.Tests/TaskbarIconTests.cs index 8b55c99..06df391 100644 --- a/MeetingAssistant.Tests/TaskbarIconTests.cs +++ b/MeetingAssistant.Tests/TaskbarIconTests.cs @@ -31,6 +31,34 @@ public sealed class TaskbarIconTests Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording); } + [Fact] + public void MenuOffersMicrophoneSubmenuWithEffectiveDeviceChecked() + { + var menu = MeetingTaskbarMenuBuilder.Build( + Status(), + [Profile("default")], + [ + new MicrophoneDevice("integrated", "integrated microphone"), + new MicrophoneDevice("other", "other microphone") + ], + "integrated"); + + var microphoneMenu = Assert.Single(menu.Items, item => item.Text == "Microphone"); + + Assert.Equal(MeetingTaskbarAction.OpenSubmenu, microphoneMenu.Action); + Assert.NotNull(microphoneMenu.Items); + Assert.Contains(microphoneMenu.Items, item => + item.Action == MeetingTaskbarAction.SelectMicrophone && + item.MicrophoneDeviceId == "integrated" && + item.Text == "integrated microphone" && + item.IsChecked); + Assert.Contains(microphoneMenu.Items, item => + item.Action == MeetingTaskbarAction.SelectMicrophone && + item.MicrophoneDeviceId == "other" && + item.Text == "other microphone" && + !item.IsChecked); + } + [Fact] public void RecordingMenuOffersStopAbortAndOtherProfileSwitches() { @@ -108,7 +136,7 @@ public sealed class TaskbarIconTests { Assert.Equal( requiresConfirmation, - MeetingTaskbarExitPolicy.RequiresConfirmation(Status(state: state))); + MeetingTaskbarExitPolicy.RequiresConfirmation(state)); } [Fact] diff --git a/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs b/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs index 8d7db8b..2ecc774 100644 --- a/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs +++ b/MeetingAssistant.Tests/WorkflowRulesEditorTests.cs @@ -442,7 +442,8 @@ public sealed class WorkflowRulesEditorTests Assert.Equal("summary body", await File.ReadAllTextAsync(Path.Combine(summariesRoot, "daily-summary.md"))); Assert.Equal("one\ntwo\nthree", await File.ReadAllTextAsync(Path.Combine(transcriptsRoot, "daily-transcript.md"))); - var note = await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md")); + var note = (await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md"))) + .Replace("\r\n", "\n", StringComparison.Ordinal); Assert.Contains("title: Fixed", note); Assert.Contains("projects:\n- Alpha", note); Assert.Contains("Existing body.", note); @@ -721,6 +722,16 @@ public sealed class WorkflowRulesEditorTests Assert.Contains("read_logs", instructions); Assert.Contains("read_spec_file", instructions); Assert.Contains("list_recent_summaries", instructions); + } + + [Fact] + public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes() + { + var builder = new WorkflowRulesEditorInstructionBuilder( + NullLogger.Instance); + + var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None); + Assert.Contains("When correcting a meeting note that has project references", instructions); Assert.Contains("read that project's AGENTS.md", instructions); Assert.Contains("new project reference", instructions); diff --git a/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs b/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs index 749eda0..dc17c96 100644 --- a/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs +++ b/MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs @@ -1,4 +1,5 @@ using MeetingAssistant.Recording; +using MeetingAssistant.MeetingNotes; using Microsoft.Extensions.Options; namespace MeetingAssistant.Calendar; @@ -84,7 +85,8 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService foreach (var meeting in cachedMeetings.OrderBy(meeting => meeting.Start)) { var promptKey = GetPromptKey(meeting); - if (promptedMeetingKeys.Contains(promptKey) || + if (meeting.IsCanceled || + promptedMeetingKeys.Contains(promptKey) || !IsInPromptWindow(meeting, now, promptOptions)) { continue; @@ -97,7 +99,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService meeting.Start); await promptService.ShowPromptAsync( new MeetingStartPromptRequest(meeting), - HandlePromptResponseAsync, + (response, token) => HandlePromptResponseAsync(meeting, response, token), cancellationToken); } } @@ -133,6 +135,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService } private async Task HandlePromptResponseAsync( + CalendarMeeting meeting, MeetingStartPromptResponse response, CancellationToken cancellationToken) { @@ -146,7 +149,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService await recordingController.StopAsync(cancellationToken); } - await recordingController.StartAsync(cancellationToken); + await recordingController.StartFromPromptAsync(meeting.Metadata, cancellationToken); } private void ResetPromptedMeetingsIfDayChanged(DateOnly today) @@ -183,6 +186,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService var syncDelay = nextSyncAt - now; var nextMeetingStart = cachedMeetings + .Where(meeting => !meeting.IsCanceled) .Where(meeting => !promptedMeetingKeys.Contains(GetPromptKey(meeting))) .Where(meeting => meeting.End > now) .Select(meeting => meeting.Start <= now ? now : meeting.Start) @@ -228,7 +232,9 @@ public sealed record CalendarMeeting( string Id, string Subject, DateTimeOffset Start, - DateTimeOffset End); + DateTimeOffset End, + MeetingMetadata? Metadata = null, + bool IsCanceled = false); public interface IMeetingStartPromptService { @@ -252,6 +258,10 @@ public interface IMeetingPromptRecordingController Task StartAsync(CancellationToken cancellationToken); + Task StartFromPromptAsync( + MeetingMetadata? metadata, + CancellationToken cancellationToken); + Task StopAsync(CancellationToken cancellationToken); } @@ -271,6 +281,13 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo return coordinator.StartAsync(cancellationToken); } + public Task StartFromPromptAsync( + MeetingMetadata? metadata, + CancellationToken cancellationToken) + { + return coordinator.StartFromPromptAsync(metadata, cancellationToken); + } + public Task StopAsync(CancellationToken cancellationToken) { return coordinator.StopAsync(cancellationToken); diff --git a/MeetingAssistant/Calendar/OutlookClassicCalendarMeetingProvider.Windows.cs b/MeetingAssistant/Calendar/OutlookClassicCalendarMeetingProvider.Windows.cs index 42aa8b5..1bd778c 100644 --- a/MeetingAssistant/Calendar/OutlookClassicCalendarMeetingProvider.Windows.cs +++ b/MeetingAssistant/Calendar/OutlookClassicCalendarMeetingProvider.Windows.cs @@ -60,18 +60,21 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End")); if (end <= dayStart || start < dayStart || - !IsTeamsAppointment(item)) + OutlookClassicCom.IsCanceledAppointment(item) || + !OutlookClassicAppointmentMetadata.IsTeamsAppointment(item)) { continue; } - var rawSubject = Convert.ToString(OutlookClassicCom.GetProperty(item, "Subject"))?.Trim(); - var subject = string.IsNullOrWhiteSpace(rawSubject) ? "Teams meeting" : rawSubject; + var subject = OutlookClassicAppointmentMetadata.ReadSubject(item); + var localEnd = OutlookClassicCom.ToLocalOffset(end); meetings.Add(new CalendarMeeting( GetAppointmentId(item, start, end, subject), subject, OutlookClassicCom.ToLocalOffset(start), - OutlookClassicCom.ToLocalOffset(end))); + localEnd, + OutlookClassicAppointmentMetadata.CreateMetadata(item, end), + IsCanceled: false)); } finally { @@ -98,13 +101,5 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv ? $"{start:O}|{end:O}|{subject}" : $"{id}|{start:O}"; } - - private static bool IsTeamsAppointment(object appointment) - { - var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? ""; - var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? ""; - var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""; - return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body); - } } #endif diff --git a/MeetingAssistant/MeetingAssistantOptions.cs b/MeetingAssistant/MeetingAssistantOptions.cs index 4e909be..f5fe6db 100644 --- a/MeetingAssistant/MeetingAssistantOptions.cs +++ b/MeetingAssistant/MeetingAssistantOptions.cs @@ -59,6 +59,7 @@ public sealed class ScreenshotOcrOptions Identify who appears to be talking, who appears to be presenting, and what is being presented when visible. If people or participant tiles are visible, state whether it is clear from the screenshot exactly who is in the meeting or whether the visible people are only a partial participant result. + Include any attendee names that are clearly visible or strongly deduced from the screenshot in the meeting-assistant metadata. The attendee list may be partial. If the screenshot contains slides, capture the visible text in clean markdown. If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately. If it contains another kind of shared screen or scene, describe the scene and the relevant visible information. @@ -67,9 +68,9 @@ public sealed class ScreenshotOcrOptions Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently. End your response with exactly one fenced json block containing meeting-assistant metadata, for example: ```json - { "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } } + { "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, "attendees": ["Ada Lovelace"] } ``` - Use `{ "crop": null }` when no confident presentation/shared-screen crop exists. + Use `{ "crop": null, "attendees": [] }` when no confident presentation/shared-screen crop exists and no attendees are visible or deducible. Do not invent information that is not visible in the screenshot. """; @@ -125,6 +126,8 @@ public sealed class RecordingOptions public int Channels { get; set; } = 1; + public string? MicrophoneDeviceId { get; set; } + public double MicrophoneMixGain { get; set; } = 1; public double SystemAudioMixGain { get; set; } = 1; diff --git a/MeetingAssistant/MeetingNotes/MeetingAttendeeNames.cs b/MeetingAssistant/MeetingNotes/MeetingAttendeeNames.cs index d493574..fda14c9 100644 --- a/MeetingAssistant/MeetingNotes/MeetingAttendeeNames.cs +++ b/MeetingAssistant/MeetingNotes/MeetingAttendeeNames.cs @@ -8,4 +8,13 @@ internal static class MeetingAttendeeNames var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal); return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed; } + + public static List NormalizeDistinct(IEnumerable attendees) + { + return attendees + .Select(attendee => attendee.Trim()) + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } } diff --git a/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs b/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs index 9a8379b..1ceb6c0 100644 --- a/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs +++ b/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs @@ -14,4 +14,24 @@ public static class MeetingNoteActionLinks var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}"; return $"[{label}]({url})"; } + + public static string CreateScreenshotOcrRetryLink( + string publicBaseUrl, + MeetingSessionArtifacts artifacts, + string screenshotPath, + string screenshotId, + string label = "Retry screenshot OCR") + { + var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl) + ? "http://localhost:5090" + : publicBaseUrl.TrimEnd('/'); + var url = $"{baseUrl}/meetings/screenshot-ocr/retry" + + $"?meetingNotePath={Uri.EscapeDataString(artifacts.MeetingNotePath)}" + + $"&transcriptPath={Uri.EscapeDataString(artifacts.TranscriptPath)}" + + $"&assistantContextPath={Uri.EscapeDataString(artifacts.AssistantContextPath)}" + + $"&summaryPath={Uri.EscapeDataString(artifacts.SummaryPath)}" + + $"&screenshotPath={Uri.EscapeDataString(screenshotPath)}" + + $"&screenshotId={Uri.EscapeDataString(screenshotId)}"; + return $"[{label}]({url})"; + } } diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs new file mode 100644 index 0000000..84f0b01 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/OutlookClassicAppointmentMetadata.Windows.cs @@ -0,0 +1,176 @@ +#if WINDOWS +using MeetingAssistant.Calendar; + +namespace MeetingAssistant.MeetingNotes; + +internal static class OutlookClassicAppointmentMetadata +{ + public static MeetingMetadata CreateMetadata(object appointment, DateTime end) + { + var title = ReadSubject(appointment); + return new MeetingMetadata( + title, + ReadAttendees(appointment), + ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""), + OutlookClassicCom.ToLocalOffset(end)); + } + + public static string ReadSubject(object appointment) + { + var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim(); + return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject; + } + + public static bool IsTeamsAppointment(object appointment) + { + var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? ""; + var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? ""; + var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""; + return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body); + } + + public static string ExtractAgenda(string body) + { + if (string.IsNullOrWhiteSpace(body)) + { + return ""; + } + + var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'); + var agendaLines = new List(); + foreach (var line in lines) + { + if (IsTeamsSeparator(line) || IsTeamsJoinLine(line)) + { + break; + } + + agendaLines.Add(line); + } + + return string.Join(Environment.NewLine, agendaLines).Trim(); + } + + public static IReadOnlyList ReadAttendees(object appointment) + { + var attendees = new List(); + var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer")); + if (!string.IsNullOrWhiteSpace(organizer)) + { + attendees.Add(organizer.Trim()); + } + + object? recipients = null; + try + { + recipients = OutlookClassicCom.GetProperty(appointment, "Recipients"); + var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count")); + for (var index = 1; index <= count; index++) + { + object? recipient = null; + try + { + recipient = OutlookClassicCom.Invoke(recipients!, "Item", index); + var formatted = FormatRecipient(recipient!); + if (!string.IsNullOrWhiteSpace(formatted)) + { + attendees.Add(formatted); + } + } + finally + { + OutlookClassicCom.ReleaseComObject(recipient); + } + } + } + finally + { + OutlookClassicCom.ReleaseComObject(recipients); + } + + return NormalizeAttendees(attendees); + } + + public static IReadOnlyList NormalizeAttendees(IEnumerable attendees) + { + return attendees + .Select(NormalizeAttendee) + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase) + .Select(group => group + .OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal)) + .ThenBy(attendee => attendee.Length) + .First()) + .ToList(); + } + + private static bool IsTeamsSeparator(string line) + { + var trimmed = line.Trim(); + return trimmed.Length >= 8 && + trimmed.All(character => character is '_' or '-' or '*' or ' '); + } + + private static bool IsTeamsJoinLine(string line) + { + return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line); + } + + private static string NormalizeAttendee(string attendee) + { + var normalized = attendee.Trim(); + var mailtoPrefix = "mailto:"; + normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase); + while (normalized.Contains(" ", StringComparison.Ordinal)) + { + normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); + } + + var emailStart = normalized.IndexOf('<', StringComparison.Ordinal); + var emailEnd = normalized.LastIndexOf('>'); + if (emailStart > 0 && emailEnd > emailStart) + { + var name = normalized[..emailStart].Trim(); + var email = normalized[(emailStart + 1)..emailEnd].Trim(); + if (name.Equals(email, StringComparison.OrdinalIgnoreCase)) + { + return email; + } + + return $"{name} <{email}>"; + } + + return normalized; + } + + private static string GetAttendeeDeduplicationKey(string attendee) + { + var emailStart = attendee.IndexOf('<', StringComparison.Ordinal); + var emailEnd = attendee.LastIndexOf('>'); + if (emailStart > 0 && emailEnd > emailStart) + { + return attendee[(emailStart + 1)..emailEnd].Trim(); + } + + return attendee.Trim(); + } + + private static string FormatRecipient(object recipient) + { + var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? ""; + var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? ""; + if (string.IsNullOrWhiteSpace(name)) + { + return email; + } + + if (string.IsNullOrWhiteSpace(email) || + email.Contains('/')) + { + return name; + } + + return $"{name} <{email}>"; + } +} +#endif diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicCom.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicCom.Windows.cs index f95166c..e0106e8 100644 --- a/MeetingAssistant/MeetingNotes/OutlookClassicCom.Windows.cs +++ b/MeetingAssistant/MeetingNotes/OutlookClassicCom.Windows.cs @@ -108,6 +108,37 @@ internal static class OutlookClassicCom } } + public static bool IsCanceledAppointment(object appointment) + { + var meetingStatus = TryGetProperty(appointment, "MeetingStatus"); + if (meetingStatus is not null) + { + try + { + var status = Convert.ToInt32(meetingStatus); + if (status is 5 or 7) + { + return true; + } + } + catch (Exception exception) when (exception is FormatException or InvalidCastException or OverflowException) + { + // Fall back to subject prefixes when Outlook exposes a non-numeric status value. + } + } + + var subject = Convert.ToString(TryGetProperty(appointment, "Subject")) ?? ""; + return SubjectIndicatesCancellation(subject); + } + + internal static bool SubjectIndicatesCancellation(string subject) + { + var normalized = subject.TrimStart(); + return normalized.StartsWith("Canceled:", StringComparison.OrdinalIgnoreCase) || + normalized.StartsWith("Cancelled:", StringComparison.OrdinalIgnoreCase) || + normalized.StartsWith("Abgesagt:", StringComparison.OrdinalIgnoreCase); + } + public static DateTimeOffset ToLocalOffset(DateTime value) { return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local)); diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs index 2e72974..b261632 100644 --- a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs +++ b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs @@ -121,7 +121,9 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv } var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End")); - if (end < windowStart || !IsTeamsAppointment(item)) + if (end < windowStart || + OutlookClassicCom.IsCanceledAppointment(item) || + !OutlookClassicAppointmentMetadata.IsTeamsAppointment(item)) { continue; } @@ -150,15 +152,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv return null; } - var appointment = selected.Appointment; - var title = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? ""; - var attendees = ReadAttendees(appointment); - var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""; - return new MeetingMetadata( - title.Trim(), - attendees, - ExtractAgenda(body), - OutlookClassicCom.ToLocalOffset(selected.End)); + return OutlookClassicAppointmentMetadata.CreateMetadata(selected.Appointment, selected.End); } finally { @@ -169,158 +163,6 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv } } - internal static string ExtractAgenda(string body) - { - if (string.IsNullOrWhiteSpace(body)) - { - return ""; - } - - var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'); - var agendaLines = new List(); - foreach (var line in lines) - { - if (IsTeamsSeparator(line) || IsTeamsJoinLine(line)) - { - break; - } - - agendaLines.Add(line); - } - - return string.Join(Environment.NewLine, agendaLines).Trim(); - } - - private static bool IsTeamsSeparator(string line) - { - var trimmed = line.Trim(); - return trimmed.Length >= 8 && - trimmed.All(character => character is '_' or '-' or '*' or ' '); - } - - private static bool IsTeamsJoinLine(string line) - { - return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line); - } - - private static bool IsTeamsAppointment(object appointment) - { - var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? ""; - var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? ""; - var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""; - return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body); - } - - private static IReadOnlyList ReadAttendees(object appointment) - { - var attendees = new List(); - var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer")); - if (!string.IsNullOrWhiteSpace(organizer)) - { - attendees.Add(organizer.Trim()); - } - - object? recipients = null; - try - { - recipients = OutlookClassicCom.GetProperty(appointment, "Recipients"); - var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count")); - for (var index = 1; index <= count; index++) - { - object? recipient = null; - try - { - recipient = OutlookClassicCom.Invoke(recipients!, "Item", index); - var formatted = FormatRecipient(recipient!); - if (!string.IsNullOrWhiteSpace(formatted)) - { - attendees.Add(formatted); - } - } - finally - { - OutlookClassicCom.ReleaseComObject(recipient); - } - } - } - finally - { - OutlookClassicCom.ReleaseComObject(recipients); - } - - return NormalizeAttendees(attendees); - } - - internal static IReadOnlyList NormalizeAttendees(IEnumerable attendees) - { - return attendees - .Select(NormalizeAttendee) - .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) - .GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase) - .Select(group => group - .OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal)) - .ThenBy(attendee => attendee.Length) - .First()) - .ToList(); - } - - private static string NormalizeAttendee(string attendee) - { - var normalized = attendee.Trim(); - var mailtoPrefix = "mailto:"; - normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase); - while (normalized.Contains(" ", StringComparison.Ordinal)) - { - normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); - } - - var emailStart = normalized.IndexOf('<', StringComparison.Ordinal); - var emailEnd = normalized.LastIndexOf('>'); - if (emailStart > 0 && emailEnd > emailStart) - { - var name = normalized[..emailStart].Trim(); - var email = normalized[(emailStart + 1)..emailEnd].Trim(); - if (name.Equals(email, StringComparison.OrdinalIgnoreCase)) - { - return email; - } - - return $"{name} <{email}>"; - } - - return normalized; - } - - private static string GetAttendeeDeduplicationKey(string attendee) - { - var emailStart = attendee.IndexOf('<', StringComparison.Ordinal); - var emailEnd = attendee.LastIndexOf('>'); - if (emailStart > 0 && emailEnd > emailStart) - { - return attendee[(emailStart + 1)..emailEnd].Trim(); - } - - return attendee.Trim(); - } - - private static string FormatRecipient(object recipient) - { - var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? ""; - var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? ""; - if (string.IsNullOrWhiteSpace(name)) - { - return email; - } - - if (string.IsNullOrWhiteSpace(email) || - email.Contains('/')) - { - return name; - } - - return $"{name} <{email}>"; - } - private sealed record OutlookAppointmentCandidate( object Appointment, DateTime Start, diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index c5ceec0..32dd717 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -19,6 +19,8 @@ builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider()); builder.Services.Configure(builder.Configuration.GetSection("MeetingAssistant")); builder.Services.AddSingleton(); #if WINDOWS +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -70,7 +72,10 @@ builder.Services.AddSingleton(); #endif builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddDbContextFactory((services, dbOptions) => { var appOptions = services.GetRequiredService>().Value; @@ -410,6 +415,81 @@ app.MapGet("/profiles/{launchProfile}/meetings/summary/retry", async ( launchProfiles, cancellationToken)); +app.MapPost("/meetings/screenshot-ocr/retry", async ( + ScreenshotOcrRetryRequest request, + IScreenshotOcrRetryRunner screenshotOcrRetryRunner, + IOptions options, + CancellationToken cancellationToken) => + await RetryScreenshotOcrAsync( + null, + request, + screenshotOcrRetryRunner, + options.Value, + null, + cancellationToken)); +app.MapPost("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async ( + string launchProfile, + ScreenshotOcrRetryRequest request, + IScreenshotOcrRetryRunner screenshotOcrRetryRunner, + IOptions options, + ILaunchProfileOptionsProvider launchProfiles, + CancellationToken cancellationToken) => + await RetryScreenshotOcrAsync( + launchProfile, + request, + screenshotOcrRetryRunner, + options.Value, + launchProfiles, + cancellationToken)); +app.MapGet("/meetings/screenshot-ocr/retry", async ( + string meetingNotePath, + string transcriptPath, + string assistantContextPath, + string summaryPath, + string screenshotPath, + string screenshotId, + IScreenshotOcrRetryRunner screenshotOcrRetryRunner, + IOptions options, + CancellationToken cancellationToken) => + await RetryScreenshotOcrAsync( + null, + new ScreenshotOcrRetryRequest( + meetingNotePath, + transcriptPath, + assistantContextPath, + summaryPath, + screenshotPath, + screenshotId), + screenshotOcrRetryRunner, + options.Value, + null, + cancellationToken)); +app.MapGet("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async ( + string launchProfile, + string meetingNotePath, + string transcriptPath, + string assistantContextPath, + string summaryPath, + string screenshotPath, + string screenshotId, + IScreenshotOcrRetryRunner screenshotOcrRetryRunner, + IOptions options, + ILaunchProfileOptionsProvider launchProfiles, + CancellationToken cancellationToken) => + await RetryScreenshotOcrAsync( + launchProfile, + new ScreenshotOcrRetryRequest( + meetingNotePath, + transcriptPath, + assistantContextPath, + summaryPath, + screenshotPath, + screenshotId), + screenshotOcrRetryRunner, + options.Value, + launchProfiles, + cancellationToken)); + static async Task RetrySummaryAsync( string? launchProfile, string? summaryPath, @@ -438,6 +518,50 @@ static async Task RetrySummaryAsync( }); } +static async Task RetryScreenshotOcrAsync( + string? launchProfile, + ScreenshotOcrRetryRequest request, + IScreenshotOcrRetryRunner screenshotOcrRetryRunner, + MeetingAssistantOptions defaultOptions, + ILaunchProfileOptionsProvider? launchProfiles, + CancellationToken cancellationToken) +{ + if (string.IsNullOrWhiteSpace(request.MeetingNotePath) || + string.IsNullOrWhiteSpace(request.TranscriptPath) || + string.IsNullOrWhiteSpace(request.AssistantContextPath) || + string.IsNullOrWhiteSpace(request.SummaryPath) || + string.IsNullOrWhiteSpace(request.ScreenshotPath) || + string.IsNullOrWhiteSpace(request.ScreenshotId)) + { + return Results.BadRequest(new { error = "Meeting artifact paths, screenshot path, and screenshot id are required." }); + } + + var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles); + var artifacts = new MeetingSessionArtifacts( + request.MeetingNotePath, + request.TranscriptPath, + request.AssistantContextPath, + request.SummaryPath); + var trigger = await screenshotOcrRetryRunner.TriggerOcrRetryAsync( + artifacts, + request.ScreenshotPath, + request.ScreenshotId, + profileOptions, + cancellationToken); + if (trigger is null) + { + return Results.NotFound(new { error = "No retryable screenshot OCR block was found for the requested screenshot." }); + } + + return Results.Accepted(value: new + { + assistantContextPath = trigger.AssistantContextPath, + screenshotPath = trigger.ScreenshotPath, + screenshotId = trigger.ScreenshotId, + state = "ocr retrying" + }); +} + app.MapPost("/asr/transcribe-file", async ( AsrDiagnosticRequest request, AsrDiagnosticService diagnostics, @@ -544,6 +668,14 @@ public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null); public sealed record SummaryRetryRequest(string SummaryPath); +public sealed record ScreenshotOcrRetryRequest( + string MeetingNotePath, + string TranscriptPath, + string AssistantContextPath, + string SummaryPath, + string ScreenshotPath, + string ScreenshotId); + public sealed record SpeakerIdentityMergeDiagnosticRequest(double? RecentDays = null); public sealed record WorkflowConfigurationReloadResponse(string? RulesPath); diff --git a/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs b/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs index 2d3e64f..1195f71 100644 --- a/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs +++ b/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs @@ -66,32 +66,22 @@ public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBackl return items; } - public async Task CompleteAsync(string id, CancellationToken cancellationToken) + public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) { var folder = GetBacklogFolder(options); - var path = GetItemPath(folder, id); + var path = GetItemPath(folder, item.Id); if (!File.Exists(path)) { - return; - } - - OfflineTranscriptionBacklogItem? item = null; - try - { - item = JsonSerializer.Deserialize( - await File.ReadAllTextAsync(path, cancellationToken), - JsonOptions); - } - catch (JsonException exception) - { - logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path); + return Task.CompletedTask; } File.Delete(path); - if (item is not null && File.Exists(item.AudioPath)) + if (File.Exists(item.AudioPath)) { DeleteCompletedAudio(item.AudioPath); } + + return Task.CompletedTask; } private static string GetBacklogFolder(MeetingAssistantOptions options) diff --git a/MeetingAssistant/Recording/IMicrophoneDeviceProvider.cs b/MeetingAssistant/Recording/IMicrophoneDeviceProvider.cs new file mode 100644 index 0000000..301485a --- /dev/null +++ b/MeetingAssistant/Recording/IMicrophoneDeviceProvider.cs @@ -0,0 +1,12 @@ +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public interface IMicrophoneDeviceProvider +{ + IReadOnlyList GetAvailableMicrophones(); + + MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options); + + IWaveIn CreateCapture(MeetingAssistantOptions options); +} diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs index 261e2dc..fd4fd4f 100644 --- a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -28,6 +28,7 @@ public sealed class MeetingRecordingCoordinator private readonly ILaunchProfileOptionsProvider? launchProfiles; private readonly IMeetingWorkflowEngine meetingWorkflowEngine; private readonly IMeetingScreenshotService screenshotService; + private readonly IMeetingNoteImageOcrService meetingNoteImageOcrService; private readonly IMeetingRunArtifactCleaner artifactCleaner; private readonly IMeetingInactivityPromptService inactivityPromptService; private readonly IMeetingInactivityClock inactivityClock; @@ -58,6 +59,7 @@ public sealed class MeetingRecordingCoordinator ILaunchProfileOptionsProvider? launchProfiles = null, IMeetingWorkflowEngine? meetingWorkflowEngine = null, IMeetingScreenshotService? screenshotService = null, + IMeetingNoteImageOcrService? meetingNoteImageOcrService = null, IMeetingRunArtifactCleaner? artifactCleaner = null, IMeetingInactivityPromptService? inactivityPromptService = null, IMeetingInactivityClock? inactivityClock = null, @@ -78,6 +80,7 @@ public sealed class MeetingRecordingCoordinator this.launchProfiles = launchProfiles; this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance; this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance; + this.meetingNoteImageOcrService = meetingNoteImageOcrService ?? NoopMeetingNoteImageOcrService.Instance; this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner(); this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService(); this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock(); @@ -142,12 +145,28 @@ public sealed class MeetingRecordingCoordinator public async Task StartAsync(CancellationToken cancellationToken) { - return await StartAsync(null, cancellationToken); + return await StartAsync(null, null, suppressMetadataLookup: false, cancellationToken); + } + + public async Task StartFromPromptAsync( + MeetingMetadata? metadata, + CancellationToken cancellationToken) + { + return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken); } public async Task StartAsync( string? launchProfileName, CancellationToken cancellationToken) + { + return await StartAsync(launchProfileName, null, suppressMetadataLookup: false, cancellationToken); + } + + private async Task StartAsync( + string? launchProfileName, + MeetingMetadata? suppliedMetadata, + bool suppressMetadataLookup, + CancellationToken cancellationToken) { await gate.WaitAsync(cancellationToken); try @@ -164,14 +183,16 @@ public sealed class MeetingRecordingCoordinator var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken); var assistantContextPath = GetAssistantContextPath(startedAt, runOptions); var summaryPath = GetSummaryPath(startedAt, runOptions); + var meetingNote = MeetingNoteTemplate.Create( + title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}", + startTime: startedAt, + attendees: [], + transcriptPath: currentSession.TranscriptPath, + assistantContextPath: assistantContextPath, + summaryPath: summaryPath); + currentMeetingNote = await meetingNoteStore.SaveAsync( - MeetingNoteTemplate.Create( - title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}", - startTime: startedAt, - attendees: [], - transcriptPath: currentSession.TranscriptPath, - assistantContextPath: assistantContextPath, - summaryPath: summaryPath), + meetingNote, runOptions, cancellationToken); currentArtifacts = new MeetingSessionArtifacts( @@ -179,12 +200,32 @@ public sealed class MeetingRecordingCoordinator currentSession.TranscriptPath, assistantContextPath, summaryPath); - await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, currentMeetingNote, "", null, cancellationToken); + await meetingArtifactStore.CreateAssistantContextAsync( + currentArtifacts, + currentMeetingNote, + "", + null, + cancellationToken); await meetingWorkflowEngine.RunAsync( MeetingWorkflowEvent.Created(currentArtifacts), runOptions, cancellationToken); currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + if (suppliedMetadata is not null) + { + await ApplyMeetingMetadataAsync(currentMeetingNote, suppliedMetadata, runOptions, cancellationToken); + currentMeetingNote = await meetingNoteStore.SaveAsync( + currentMeetingNote, + runOptions, + cancellationToken); + await meetingArtifactStore.UpdateAssistantContextMetadataAsync( + currentArtifacts, + currentMeetingNote, + suppliedMetadata.Agenda, + suppliedMetadata.ScheduledEnd, + cancellationToken); + } + await meetingArtifactStore.UpdateAssistantContextMeetingAsync( currentArtifacts, currentMeetingNote, @@ -226,9 +267,20 @@ public sealed class MeetingRecordingCoordinator } currentRun = run; - _ = Task.Run( - () => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run), - CancellationToken.None); + if (suppliedMetadata is null && !suppressMetadataLookup) + { + _ = Task.Run( + () => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run), + CancellationToken.None); + } + else + { + await TransitionMeetingAsync( + run, + AssistantContextState.Transcribing, + cancellationToken); + } + logger.LogInformation("Meeting recording started"); return CurrentStatus; @@ -445,7 +497,7 @@ public sealed class MeetingRecordingCoordinator TimeSpan.Zero, TimeSpan.Zero, "System", - $"Transcription profile changed to {targetProfile.Name}."); + $"Transcription profile changed to {targetProfile.Name}. Speaker recognition and identities reset."); run.AddLiveSegment(marker); await AppendTranscriptSegmentAsync(run, marker, cancellationToken); @@ -642,14 +694,23 @@ public sealed class MeetingRecordingCoordinator CancellationToken cancellationToken) { var formatted = TranscriptLineFormatter.Format(segment); - var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId && - run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference) - ? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken) - : segment.Kind == TranscriptionSegmentKind.Marker && - segment.MarkerId is { } existingMarkerId && - run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference) - ? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken) - : await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken); + TranscriptLineReference lineReference; + if (segment.ReplacesMarkerId is { } replacesMarkerId && + run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference)) + { + lineReference = await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken); + } + else if (segment.Kind == TranscriptionSegmentKind.Marker && + segment.MarkerId is { } existingMarkerId && + run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference)) + { + lineReference = await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken); + } + else + { + lineReference = await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken); + } + if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId) { run.SetTranscriptMarker(markerId, lineReference); @@ -1505,6 +1566,10 @@ public sealed class MeetingRecordingCoordinator try { + await meetingNoteImageOcrService.ProcessMeetingNoteImageEmbedsAsync( + run.Artifacts, + run.Options, + cancellationToken); await screenshotService.WaitForPendingOcrAsync( run.Artifacts, run.Options.Screenshots.Ocr.GetEffectiveTimeout(), diff --git a/MeetingAssistant/Recording/MicrophoneDevice.cs b/MeetingAssistant/Recording/MicrophoneDevice.cs new file mode 100644 index 0000000..7050e30 --- /dev/null +++ b/MeetingAssistant/Recording/MicrophoneDevice.cs @@ -0,0 +1,9 @@ +namespace MeetingAssistant.Recording; + +public sealed record MicrophoneDevice( + string Id, + string Name); + +public sealed record MicrophoneDeviceSnapshot( + IReadOnlyList Available, + MicrophoneDevice? Current); diff --git a/MeetingAssistant/Recording/MicrophoneDeviceSelection.cs b/MeetingAssistant/Recording/MicrophoneDeviceSelection.cs new file mode 100644 index 0000000..d53079e --- /dev/null +++ b/MeetingAssistant/Recording/MicrophoneDeviceSelection.cs @@ -0,0 +1,54 @@ +namespace MeetingAssistant.Recording; + +public sealed class MicrophoneDeviceSelection +{ + private readonly object sync = new(); + private string? selectedDeviceId; + + public string? SelectedDeviceId + { + get + { + lock (sync) + { + return selectedDeviceId; + } + } + } + + public void Select(string deviceId) + { + lock (sync) + { + selectedDeviceId = string.IsNullOrWhiteSpace(deviceId) + ? null + : deviceId.Trim(); + } + } + + public MicrophoneDevice? Resolve( + string? configuredDeviceId, + MicrophoneDevice? defaultDevice, + IReadOnlyList availableDevices) + { + var selected = SelectedDeviceId; + return FindById(availableDevices, selected) ?? + FindById(availableDevices, configuredDeviceId) ?? + defaultDevice; + } + + private static MicrophoneDevice? FindById( + IReadOnlyList devices, + string? id) + { + if (string.IsNullOrWhiteSpace(id)) + { + return null; + } + + return devices.FirstOrDefault(device => string.Equals( + device.Id, + id.Trim(), + StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/MeetingAssistant/Recording/NaudioCaptureSource.cs b/MeetingAssistant/Recording/NaudioCaptureSource.cs index aa74053..69a570a 100644 --- a/MeetingAssistant/Recording/NaudioCaptureSource.cs +++ b/MeetingAssistant/Recording/NaudioCaptureSource.cs @@ -6,10 +6,14 @@ namespace MeetingAssistant.Recording; public sealed class MicrophoneAudioSource : IMeetingAudioSource { + private readonly IMicrophoneDeviceProvider microphones; private readonly ILogger logger; - public MicrophoneAudioSource(ILogger logger) + public MicrophoneAudioSource( + IMicrophoneDeviceProvider microphones, + ILogger logger) { + this.microphones = microphones; this.logger = logger; } @@ -22,7 +26,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource MeetingAssistantOptions options, CancellationToken cancellationToken) { - return CaptureAsync(new WasapiCapture(), options, cancellationToken); + return CaptureAsync(microphones.CreateCapture(options), options, cancellationToken); } private IAsyncEnumerable CaptureAsync( diff --git a/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs b/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs index 6c1534e..b17f816 100644 --- a/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs +++ b/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs @@ -6,7 +6,7 @@ public interface IOfflineTranscriptionBacklog Task> ListAsync(CancellationToken cancellationToken); - Task CompleteAsync(string id, CancellationToken cancellationToken); + Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken); } public sealed record OfflineTranscriptionBacklogItem( @@ -38,7 +38,7 @@ public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBackl return Task.FromResult>([]); } - public Task CompleteAsync(string id, CancellationToken cancellationToken) + public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs b/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs index f8e8120..50970f1 100644 --- a/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs +++ b/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs @@ -4,14 +4,11 @@ using MeetingAssistant.Summary; using MeetingAssistant.Transcription; using MeetingAssistant.Workflow; using Microsoft.Extensions.Options; -using NAudio.Wave; namespace MeetingAssistant.Recording; public sealed class OfflineTranscriptionBacklogProcessor { - private const int MaxChunkDurationMilliseconds = 1000; - private readonly IOfflineTranscriptionBacklog backlog; private readonly ISpeechRecognitionPipelineFactory pipelineFactory; private readonly ITranscriptStore transcriptStore; @@ -72,7 +69,7 @@ public sealed class OfflineTranscriptionBacklogProcessor try { await ProcessAsync(item, cancellationToken); - await backlog.CompleteAsync(item.Id, cancellationToken); + await backlog.CompleteAsync(item, cancellationToken); logger.LogInformation( "Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}", item.Id, @@ -215,29 +212,9 @@ public sealed class OfflineTranscriptionBacklogProcessor ISpeechRecognitionPipeline pipeline, CancellationToken cancellationToken) { - using var reader = new WaveFileReader(audioPath); - var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond; - var chunkSize = Math.Max( - reader.WaveFormat.BlockAlign, - bytesPerSecond * MaxChunkDurationMilliseconds / 1000); - chunkSize -= chunkSize % reader.WaveFormat.BlockAlign; - - var buffer = new byte[chunkSize]; - while (true) + await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(audioPath, cancellationToken: cancellationToken)) { - cancellationToken.ThrowIfCancellationRequested(); - var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); - if (read == 0) - { - break; - } - - await pipeline.WriteAsync( - new AudioChunk( - buffer[..read], - reader.WaveFormat.SampleRate, - reader.WaveFormat.Channels), - cancellationToken); + await pipeline.WriteAsync(chunk, cancellationToken); } } } diff --git a/MeetingAssistant/Recording/PcmWavAudioChunkReader.cs b/MeetingAssistant/Recording/PcmWavAudioChunkReader.cs new file mode 100644 index 0000000..7a9b238 --- /dev/null +++ b/MeetingAssistant/Recording/PcmWavAudioChunkReader.cs @@ -0,0 +1,48 @@ +using System.Runtime.CompilerServices; +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public static class PcmWavAudioChunkReader +{ + private const int MaxChunkDurationMilliseconds = 1000; + + public static async IAsyncEnumerable ReadChunksAsync( + string path, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var reader = new WaveFileReader(path); + try + { + var format = reader.WaveFormat; + if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16) + { + throw new InvalidDataException( + $"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample."); + } + + var chunkSize = Math.Max( + format.BlockAlign, + format.AverageBytesPerSecond * MaxChunkDurationMilliseconds / 1000); + chunkSize -= chunkSize % format.BlockAlign; + + var buffer = new byte[chunkSize]; + int read; + while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0) + { + yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels); + } + } + finally + { + try + { + reader.Dispose(); + } + catch (NotSupportedException) + { + // NAudio can throw while disposing reader streams from async iterators after all chunks were read. + } + } + } +} diff --git a/MeetingAssistant/Recording/WindowsMicrophoneDeviceProvider.cs b/MeetingAssistant/Recording/WindowsMicrophoneDeviceProvider.cs new file mode 100644 index 0000000..fe17c45 --- /dev/null +++ b/MeetingAssistant/Recording/WindowsMicrophoneDeviceProvider.cs @@ -0,0 +1,79 @@ +using NAudio.CoreAudioApi; +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider +{ + private readonly MicrophoneDeviceSelection selection; + private readonly ILogger logger; + + public WindowsMicrophoneDeviceProvider( + MicrophoneDeviceSelection selection, + ILogger logger) + { + this.selection = selection; + this.logger = logger; + } + + public IReadOnlyList GetAvailableMicrophones() + { + try + { + using var enumerator = new MMDeviceEnumerator(); + return enumerator + .EnumerateAudioEndPoints(DataFlow.Capture, DeviceState.Active) + .Select(ToMicrophoneDevice) + .OrderBy(device => device.Name, StringComparer.CurrentCultureIgnoreCase) + .ToArray(); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not enumerate microphone capture endpoints"); + return []; + } + } + + public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options) + { + var devices = GetAvailableMicrophones(); + return new MicrophoneDeviceSnapshot( + devices, + selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices)); + } + + public IWaveIn CreateCapture(MeetingAssistantOptions options) + { + var current = GetMicrophoneSnapshot(options).Current; + if (current is null) + { + logger.LogInformation("Starting microphone capture from Windows default capture endpoint"); + return new WasapiCapture(); + } + + logger.LogInformation( + "Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})", + current.Name, + current.Id); + using var enumerator = new MMDeviceEnumerator(); + return new WasapiCapture(enumerator.GetDevice(current.Id)); + } + + private static MicrophoneDevice? GetDefaultMicrophone() + { + try + { + using var enumerator = new MMDeviceEnumerator(); + return ToMicrophoneDevice(enumerator.GetDefaultAudioEndpoint(DataFlow.Capture, Role.Console)); + } + catch + { + return null; + } + } + + private static MicrophoneDevice ToMicrophoneDevice(MMDevice device) + { + return new MicrophoneDevice(device.ID, device.FriendlyName); + } +} diff --git a/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs b/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs index 9f23e57..9ad3f45 100644 --- a/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs +++ b/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs @@ -3,6 +3,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; +using MeetingAssistant.MeetingNotes; namespace MeetingAssistant.Screenshots; @@ -104,41 +105,69 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient private static ScreenshotOcrResult ParseOcrResult(string text) { ScreenshotCropCoordinates? crop = null; + IReadOnlyList attendees = []; var cleaned = JsonCodeBlockRegex().Replace(text, match => { - if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop)) + if (TryParseMetadata(match.Groups["json"].Value, out var parsedCrop, out var parsedAttendees)) { crop = parsedCrop; + attendees = parsedAttendees; return ""; } return match.Value; }).Trim(); - return new ScreenshotOcrResult(cleaned, crop); + return new ScreenshotOcrResult(cleaned, crop, attendees); } - private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop) + private static bool TryParseMetadata( + string json, + out ScreenshotCropCoordinates? crop, + out IReadOnlyList attendees) { crop = null; + attendees = []; try { using var document = JsonDocument.Parse(json); - if (!document.RootElement.TryGetProperty("crop", out var cropElement) || - cropElement.ValueKind == JsonValueKind.Null) - { - return true; - } - - if (cropElement.ValueKind != JsonValueKind.Object || - !TryGetInt(cropElement, "x", out var x) || - !TryGetInt(cropElement, "y", out var y) || - !TryGetInt(cropElement, "width", out var width) || - !TryGetInt(cropElement, "height", out var height)) + if (document.RootElement.ValueKind != JsonValueKind.Object) { return false; } - crop = new ScreenshotCropCoordinates(x, y, width, height); + var hasMetadata = false; + if (document.RootElement.TryGetProperty("attendees", out var attendeesElement)) + { + hasMetadata = true; + if (attendeesElement.ValueKind == JsonValueKind.Array) + { + attendees = MeetingAttendeeNames.NormalizeDistinct(attendeesElement + .EnumerateArray() + .Where(element => element.ValueKind == JsonValueKind.String) + .Select(element => element.GetString() ?? "")); + } + } + + if (!document.RootElement.TryGetProperty("crop", out var cropElement)) + { + return hasMetadata; + } + + hasMetadata = true; + if (cropElement.ValueKind == JsonValueKind.Null) + { + return true; + } + + if (cropElement.ValueKind == JsonValueKind.Object && + TryGetInt(cropElement, "x", out var x) && + TryGetInt(cropElement, "y", out var y) && + TryGetInt(cropElement, "width", out var width) && + TryGetInt(cropElement, "height", out var height)) + { + crop = new ScreenshotCropCoordinates(x, y, width, height); + } + return true; } catch (JsonException) diff --git a/MeetingAssistant/Screenshots/MeetingScreenshotService.cs b/MeetingAssistant/Screenshots/MeetingScreenshotService.cs index 3cb860f..36bceef 100644 --- a/MeetingAssistant/Screenshots/MeetingScreenshotService.cs +++ b/MeetingAssistant/Screenshots/MeetingScreenshotService.cs @@ -2,7 +2,9 @@ using System.Collections.Concurrent; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; +using System.Text.RegularExpressions; using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Speakers; namespace MeetingAssistant.Screenshots; @@ -22,7 +24,24 @@ public interface IScreenshotOcrClient CancellationToken cancellationToken); } -public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop); +public sealed record ScreenshotOcrResult +{ + public ScreenshotOcrResult( + string text, + ScreenshotCropCoordinates? crop, + IReadOnlyList? attendees = null) + { + Text = text; + Crop = crop; + Attendees = attendees ?? []; + } + + public string Text { get; init; } + + public ScreenshotCropCoordinates? Crop { get; init; } + + public IReadOnlyList Attendees { get; init; } +} public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height); @@ -46,11 +65,36 @@ public interface IMeetingScreenshotService CancellationToken cancellationToken); } +public interface IMeetingNoteImageOcrService +{ + Task ProcessMeetingNoteImageEmbedsAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions options, + CancellationToken cancellationToken); +} + +public interface IScreenshotOcrRetryRunner +{ + Task TriggerOcrRetryAsync( + MeetingSessionArtifacts artifacts, + string screenshotPath, + string screenshotId, + MeetingAssistantOptions options, + CancellationToken cancellationToken); +} + public sealed record MeetingScreenshotCaptureResult( string ScreenshotPath, TimeSpan MeetingTimestamp, bool OcrStarted); +public sealed record MeetingScreenshotOcrRetryTriggerResult( + string AssistantContextPath, + string ScreenshotPath, + string ScreenshotId); + +public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount); + public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService { public static NoopMeetingScreenshotService Instance { get; } = new(); @@ -80,12 +124,31 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService { return Task.CompletedTask; } + } -public sealed class MeetingScreenshotService : IMeetingScreenshotService +public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService +{ + public static NoopMeetingNoteImageOcrService Instance { get; } = new(); + + public Task ProcessMeetingNoteImageEmbedsAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult(new MeetingNoteImageOcrQueueResult(0)); + } +} + +public sealed partial class MeetingScreenshotService : + IMeetingScreenshotService, + IScreenshotOcrRetryRunner, + IMeetingNoteImageOcrService { private readonly IActiveWindowScreenshotCapture screenshotCapture; private readonly IMeetingArtifactStore artifactStore; + private readonly IMeetingNoteStore meetingNoteStore; + private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer; private readonly IScreenshotOcrClient ocrClient; private readonly ILogger logger; private readonly ConcurrentDictionary> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase); @@ -94,11 +157,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService public MeetingScreenshotService( IActiveWindowScreenshotCapture screenshotCapture, IMeetingArtifactStore artifactStore, + IMeetingNoteStore meetingNoteStore, + ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer, IScreenshotOcrClient ocrClient, ILogger logger) { this.screenshotCapture = screenshotCapture; this.artifactStore = artifactStore; + this.meetingNoteStore = meetingNoteStore; + this.attendeeCanonicalizer = attendeeCanonicalizer; this.ocrClient = ocrClient; this.logger = logger; } @@ -135,7 +202,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService TrackOcr( artifacts, ocrCancellation, - RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token)); + RunOcrAsync( + artifacts, + screenshotPath, + screenshotId, + options, + ScreenshotOcrPolicy.CapturedScreenshot, + ocrCancellation.Token)); } logger.LogInformation( @@ -212,6 +285,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken); } + public async Task TriggerOcrRetryAsync( + MeetingSessionArtifacts artifacts, + string screenshotPath, + string screenshotId, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(screenshotId) || + string.IsNullOrWhiteSpace(screenshotPath) || + !File.Exists(screenshotPath) || + !File.Exists(artifacts.AssistantContextPath)) + { + return null; + } + + var replaced = await ReplaceOcrPlaceholderAsync( + artifacts.AssistantContextPath, + screenshotId, + "_OCR retry pending..._", + cancellationToken, + preserveMarkers: true, + appendWhenMissing: false); + if (!replaced) + { + return null; + } + + var ocrCancellation = new CancellationTokenSource(); + TrackOcr( + artifacts, + ocrCancellation, + RunOcrAsync( + artifacts, + screenshotPath, + screenshotId, + options, + ScreenshotOcrPolicy.CapturedScreenshot, + ocrCancellation.Token)); + return new MeetingScreenshotOcrRetryTriggerResult( + artifacts.AssistantContextPath, + screenshotPath, + screenshotId); + } + + public async Task ProcessMeetingNoteImageEmbedsAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath)) + { + return new MeetingNoteImageOcrQueueResult(0); + } + + var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken); + var embeds = FindMeetingNoteImageEmbeds( + meetingNote.UserNotes, + artifacts.MeetingNotePath, + options); + var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>(); + foreach (var embed in embeds) + { + cancellationToken.ThrowIfCancellationRequested(); + var ocrId = Guid.NewGuid().ToString("N"); + await artifactStore.AppendAssistantContextAsync( + artifacts, + CreateMeetingNoteImageMarkdown( + ocrId, + embed.OriginalEmbed, + embed.ImagePath, + artifacts.AssistantContextPath), + cancellationToken); + queuedEmbeds.Add((embed, ocrId)); + } + + foreach (var (embed, ocrId) in queuedEmbeds) + { + cancellationToken.ThrowIfCancellationRequested(); + var ocrCancellation = new CancellationTokenSource(); + TrackOcr( + artifacts, + ocrCancellation, + RunOcrAsync( + artifacts, + embed.ImagePath, + ocrId, + options, + ScreenshotOcrPolicy.MeetingNoteImage, + ocrCancellation.Token)); + } + + return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count); + } + private async Task SaveScreenshotAsync( MeetingSessionArtifacts artifacts, byte[] pngBytes, @@ -234,6 +401,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService string screenshotPath, string screenshotId, MeetingAssistantOptions options, + ScreenshotOcrPolicy policy, CancellationToken cancellationToken) { var timeout = options.Screenshots.Ocr.GetEffectiveTimeout(); @@ -248,17 +416,26 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService : options.Screenshots.Ocr.Prompt, options, timeoutSource.Token); - var cropMarkdown = await TryCreateCropMarkdownAsync( - artifacts.AssistantContextPath, - screenshotPath, - result.Crop, - timeoutSource.Token); + var cropMarkdown = policy.AllowCrop + ? await TryCreateCropMarkdownAsync( + artifacts.AssistantContextPath, + screenshotPath, + result.Crop, + timeoutSource.Token) + : ""; await ReplaceOcrPlaceholderAsync( artifacts.AssistantContextPath, screenshotId, cropMarkdown + "### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(), CancellationToken.None); + if (policy.AddAttendees) + { + await AddOcrAttendeesAsync( + artifacts, + result.Attendees, + CancellationToken.None); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -269,8 +446,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService await ReplaceOcrPlaceholderAsync( artifacts.AssistantContextPath, screenshotId, - $"_OCR timed out after {timeout:g}._", - CancellationToken.None); + CreateOcrFailureMarkdown( + artifacts, + screenshotPath, + screenshotId, + options, + $"_OCR timed out after {timeout:g}._", + policy.IncludeRetryLink), + CancellationToken.None, + preserveMarkers: true); } catch (Exception exception) { @@ -278,16 +462,63 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService await ReplaceOcrPlaceholderAsync( artifacts.AssistantContextPath, screenshotId, - $"_OCR failed: {exception.Message}_", - CancellationToken.None); + CreateOcrFailureMarkdown( + artifacts, + screenshotPath, + screenshotId, + options, + $"_OCR failed: {exception.Message}_", + policy.IncludeRetryLink), + CancellationToken.None, + preserveMarkers: true); } } - private async Task ReplaceOcrPlaceholderAsync( + private async Task AddOcrAttendeesAsync( + MeetingSessionArtifacts artifacts, + IReadOnlyList attendees, + CancellationToken cancellationToken) + { + var additions = MeetingAttendeeNames.NormalizeDistinct(attendees); + if (additions.Count == 0) + { + return; + } + + try + { + var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken); + var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync( + meetingNote.Frontmatter.Attendees.Concat(additions).ToList(), + cancellationToken); + if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal)) + { + return; + } + + meetingNote.Frontmatter.Attendees = canonicalized.ToList(); + await meetingNoteStore.SaveAsync(meetingNote, cancellationToken); + logger.LogInformation( + "Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}", + additions.Count, + artifacts.MeetingNotePath); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger.LogWarning( + exception, + "Could not add screenshot OCR attendees to meeting note {MeetingNotePath}", + artifacts.MeetingNotePath); + } + } + + private async Task ReplaceOcrPlaceholderAsync( string assistantContextPath, string screenshotId, string replacement, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool preserveMarkers = false, + bool appendWhenMissing = true) { var fileLock = contextFileLocks.GetOrAdd( NormalizeContextKey(assistantContextPath), @@ -297,28 +528,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService { if (!File.Exists(assistantContextPath)) { - return; + return false; } var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken); - var startMarker = $""; - var endMarker = $""; + var startMarker = CreateOcrStartMarker(screenshotId); + var endMarker = CreateOcrEndMarker(screenshotId); var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal); var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal); if (startIndex < 0 || endIndex < startIndex) { - await File.AppendAllTextAsync( - assistantContextPath, - Environment.NewLine + replacement.TrimEnd() + Environment.NewLine, - cancellationToken); - return; + if (appendWhenMissing) + { + await File.AppendAllTextAsync( + assistantContextPath, + Environment.NewLine + replacement.TrimEnd() + Environment.NewLine, + cancellationToken); + } + + return false; } endIndex += endMarker.Length; - var updated = content[..startIndex] + - replacement.TrimEnd() + - content[endIndex..]; + var updated = preserveMarkers + ? content[..startIndex] + + startMarker + + Environment.NewLine + + replacement.TrimEnd() + + Environment.NewLine + + endMarker + + content[endIndex..] + : content[..startIndex] + + replacement.TrimEnd() + + content[endIndex..]; await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken); + return true; } finally { @@ -326,6 +570,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService } } + private static string CreateOcrFailureMarkdown( + MeetingSessionArtifacts artifacts, + string screenshotPath, + string screenshotId, + MeetingAssistantOptions options, + string status, + bool includeRetryLink = true) + { + var markdown = status.TrimEnd(); + if (includeRetryLink) + { + markdown += Environment.NewLine + + Environment.NewLine + + MeetingNoteActionLinks.CreateScreenshotOcrRetryLink( + options.Api.PublicBaseUrl, + artifacts, + screenshotPath, + screenshotId); + } + + return markdown; + } + private async Task TryCreateCropMarkdownAsync( string assistantContextPath, string screenshotPath, @@ -435,13 +702,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService return content + Environment.NewLine + Environment.NewLine + - $"" + - Environment.NewLine + - "_OCR pending..._" + - Environment.NewLine + - $""; + CreateOcrBlock(screenshotId, "_OCR pending..._"); } + private static string CreateMeetingNoteImageMarkdown( + string ocrId, + string originalEmbed, + string imagePath, + string assistantContextPath) + { + var relativePath = ToMarkdownPath(Path.GetRelativePath( + Path.GetDirectoryName(assistantContextPath)!, + imagePath)); + return "## Meeting Note Image" + + Environment.NewLine + + "Image from meeting note." + + Environment.NewLine + + $"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" + + Environment.NewLine + + $"![Meeting note image]({relativePath})" + + Environment.NewLine + + Environment.NewLine + + CreateOcrBlock(ocrId, "_OCR pending..._"); + } + + private static string CreateOcrBlock(string screenshotId, string content) + { + return CreateOcrStartMarker(screenshotId) + + Environment.NewLine + + content.TrimEnd() + + Environment.NewLine + + CreateOcrEndMarker(screenshotId); + } + + private static string CreateOcrStartMarker(string screenshotId) + { + return $""; + } + + private static string CreateOcrEndMarker(string screenshotId) + { + return $""; + } + + private static IReadOnlyList FindMeetingNoteImageEmbeds( + string userNotes, + string meetingNotePath, + MeetingAssistantOptions options) + { + if (string.IsNullOrWhiteSpace(userNotes)) + { + return []; + } + + var result = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes)) + { + AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options); + } + + foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes)) + { + AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options); + } + + return result; + } + + private static void AddResolvedEmbed( + List embeds, + HashSet seen, + string originalEmbed, + string target, + string meetingNotePath, + MeetingAssistantOptions options) + { + var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options); + if (imagePath is null || !seen.Add(imagePath)) + { + return; + } + + embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath)); + } + + private static string? ResolveMeetingNoteImagePath( + string target, + string meetingNotePath, + MeetingAssistantOptions options) + { + var cleaned = CleanEmbedTarget(target); + if (string.IsNullOrWhiteSpace(cleaned) || + cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) || + !IsSupportedImagePath(cleaned)) + { + return null; + } + + var candidates = new List(); + if (Path.IsPathRooted(cleaned)) + { + candidates.Add(VaultPath.Resolve(cleaned)); + } + else + { + candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned))); + candidates.Add(VaultPath.Resolve(options.Vault, cleaned)); + } + + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + + if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal)) + { + try + { + var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder); + return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories) + .FirstOrDefault(IsSupportedImagePath); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException) + { + return null; + } + } + + return null; + } + + private static string CleanEmbedTarget(string target) + { + var cleaned = target.Trim(); + if (cleaned.StartsWith("<", StringComparison.Ordinal) && + cleaned.EndsWith(">", StringComparison.Ordinal) && + cleaned.Length > 1) + { + cleaned = cleaned[1..^1].Trim(); + } + + var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal); + if (pipeIndex >= 0) + { + cleaned = cleaned[..pipeIndex].Trim(); + } + + var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal); + if (hashIndex >= 0) + { + cleaned = cleaned[..hashIndex].Trim(); + } + + try + { + return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar)); + } + catch (UriFormatException) + { + return cleaned.Replace('/', Path.DirectorySeparatorChar); + } + } + + private static bool IsSupportedImagePath(string path) + { + return Path.GetExtension(path).ToLowerInvariant() switch + { + ".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true, + _ => false + }; + } + + [GeneratedRegex(@"!\[\[(?[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)] + private static partial Regex ObsidianImageEmbedRegex(); + + [GeneratedRegex(@"!\[[^\]\r\n]*\]\((?[^)\r\n]+)\)", RegexOptions.CultureInvariant)] + private static partial Regex MarkdownImageEmbedRegex(); + private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder) { var expanded = Environment.ExpandEnvironmentVariables( @@ -479,4 +922,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService } private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation); + + private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink) + { + public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new( + AllowCrop: true, + AddAttendees: true, + IncludeRetryLink: true); + + public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new( + AllowCrop: false, + AddAttendees: false, + IncludeRetryLink: false); + } + + private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath); } diff --git a/MeetingAssistant/Speakers/SpeakerIdentityAttendeeCanonicalizer.cs b/MeetingAssistant/Speakers/SpeakerIdentityAttendeeCanonicalizer.cs index 9ef932e..b1d6abc 100644 --- a/MeetingAssistant/Speakers/SpeakerIdentityAttendeeCanonicalizer.cs +++ b/MeetingAssistant/Speakers/SpeakerIdentityAttendeeCanonicalizer.cs @@ -114,11 +114,7 @@ public sealed class PassthroughSpeakerIdentityAttendeeCanonicalizer : ISpeakerId IReadOnlyList attendees, CancellationToken cancellationToken) { - var distinctAttendees = attendees - .Select(attendee => attendee.Trim()) - .Where(name => !string.IsNullOrWhiteSpace(name)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToList(); + var distinctAttendees = MeetingAttendeeNames.NormalizeDistinct(attendees); return Task.FromResult>(distinctAttendees); } } diff --git a/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs b/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs index e56b37e..1c78d9d 100644 --- a/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs +++ b/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs @@ -6,10 +6,12 @@ namespace MeetingAssistant.Taskbar; public enum MeetingTaskbarAction { EditRules, + OpenSubmenu, StartRecording, StopRecording, AbortRecording, SwitchProfile, + SelectMicrophone, Exit } @@ -21,19 +23,29 @@ public sealed record MeetingTaskbarMenu( public sealed record MeetingTaskbarMenuItem( string Text, MeetingTaskbarAction Action, - string? ProfileName = null); + string? ProfileName = null, + string? MicrophoneDeviceId = null, + bool IsChecked = false, + IReadOnlyList? Items = null); public static class MeetingTaskbarMenuBuilder { public static MeetingTaskbarMenu Build( RecordingStatus status, - IReadOnlyList launchProfiles) + IReadOnlyList launchProfiles, + IReadOnlyList? microphones = null, + string? currentMicrophoneDeviceId = null) { var items = new List { new("Settings and logs", MeetingTaskbarAction.EditRules) }; + if (microphones is { Count: > 0 }) + { + items.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId)); + } + if (status.IsRecording) { items.Add(new MeetingTaskbarMenuItem( @@ -72,6 +84,24 @@ public static class MeetingTaskbarMenuBuilder items); } + private static MeetingTaskbarMenuItem BuildMicrophoneMenu( + IReadOnlyList microphones, + string? currentMicrophoneDeviceId) + { + var microphoneItems = microphones + .Select(microphone => new MeetingTaskbarMenuItem( + microphone.Name, + MeetingTaskbarAction.SelectMicrophone, + MicrophoneDeviceId: microphone.Id, + IsChecked: string.Equals(microphone.Id, currentMicrophoneDeviceId, StringComparison.Ordinal))) + .ToArray(); + + return new MeetingTaskbarMenuItem( + "Microphone", + MeetingTaskbarAction.OpenSubmenu, + Items: microphoneItems); + } + private static string BuildTooltip(RecordingStatus status) { return status.State switch @@ -104,8 +134,8 @@ public static class MeetingTaskbarMenuBuilder public static class MeetingTaskbarExitPolicy { - public static bool RequiresConfirmation(RecordingStatus status) + public static bool RequiresConfirmation(RecordingProcessState state) { - return status.State != RecordingProcessState.Idle; + return state != RecordingProcessState.Idle; } } diff --git a/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs b/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs index 93d1ab2..b50f104 100644 --- a/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs +++ b/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs @@ -14,6 +14,8 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable private readonly MeetingRecordingCoordinator coordinator; private readonly ILaunchProfileOptionsProvider launchProfiles; + private readonly IMicrophoneDeviceProvider microphones; + private readonly MicrophoneDeviceSelection microphoneSelection; private readonly IWorkflowRulesEditorWindowService rulesEditorWindow; private readonly IHostApplicationLifetime applicationLifetime; private readonly ILogger logger; @@ -29,12 +31,16 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable public UnoTaskbarIconService( MeetingRecordingCoordinator coordinator, ILaunchProfileOptionsProvider launchProfiles, + IMicrophoneDeviceProvider microphones, + MicrophoneDeviceSelection microphoneSelection, IWorkflowRulesEditorWindowService rulesEditorWindow, IHostApplicationLifetime applicationLifetime, ILogger logger) { this.coordinator = coordinator; this.launchProfiles = launchProfiles; + this.microphones = microphones; + this.microphoneSelection = microphoneSelection; this.rulesEditorWindow = rulesEditorWindow; this.applicationLifetime = applicationLifetime; this.logger = logger; @@ -167,6 +173,8 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable private MeetingTaskbarMenu BuildMenu() { var profiles = launchProfiles.GetProfiles(); + var activeOptions = GetActiveProfileOptions(profiles); + var microphoneSnapshot = microphones.GetMicrophoneSnapshot(activeOptions); var profilesSignature = string.Join( ", ", profiles.Select(profile => $"{profile.Name}:{profile.Options.Hotkey.Toggle}")); @@ -178,7 +186,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable return MeetingTaskbarMenuBuilder.Build( coordinator.CurrentStatus, - profiles); + profiles, + microphoneSnapshot.Available, + microphoneSnapshot.Current?.Id); } private PopupMenu BuildContextMenu(MeetingTaskbarMenu menu) @@ -194,14 +204,33 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable } var menuItem = menu.Items[index]; - popupMenu.Items.Add(new PopupMenuItem( - menuItem.Text, - (_, _) => _ = ExecuteMenuItemAsync(menuItem))); + popupMenu.Items.Add(BuildPopupItem(menuItem)); } return popupMenu; } + private PopupItem BuildPopupItem(MeetingTaskbarMenuItem menuItem) + { + if (menuItem.Items is { Count: > 0 }) + { + var submenu = new PopupSubMenu(menuItem.Text); + foreach (var child in menuItem.Items) + { + submenu.Items.Add(BuildPopupItem(child)); + } + + return submenu; + } + + return new PopupMenuItem( + menuItem.Text, + (_, _) => _ = ExecuteMenuItemAsync(menuItem)) + { + Checked = menuItem.IsChecked + }; + } + private async Task ExecuteMenuItemAsync(MeetingTaskbarMenuItem item) { try @@ -227,6 +256,13 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable break; case MeetingTaskbarAction.SwitchProfile: await coordinator.ToggleAsync(item.ProfileName, CancellationToken.None); + break; + case MeetingTaskbarAction.SelectMicrophone: + if (!string.IsNullOrWhiteSpace(item.MicrophoneDeviceId)) + { + microphoneSelection.Select(item.MicrophoneDeviceId); + } + break; case MeetingTaskbarAction.Exit: ExitApplication(); @@ -253,13 +289,47 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable { return string.Join( "|", - menu.Items.Select(item => $"{item.Action}:{item.ProfileName}:{item.Text}")); + FlattenMenuItems(menu.Items).Select(item => + $"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.Text}")); + } + + private static IEnumerable FlattenMenuItems( + IEnumerable items) + { + foreach (var item in items) + { + yield return item; + if (item.Items is null) + { + continue; + } + + foreach (var child in FlattenMenuItems(item.Items)) + { + yield return child; + } + } + } + + private MeetingAssistantOptions GetActiveProfileOptions(IReadOnlyList profiles) + { + var activeProfile = coordinator.CurrentStatus.LaunchProfile; + return profiles.FirstOrDefault(profile => string.Equals( + profile.Name, + activeProfile, + StringComparison.OrdinalIgnoreCase))?.Options ?? + profiles.FirstOrDefault(profile => string.Equals( + profile.Name, + "default", + StringComparison.OrdinalIgnoreCase))?.Options ?? + profiles.FirstOrDefault()?.Options ?? + new MeetingAssistantOptions(); } private void ExitApplication() { var status = coordinator.CurrentStatus; - if (MeetingTaskbarExitPolicy.RequiresConfirmation(status) && !ConfirmExitDuringProcessing(status.State)) + if (MeetingTaskbarExitPolicy.RequiresConfirmation(status.State) && !ConfirmExitDuringProcessing(status.State)) { logger.LogInformation("Taskbar exit canceled while meeting state was {State}", status.State); return; diff --git a/MeetingAssistant/Transcription/AsrDiagnosticService.cs b/MeetingAssistant/Transcription/AsrDiagnosticService.cs index 2e5bb64..6f840e2 100644 --- a/MeetingAssistant/Transcription/AsrDiagnosticService.cs +++ b/MeetingAssistant/Transcription/AsrDiagnosticService.cs @@ -1,5 +1,4 @@ using MeetingAssistant.Recording; -using NAudio.Wave; namespace MeetingAssistant.Transcription; @@ -77,7 +76,7 @@ public sealed class AsrDiagnosticService bool paceAudio, CancellationToken cancellationToken) { - await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken)) + await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(fullPath, cancellationToken: cancellationToken)) { await pipeline.WriteAsync(chunk, cancellationToken); if (paceAudio && chunk.Duration > TimeSpan.Zero) @@ -102,40 +101,6 @@ public sealed class AsrDiagnosticService return fullPath; } - - private static async IAsyncEnumerable ReadPcmWavChunks( - string path, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var reader = new WaveFileReader(path); - try - { - var format = reader.WaveFormat; - if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16) - { - throw new InvalidDataException( - $"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample."); - } - - var buffer = new byte[format.AverageBytesPerSecond]; - int read; - while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0) - { - yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels); - } - } - finally - { - try - { - reader.Dispose(); - } - catch (NotSupportedException) - { - // NAudio can throw while disposing reader streams from async iterators; diagnostics should not fail after reading all chunks. - } - } - } } public sealed record AsrDiagnosticResult(string Path, IReadOnlyList Segments); diff --git a/MeetingAssistant/appsettings.json b/MeetingAssistant/appsettings.json index e3de8ee..fd279d2 100644 --- a/MeetingAssistant/appsettings.json +++ b/MeetingAssistant/appsettings.json @@ -17,6 +17,7 @@ "TranscriptionProvider": "azure-speech", "SampleRate": 16000, "Channels": 1, + "MicrophoneDeviceId": "", "MicrophoneMixGain": 1, "SystemAudioMixGain": 1, "StopProcessingTimeout": "00:10:00", diff --git a/README.md b/README.md index 04812c6..862e210 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,9 @@ The main local endpoints are: - `POST /diagnostics/workflow/rules-editor/show` - `POST /meetings/current/summary/run` - `POST` or `GET /meetings/summary/retry` +- `POST` or `GET /meetings/screenshot-ocr/retry` -Stopping a recording stops new audio capture but lets that run continue transcription drain, speaker processing, screenshot OCR waits, summary generation, and final artifact updates. A later recording can start while an older stopped run is still finalizing; each run keeps isolated artifact paths and options. +Stopping a recording stops new audio capture but lets that run continue transcription drain, speaker processing, meeting-note image OCR, screenshot OCR waits, summary generation, and final artifact updates. A later recording can start while an older stopped run is still finalizing; each run keeps isolated artifact paths and options. ## Data And Side Effects @@ -99,6 +100,7 @@ Important settings: - `Vault`: controls the Obsidian vault root and the relative folders for notes, transcripts, summaries, assistant context, project knowledge, and dictation words. - `Recording:TranscriptionProvider`: selects `azure-speech`, `funasr`, or `whisper-local`. +- `Recording:MicrophoneDeviceId`: optionally pins the Windows microphone endpoint; blank follows the Windows default, and the tray icon can override it for later starts. - `Recording:TemporaryRecordingsFolder`: controls temporary mixed WAV storage. - `Recording:InactivitySafeguard`: prompts and then auto-stops forgotten silent recordings without aborting artifacts. - `LaunchProfiles`: overlays named profile settings onto the default profile; profile hotkeys must be distinct. @@ -156,7 +158,7 @@ Documentation-only README maintenance does not need a new OpenSpec change. - Treat the app as live user work. Check `/recording/status` before restarting, killing processes, deleting runtime files, or running scripts that might take over port `5090`. - Abort is destructive for the active run: it removes that meeting's note, transcript, assistant context, summary if present, and linked screenshot attachments, and it skips summary generation. - Too-short or empty normal stops can also delete generated artifacts instead of producing summaries. -- Summary retry links use `Api:PublicBaseUrl`, defaulting to `http://localhost:5090`. +- Summary and screenshot OCR retry links use `Api:PublicBaseUrl`, defaulting to `http://localhost:5090`. - The workflow reload endpoint reloads configuration for future workflow reads, but a meeting run that already captured options may continue with those captured options. - Public hostnames and homelab ingress are intentionally out of scope for this repo. diff --git a/docs/meeting-assistant-configuration.md b/docs/meeting-assistant-configuration.md index 07ae7c1..37dcc2d 100644 --- a/docs/meeting-assistant-configuration.md +++ b/docs/meeting-assistant-configuration.md @@ -26,6 +26,7 @@ This example is abbreviated so the most common shape is readable. The checked-in "TranscriptionProvider": "funasr", "SampleRate": 16000, "Channels": 1, + "MicrophoneDeviceId": "", "MicrophoneMixGain": 1, "SystemAudioMixGain": 1, "StopProcessingTimeout": "00:10:00", @@ -115,6 +116,7 @@ The default profile is always named `default`. Non-default profile hotkeys are r | --- | --- | | `SampleRate` | PCM sample rate for captured audio. The current providers expect 16000 Hz. | | `Channels` | PCM channel count for the mixed stream. The current providers expect mono (`1`). | +| `MicrophoneDeviceId` | Optional Windows microphone capture endpoint id. Blank uses the Windows default capture endpoint. | | `MicrophoneMixGain` | Gain applied to cleaned microphone samples before final mixing. | | `SystemAudioMixGain` | Gain applied to system loopback samples before final mixing. | | `StopProcessingTimeout` | Maximum time to wait for post-stop processing such as transcription drain, finalization, OCR wait, and summary handoff. | @@ -129,6 +131,8 @@ The default profile is always named `default`. Non-default profile hotkeys are r During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription. +On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits. + `Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV. `Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings. @@ -301,7 +305,7 @@ See `docs/meeting-workflow-engine.md` for the detailed YAML format, supported va `CalendarRecordingPrompts` controls optional Outlook Classic calendar prompts for starting recordings. It is enabled by default on Windows. -When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Classic calendar appointments through COM, filters all Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. It does not query Outlook for every prompt. The notification asks `Record meeting?` with `Yes` and `No` actions when a cached Teams meeting reaches its start window, requests reminder-style toast behavior, and remains actionable for 5 minutes. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply. +When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Classic calendar appointments through COM, filters all non-canceled Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. It does not query Outlook for every prompt. The notification asks `Record meeting?` with `Yes` and `No` actions when a cached Teams meeting reaches its start window, requests reminder-style toast behavior, and remains actionable for 5 minutes. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply. | Setting | Purpose | | --- | --- | @@ -313,7 +317,7 @@ When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Cl `Screenshots:Hotkey` configures a global hotkey that captures the currently active window during an active meeting. Screenshots are written under `Screenshots:AttachmentsFolder`, which defaults to an `Attachments` folder beside the assistant context note, and each capture appends a timestamped markdown image link to the assistant context. -`Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Automatic summarization waits for pending screenshot OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. +`Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Automatic summarization scans the meeting note for user-added Obsidian image embeds such as `![[whiteboard.png]]` and Markdown image embeds such as `![Diagram](attachments/diagram.png)`, adds resolvable local images to the assistant context without copying them or changing the meeting note, runs OCR without crop or attendee updates, and waits for all pending OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. Failed or timed-out screenshot OCR writes a retry link that targets `/meetings/screenshot-ocr/retry` with the exact saved screenshot and OCR block id. | Setting | Purpose | | --- | --- | @@ -322,7 +326,7 @@ When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Cl | `Key` | Optional inline API key. Prefer `KeyEnv`. | | `KeyEnv` | Environment variable name for the OCR API key. | | `Model` | Optional OCR model override. Blank falls back to `Agent:Model`. | -| `Prompt` | OCR instruction prompt. The default asks for visible speakers/presenters, slide text as markdown, diagrams as Mermaid, scene descriptions, participant certainty, and presentation/shared-screen crop coordinates when confidently isolatable. | +| `Prompt` | OCR instruction prompt. The default asks for visible speakers/presenters, slide text as markdown, diagrams as Mermaid, scene descriptions, participant certainty, attendee metadata, and presentation/shared-screen crop coordinates when confidently isolatable. | | `Timeout` | Maximum time to wait for OCR before summarization continues. | ## Agent And Summary diff --git a/docs/meeting-workflow-engine.md b/docs/meeting-workflow-engine.md index 9c8a95e..ef4e05c 100644 --- a/docs/meeting-workflow-engine.md +++ b/docs/meeting-workflow-engine.md @@ -70,6 +70,8 @@ The engine runs when `MeetingRecordingCoordinator` emits a meeting workflow even - `speaker_identified`: when live or final speaker identification reports a display name. - `transcript_line`: after a formatted live transcript line is durably appended, before any changed line is rewritten in place, and before lines are used in full transcript rewrites. +When a recording is started by accepting a calendar notification, the cached appointment metadata is applied before the `created` event. Rules still receive the normal `created` event and the normal `collecting metadata` to `transcribing` state transition, but they see the accepted appointment title, attendees, agenda, and scheduled end instead of a generated placeholder or a separate Outlook current-meeting lookup result. + For every event, the engine: 1. Loads the current rules file. diff --git a/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/proposal.md b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/proposal.md new file mode 100644 index 0000000..9e66134 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/proposal.md @@ -0,0 +1,16 @@ +# Proposal + +Meeting Assistant currently records the Windows default microphone endpoint and offers no in-app way to choose a different input. Users with multiple microphones need a stable configuration default and a quick tray-menu override without changing Windows settings. + +## Changes + +- Add an optional `Recording:MicrophoneDeviceId` setting. +- Keep using the Windows default capture endpoint when the setting is blank or absent. +- Enumerate active microphone capture endpoints for the tray icon menu. +- Add a `Microphone` submenu to the tray icon menu with one checked item for the effective microphone. +- Let selecting a microphone from the tray menu change the microphone used for the next capture start. + +## Impact + +- Affects Windows audio capture setup and tray menu rendering. +- Does not change system loopback capture or transcription provider behavior. diff --git a/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/specs/meeting-recording/spec.md new file mode 100644 index 0000000..0e0dc29 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/specs/meeting-recording/spec.md @@ -0,0 +1,105 @@ +## MODIFIED Requirements + +### Requirement: Recording mode captures microphone and computer output +Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription. + +Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline. + +Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks. + +Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference. + +Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples. + +Meeting Assistant SHALL align microphone and system samples through per-source buffers before mixing and SHALL NOT emit normal live audio chunks that contain only one source while the other source is merely delayed. + +When one source stays quiet beyond the alignment timeout, Meeting Assistant SHALL mix the available source with synthetic silence for the missing source instead of blocking transcription. + +Meeting Assistant SHALL allow the final microphone/system mono mix to apply configurable microphone and system gain before combining samples. + +Meeting Assistant SHALL use the active run or launch profile recording options when configuring capture format and final microphone/system gains. + +Meeting Assistant SHALL clamp mixed samples after gain is applied. + +Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization. + +Meeting Assistant SHALL allow `Recording:MicrophoneDeviceId` to select a Windows microphone capture endpoint. + +When `Recording:MicrophoneDeviceId` is blank or absent, Meeting Assistant SHALL use the Windows default capture endpoint. + +When a microphone is selected from the tray icon menu, Meeting Assistant SHALL use that selected microphone for later recording starts until another microphone is selected or the process exits. + +The tray icon right-click menu SHALL expose a `Microphone` submenu listing active microphone capture endpoints. + +The `Microphone` submenu SHALL mark exactly one effective microphone as checked. + +When no runtime microphone override is selected, the checked microphone SHALL be the configured microphone when it is available, otherwise the Windows default capture endpoint. + +#### Scenario: Both sources produce audio +- **WHEN** microphone and computer output audio chunks are available +- **THEN** Meeting Assistant mixes them into one PCM stream before transcription + +#### Scenario: Mixed audio uses cleaned microphone and system audio +- **GIVEN** the echo canceller cleans a microphone chunk to sample `2000` +- **AND** the matching system chunk has sample `10000` +- **WHEN** microphone and system chunks are mixed with gains `1` and `1` +- **THEN** the mixed sample is `12000` + +#### Scenario: Temporary recording stores only the mixed stream +- **GIVEN** Meeting Assistant has mixed microphone and system audio into one PCM chunk +- **WHEN** Meeting Assistant appends the chunk to the temporary recording +- **THEN** the main temporary WAV contains the mixed PCM +- **AND** no microphone or system sidecar WAV is written + +#### Scenario: Launch profile recording options configure capture and gains +- **GIVEN** an active launch profile configures sample format and microphone/system mix gains +- **WHEN** Meeting Assistant captures and mixes audio for that run +- **THEN** the microphone and system capture sources receive that launch profile recording configuration +- **AND** the mixed output uses that launch profile's microphone/system gains + +#### Scenario: Delayed sources are buffered before mixing +- **GIVEN** microphone audio arrives before matching system audio +- **WHEN** matching system audio arrives after a short delay +- **THEN** Meeting Assistant emits one mixed chunk for the aligned samples +- **AND** it does not emit separate microphone-only and system-only chunks for that delayed pair + +#### Scenario: Quiet system audio does not block microphone transcription +- **GIVEN** microphone audio arrives +- **AND** system loopback audio does not arrive within the alignment timeout +- **WHEN** Meeting Assistant mixes the available audio +- **THEN** it emits the microphone audio mixed with silent system audio +- **AND** live transcription can continue while system loopback is quiet + +#### Scenario: Continuous microphone audio does not suppress the alignment timeout +- **GIVEN** microphone audio keeps arriving +- **AND** system loopback audio stays unavailable past the alignment timeout +- **WHEN** Meeting Assistant checks the buffered microphone audio +- **THEN** it emits the buffered microphone audio mixed with silent system audio +- **AND** it does not wait indefinitely for a loopback chunk + +#### Scenario: Device-level capture cannot be verified in tests +- **WHEN** automated tests run without live audio devices +- **THEN** Meeting Assistant verifies the audio mixer through deterministic source abstractions rather than depending on physical microphone or speaker devices + +#### Scenario: Configured microphone is used for capture +- **GIVEN** `Recording:MicrophoneDeviceId` identifies an active microphone endpoint +- **WHEN** Meeting Assistant starts microphone capture +- **THEN** it captures from that endpoint + +#### Scenario: Blank microphone setting uses Windows default +- **GIVEN** `Recording:MicrophoneDeviceId` is blank +- **WHEN** Meeting Assistant starts microphone capture +- **THEN** it captures from the Windows default capture endpoint + +#### Scenario: Tray menu lists microphones with current selection checked +- **GIVEN** active microphone endpoints `integrated microphone` and `other microphone` +- **AND** `integrated microphone` is the effective microphone +- **WHEN** the taskbar menu is opened +- **THEN** it shows a `Microphone` submenu +- **AND** the `integrated microphone` item is checked +- **AND** the `other microphone` item is unchecked + +#### Scenario: Tray microphone selection changes later capture +- **GIVEN** active microphone endpoints `integrated microphone` and `other microphone` +- **WHEN** the user selects `other microphone` from the taskbar microphone submenu +- **THEN** later recording starts capture from `other microphone` diff --git a/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/tasks.md b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/tasks.md new file mode 100644 index 0000000..7f4bf7f --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-runtime-microphone-selection/tasks.md @@ -0,0 +1,9 @@ +# Tasks + +- [x] Add requirement scenarios for configured microphone selection and tray runtime selection. +- [x] Add a failing behavior test for the tray microphone submenu. +- [x] Add a failing behavior test for configured/default microphone resolution. +- [x] Implement microphone device enumeration, selection state, and capture endpoint resolution. +- [x] Render the microphone submenu and execute microphone selection actions. +- [x] Update configuration documentation. +- [x] Run focused tests and `openspec validate add-runtime-microphone-selection --strict`. diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/proposal.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/proposal.md new file mode 100644 index 0000000..2100ede --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/proposal.md @@ -0,0 +1,12 @@ +## Why +Screenshot OCR can identify visible meeting participants before the summary agent runs. When the model provides attendee names directly, Meeting Assistant should use that evidence immediately instead of waiting for later summary refinement. + +## What Changes +- Extend screenshot OCR metadata so the model can return a partial attendee list. +- Merge OCR-provided attendees into the meeting note without duplicates. +- Canonicalize attendee additions through speaker identity aliases before saving. + +## Impact +- Updates screenshot OCR prompt/default metadata shape. +- Updates OCR result parsing and screenshot OCR completion handling. +- Adds behavior tests for parsing attendees and updating meeting notes. diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/specs/meeting-summary/spec.md new file mode 100644 index 0000000..053be0c --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/specs/meeting-summary/spec.md @@ -0,0 +1,71 @@ +## MODIFIED Requirements + +### Requirement: Meeting screenshots are captured into assistant context +Meeting Assistant SHALL expose a configurable screenshot hotkey. + +When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window. + +The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note. + +After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps. + +Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout. + +When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note. + +The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content. + +When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note. + +When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image. + +When screenshot OCR metadata includes attendee names visible or otherwise deduced from the screenshot, Meeting Assistant SHALL add those attendees to the meeting note. + +Screenshot OCR attendee additions SHALL preserve existing attendees, SHALL NOT create duplicate attendees, and SHALL canonicalize names through speaker identity aliases before saving. + +Screenshot OCR attendee metadata MAY be partial and SHALL NOT remove existing attendees that are absent from the screenshot result. + +When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link. + +The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, indicate whether visible people are clearly the exact meeting participants or only a partial result, return crop coordinates only for confidently isolated presentation/shared-screen content, return any visible or deduced attendees as metadata, and otherwise describe the scene. + +#### Scenario: Screenshot is linked with meeting timestamp +- **GIVEN** a meeting started at `10:00:00` +- **WHEN** the user captures a screenshot at `10:03:05` +- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder +- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]` + +#### Scenario: OCR result is appended after screenshot +- **GIVEN** screenshot OCR is configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant appends the screenshot link to assistant context +- **AND** appends the OCR result for that screenshot after the link when processing completes + +#### Scenario: OCR crop is saved and linked before OCR text +- **GIVEN** screenshot OCR is configured +- **AND** OCR returns valid crop coordinates for a shared screen +- **WHEN** OCR processing completes +- **THEN** Meeting Assistant saves a cropped screenshot beside the original image +- **AND** links the cropped screenshot before the OCR text + +#### Scenario: OCR attendees are added to the meeting note +- **GIVEN** screenshot OCR is configured +- **AND** the current meeting note already has attendees +- **WHEN** screenshot OCR metadata returns attendee names from the screenshot +- **THEN** Meeting Assistant adds those attendees to the meeting note +- **AND** keeps existing attendees +- **AND** does not duplicate attendees that match existing names or speaker identity aliases + +#### Scenario: OCR partial attendees do not remove existing attendees +- **GIVEN** the current meeting note has attendees `Ada` and `Grace` +- **WHEN** screenshot OCR metadata returns only `Ada` +- **THEN** Meeting Assistant keeps `Grace` in the meeting note + +#### Scenario: OCR is skipped when not configured +- **GIVEN** screenshot OCR is not configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint + +#### Scenario: OCR reports whether visible people are complete or partial +- **WHEN** Meeting Assistant uses the built-in screenshot OCR prompt +- **THEN** the prompt asks the model to state whether the screenshot clearly shows exactly who is in the meeting or only a partial participant result diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/tasks.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/tasks.md new file mode 100644 index 0000000..ee20f04 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-attendees/tasks.md @@ -0,0 +1,5 @@ +- [x] 1. Add OpenSpec scenario for screenshot OCR attendee additions. +- [x] 2. Add failing behavior tests for OCR attendee parsing and meeting-note updates. +- [x] 3. Parse `attendees` from screenshot OCR metadata. +- [x] 4. Merge OCR attendees into the meeting note through alias-aware canonicalization. +- [x] 5. Run focused tests and `openspec validate add-screenshot-ocr-attendees --strict`. diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/proposal.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/proposal.md new file mode 100644 index 0000000..bca9fb8 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/proposal.md @@ -0,0 +1,12 @@ +## Why +Screenshot OCR can fail after a useful screenshot has already been saved to the assistant context. The user should be able to retry OCR for that exact screenshot from the note, matching the existing summary retry workflow. + +## What Changes +- Add a retry link when screenshot OCR fails or times out. +- Add a local retry endpoint for screenshot OCR. +- Rerun OCR for the exact saved screenshot and replace the same assistant-context OCR block. + +## Impact +- Updates screenshot OCR failure handling. +- Adds local API surface for screenshot OCR retry. +- Adds behavior tests for retry link generation and retry execution. diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/specs/meeting-summary/spec.md new file mode 100644 index 0000000..ae52c2c --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/specs/meeting-summary/spec.md @@ -0,0 +1,65 @@ +## MODIFIED Requirements + +### Requirement: Meeting screenshots are captured into assistant context +Meeting Assistant SHALL expose a configurable screenshot hotkey. + +When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window. + +The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note. + +After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps. + +Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout. + +When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note. + +The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content. + +When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note. + +When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image. + +When screenshot OCR fails or times out, Meeting Assistant SHALL write the failure status into the assistant context note with a retry link for that exact screenshot. + +When the screenshot OCR retry link is activated, Meeting Assistant SHALL rerun OCR for the saved screenshot and replace that screenshot's OCR block in the assistant context note. + +When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link. + +The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, indicate whether visible people are clearly the exact meeting participants or only a partial result, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene. + +#### Scenario: Screenshot is linked with meeting timestamp +- **GIVEN** a meeting started at `10:00:00` +- **WHEN** the user captures a screenshot at `10:03:05` +- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder +- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]` + +#### Scenario: OCR result is appended after screenshot +- **GIVEN** screenshot OCR is configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant appends the screenshot link to assistant context +- **AND** appends the OCR result for that screenshot after the link when processing completes + +#### Scenario: OCR crop is saved and linked before OCR text +- **GIVEN** screenshot OCR is configured +- **AND** OCR returns valid crop coordinates for a shared screen +- **WHEN** OCR processing completes +- **THEN** Meeting Assistant saves a cropped screenshot beside the original image +- **AND** links the cropped screenshot before the OCR text in assistant context + +#### Scenario: OCR failure can be retried for the same screenshot +- **GIVEN** screenshot OCR is configured +- **AND** OCR fails or times out for a captured screenshot +- **WHEN** Meeting Assistant writes the OCR failure status +- **THEN** the assistant context includes a retry link for that exact screenshot +- **WHEN** the retry link is activated +- **THEN** Meeting Assistant reruns OCR against the saved screenshot +- **AND** replaces that screenshot's OCR block with the retry result + +#### Scenario: OCR is skipped when not configured +- **GIVEN** screenshot OCR is not configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint + +#### Scenario: OCR reports whether visible people are complete or partial +- **WHEN** Meeting Assistant uses the built-in screenshot OCR prompt +- **THEN** the prompt asks the model to state whether the screenshot clearly shows exactly who is in the meeting or only a partial participant result diff --git a/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/tasks.md b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/tasks.md new file mode 100644 index 0000000..396d94f --- /dev/null +++ b/openspec/changes/archive/2026-06-26-add-screenshot-ocr-retry/tasks.md @@ -0,0 +1,5 @@ +- [x] 1. Add OpenSpec scenario for screenshot OCR retry. +- [x] 2. Add failing behavior tests for OCR failure retry links and retry execution. +- [x] 3. Preserve retryable OCR blocks on failure/timeout and include retry links. +- [x] 4. Add local retry endpoint for screenshot OCR. +- [x] 5. Run focused tests, full tests, and `openspec validate add-screenshot-ocr-retry --strict`. diff --git a/openspec/changes/add-tray-exit/.openspec.yaml b/openspec/changes/archive/2026-06-26-add-tray-exit/.openspec.yaml similarity index 100% rename from openspec/changes/add-tray-exit/.openspec.yaml rename to openspec/changes/archive/2026-06-26-add-tray-exit/.openspec.yaml diff --git a/openspec/changes/add-tray-exit/design.md b/openspec/changes/archive/2026-06-26-add-tray-exit/design.md similarity index 100% rename from openspec/changes/add-tray-exit/design.md rename to openspec/changes/archive/2026-06-26-add-tray-exit/design.md diff --git a/openspec/changes/add-tray-exit/proposal.md b/openspec/changes/archive/2026-06-26-add-tray-exit/proposal.md similarity index 100% rename from openspec/changes/add-tray-exit/proposal.md rename to openspec/changes/archive/2026-06-26-add-tray-exit/proposal.md diff --git a/openspec/changes/add-tray-exit/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-06-26-add-tray-exit/specs/meeting-recording/spec.md similarity index 100% rename from openspec/changes/add-tray-exit/specs/meeting-recording/spec.md rename to openspec/changes/archive/2026-06-26-add-tray-exit/specs/meeting-recording/spec.md diff --git a/openspec/changes/add-tray-exit/tasks.md b/openspec/changes/archive/2026-06-26-add-tray-exit/tasks.md similarity index 100% rename from openspec/changes/add-tray-exit/tasks.md rename to openspec/changes/archive/2026-06-26-add-tray-exit/tasks.md diff --git a/openspec/changes/azure-speech-offline-resilience/design.md b/openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/design.md similarity index 100% rename from openspec/changes/azure-speech-offline-resilience/design.md rename to openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/design.md diff --git a/openspec/changes/azure-speech-offline-resilience/proposal.md b/openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/proposal.md similarity index 100% rename from openspec/changes/azure-speech-offline-resilience/proposal.md rename to openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/proposal.md diff --git a/openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md b/openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/specs/meeting-transcription/spec.md similarity index 100% rename from openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md rename to openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/specs/meeting-transcription/spec.md diff --git a/openspec/changes/azure-speech-offline-resilience/tasks.md b/openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/tasks.md similarity index 100% rename from openspec/changes/azure-speech-offline-resilience/tasks.md rename to openspec/changes/archive/2026-06-26-azure-speech-offline-resilience/tasks.md diff --git a/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/proposal.md b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/proposal.md new file mode 100644 index 0000000..8378e75 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/proposal.md @@ -0,0 +1,8 @@ +## Why +The transcript marker for switching transcription profiles currently tells the summarizer that the profile changed, but not that speaker recognition state was reset. That can make later transcript interpretation less clear. + +## What Changes +- Extend the profile-switch transcript marker to state that speaker recognition and identities were reset. + +## Impact +- Updates the active profile-switch recording behavior and its test coverage. diff --git a/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/specs/meeting-recording/spec.md new file mode 100644 index 0000000..34d9438 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/specs/meeting-recording/spec.md @@ -0,0 +1,36 @@ +## MODIFIED Requirements + +### Requirement: Active recording can switch launch profiles +Meeting Assistant SHALL allow a launch profile hotkey or named-profile recording toggle request to switch the active meeting to that profile while recording capture is active. + +When switching profiles during an active meeting, Meeting Assistant SHALL keep the existing meeting note, transcript, assistant context, summary path, user notes, calendar metadata, projects, attendees, and assistant-context state. + +When switching profiles during an active meeting, Meeting Assistant SHALL NOT create new meeting artifacts, SHALL NOT collect calendar metadata again, and SHALL NOT run the summary pipeline for the old profile segment. + +When switching profiles during an active meeting, Meeting Assistant SHALL stop and drain the current speech recognition pipeline, append a transcript marker indicating the target profile and that speaker recognition and identities were reset, start a new speech recognition pipeline using the target profile's resolved options, and continue writing live transcription to the same transcript file. + +When audio is captured while the old speech recognition pipeline is draining, Meeting Assistant SHALL buffer that audio and send it to the new profile's speech recognition pipeline after it is ready. + +When switching profiles during an active meeting, Meeting Assistant SHALL clear run-local speaker label mappings for future transcript segments because backend speaker IDs can change across speech recognition processes. Already-written named transcript segments SHALL remain unchanged. + +#### Scenario: Active meeting switches to named profile +- **GIVEN** a meeting was started with the default launch profile +- **WHEN** the user triggers launch profile `english` while recording is active +- **THEN** Meeting Assistant keeps the existing meeting artifacts +- **AND** drains the old speech recognition pipeline without running summary generation +- **AND** appends a transcript marker for the profile switch +- **AND** the transcript marker states that speaker recognition and identities were reset +- **AND** starts transcription with the resolved `english` profile +- **AND** continues writing transcript segments to the same transcript file + +#### Scenario: Captured audio is buffered while switching +- **GIVEN** a meeting is switching from one launch profile to another +- **WHEN** audio chunks are captured before the new speech recognition pipeline is ready +- **THEN** Meeting Assistant buffers those chunks +- **AND** sends the buffered chunks to the new profile's speech recognition pipeline after it starts + +#### Scenario: Switching clears future speaker mappings only +- **GIVEN** a live speaker label was mapped to a named speaker before switching profiles +- **WHEN** Meeting Assistant switches to another launch profile +- **THEN** already-written transcript text keeps the named speaker +- **AND** later transcript segments from the new pipeline do not reuse the old backend speaker label mapping diff --git a/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/tasks.md b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/tasks.md new file mode 100644 index 0000000..018c75f --- /dev/null +++ b/openspec/changes/archive/2026-06-26-clarify-profile-switch-marker/tasks.md @@ -0,0 +1,4 @@ +- [x] 1. Add OpenSpec scenario for the clarified profile-switch marker. +- [x] 2. Add failing behavior coverage for the marker text. +- [x] 3. Update the profile-switch marker. +- [x] 4. Run focused tests and `openspec validate clarify-profile-switch-marker --strict`. diff --git a/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/proposal.md b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/proposal.md new file mode 100644 index 0000000..dd1869e --- /dev/null +++ b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/proposal.md @@ -0,0 +1,12 @@ +## Why +Users sometimes add screenshots or image references directly to the meeting note while a meeting is in progress. The summary agent should see OCR for those manually added images even when they were not captured through the Meeting Assistant screenshot hotkey. + +## What Changes +- Scan the meeting note after transcription and before summarization for Obsidian and Markdown image embeds. +- Add matching image references and OCR output to the assistant context without modifying the meeting note or copying image files. +- Wait for this OCR pass before moving the assistant context into summarization. + +## Impact +- Adds a pre-summary screenshot OCR pass over user-authored meeting note image embeds. +- Reuses the existing screenshot OCR client and pending OCR wait path. +- Does not perform crop extraction or attendee updates for meeting-note images. diff --git a/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/specs/meeting-summary/spec.md new file mode 100644 index 0000000..69d68a3 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/specs/meeting-summary/spec.md @@ -0,0 +1,70 @@ +## MODIFIED Requirements + +### Requirement: Meeting screenshots are captured into assistant context +Meeting Assistant SHALL expose a configurable screenshot hotkey. + +When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window. + +The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note. + +After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps. + +Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout. + +When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note. + +The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content. + +When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note. + +When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image. + +After transcription finishes and before summarization starts, Meeting Assistant SHALL scan the meeting note for user-authored Obsidian image embeds and Markdown image embeds. + +When configured screenshot OCR is enabled and the meeting note contains image embeds, Meeting Assistant SHALL append each resolvable image to the assistant context note, state that the image came from the meeting note, preserve the original embed text for cross-reference, and run OCR for the linked image. + +Meeting-note image OCR SHALL NOT copy the image file, SHALL NOT write crop images, SHALL NOT add attendees from OCR metadata, and SHALL NOT modify the meeting note. + +Meeting Assistant SHALL wait for meeting-note image OCR to finish or time out before transitioning the assistant context to summarizing. + +When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link. + +The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, indicate whether visible people are clearly the exact meeting participants or only a partial result, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene. + +#### Scenario: Screenshot is linked with meeting timestamp +- **GIVEN** a meeting started at `10:00:00` +- **WHEN** the user captures a screenshot at `10:03:05` +- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder +- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]` + +#### Scenario: OCR result is appended after screenshot +- **GIVEN** screenshot OCR is configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant appends the screenshot link to assistant context +- **AND** appends the OCR result for that screenshot after the link when processing completes + +#### Scenario: OCR crop is saved and linked before OCR text +- **GIVEN** screenshot OCR is configured +- **AND** OCR returns valid crop coordinates for a shared screen +- **WHEN** OCR processing completes +- **THEN** Meeting Assistant saves a cropped screenshot beside the original image +- **AND** links the cropped screenshot before the OCR text in assistant context + +#### Scenario: Meeting note image embeds are OCRed before summarization +- **GIVEN** screenshot OCR is configured +- **AND** the meeting note contains `![[whiteboard.png]]` +- **AND** the meeting note contains `![Diagram](attachments/diagram.png)` +- **WHEN** transcription finishes +- **THEN** Meeting Assistant appends both images to the assistant context as images from the meeting note +- **AND** includes the original embed text for each image +- **AND** runs OCR for each image without copying files, writing crop images, adding attendees, or modifying the meeting note +- **AND** waits for this OCR to finish or time out before transitioning to summarizing + +#### Scenario: OCR is skipped when not configured +- **GIVEN** screenshot OCR is not configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint + +#### Scenario: OCR reports whether visible people are complete or partial +- **WHEN** Meeting Assistant uses the built-in screenshot OCR prompt +- **THEN** the prompt asks the model to state whether the screenshot clearly shows exactly who is in the meeting or only a partial participant result diff --git a/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/tasks.md b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/tasks.md new file mode 100644 index 0000000..c149a6e --- /dev/null +++ b/openspec/changes/archive/2026-06-26-ocr-meeting-note-image-embeds/tasks.md @@ -0,0 +1,5 @@ +- [x] 1. Add OpenSpec scenario for meeting-note image embed OCR before summarization. +- [x] 2. Add failing behavior tests for meeting-note image OCR and pre-summary waiting. +- [x] 3. Implement image embed discovery and assistant-context OCR entries without modifying the meeting note. +- [x] 4. Hook the scan before summary state transition and wait for completion. +- [x] 5. Run focused tests, full tests, and `openspec validate ocr-meeting-note-image-embeds --strict`. diff --git a/openspec/changes/sync-projects-after-note-corrections/proposal.md b/openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/proposal.md similarity index 100% rename from openspec/changes/sync-projects-after-note-corrections/proposal.md rename to openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/proposal.md diff --git a/openspec/changes/sync-projects-after-note-corrections/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/specs/meeting-summary/spec.md similarity index 100% rename from openspec/changes/sync-projects-after-note-corrections/specs/meeting-summary/spec.md rename to openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/specs/meeting-summary/spec.md diff --git a/openspec/changes/sync-projects-after-note-corrections/tasks.md b/openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/tasks.md similarity index 100% rename from openspec/changes/sync-projects-after-note-corrections/tasks.md rename to openspec/changes/archive/2026-06-26-sync-projects-after-note-corrections/tasks.md diff --git a/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/proposal.md b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/proposal.md new file mode 100644 index 0000000..38081d4 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/proposal.md @@ -0,0 +1,14 @@ +## Why +When several Teams meetings are plausible in Outlook, a standalone "current meeting" lookup can pick the wrong appointment or no appointment. The recording-start notification already represents one specific cached calendar meeting, so accepting that notification should start the recording with that meeting's metadata. + +## What Changes +- Carry meeting title, attendees, agenda, and scheduled end in cached calendar prompt candidates. +- Start recordings accepted from a meeting-start notification with the accepted candidate's metadata instead of running a separate Outlook current-meeting lookup. +- Preserve the normal workflow lifecycle events and rules for prompted starts. +- Exclude canceled Outlook appointments from prompt candidates and standalone current-meeting metadata lookup. + +## Impact +- Updates the calendar prompt scheduler and recording controller seam. +- Extends Windows Outlook calendar prompt extraction to include metadata already used by meeting notes. +- Filters canceled Outlook appointments before prompting or selecting metadata. +- Adds tests for prompt-supplied metadata and workflow rule execution. diff --git a/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/specs/meeting-session/spec.md b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/specs/meeting-session/spec.md new file mode 100644 index 0000000..7352f8f --- /dev/null +++ b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/specs/meeting-session/spec.md @@ -0,0 +1,120 @@ +## MODIFIED Requirements + +### Requirement: Outlook Teams meetings can prompt recording start +Meeting Assistant SHALL enable scheduled Outlook Classic calendar checks for recording-start prompts by default. + +When scheduled recording prompts are enabled on Windows, Meeting Assistant SHALL periodically read the user's Outlook Classic calendar appointments for the current local day through COM into an in-memory cache. + +Meeting Assistant SHALL default the Outlook calendar sync interval to 30 minutes when scheduled recording prompts are enabled. + +Meeting Assistant SHALL schedule recording-start prompts from the cached calendar appointments rather than querying Outlook for each prompt. + +Meeting Assistant SHALL consider Teams appointments from Outlook calendar data as initial prompt candidates. The detection MAY be extended later for other meeting providers. + +Meeting Assistant SHALL exclude canceled Outlook appointments from recording-start prompt candidates. + +When a Teams appointment reaches its scheduled start window, Meeting Assistant SHALL show a native Windows app notification asking whether to record the meeting, with affirmative and negative actions. + +On Windows, the recording-start notification SHALL request reminder-style toast behavior and remain actionable for 5 minutes. + +Meeting Assistant SHALL prompt at most once per calendar appointment during a local day, regardless of whether the user accepts, declines, or ignores the notification. + +If the user accepts the recording prompt while no recording is active, Meeting Assistant SHALL start a new recording normally. + +If the user accepts the recording prompt while another recording is active, Meeting Assistant SHALL stop the active recording normally and then start the prompted meeting recording. + +When stopping an active recording for an accepted prompt, Meeting Assistant SHALL use the normal stop path so empty or too-short recordings are removed according to existing settings and other completed recordings continue normal transcription, speaker recognition, and summary processing. + +When the user accepts a recording prompt, Meeting Assistant SHALL start the recording with the accepted cached appointment's metadata and SHALL NOT perform a separate current-meeting metadata lookup for that prompted start. + +Prompted-start metadata SHALL include the accepted appointment title, attendees, agenda, and scheduled end when those values are available from the cached appointment. + +Prompted starts SHALL run the normal meeting workflow `created` rules before applying the accepted appointment metadata. + +After `created` rules run, prompted starts SHALL apply the accepted appointment metadata and then run the normal `collecting metadata` to `transcribing` state-transition workflow rules with that metadata available in the meeting note. + +#### Scenario: Teams meeting start prompts the user +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has synced Outlook Classic Teams appointments for today +- **WHEN** the appointment reaches its scheduled start window +- **THEN** Meeting Assistant shows a native Windows app notification asking whether to record the meeting +- **AND** the notification remains actionable for 5 minutes +- **AND** marks that appointment as prompted for the day + +#### Scenario: Canceled Teams meeting does not prompt recording +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has synced a canceled Outlook Classic Teams appointment for today +- **WHEN** the canceled appointment reaches its scheduled start window +- **THEN** Meeting Assistant does not show a recording prompt for that appointment + +#### Scenario: Back-to-back cached Teams meetings prompt without another Outlook sync +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has synced two Teams appointments for today that start ten minutes apart +- **WHEN** each appointment reaches its scheduled start window +- **THEN** Meeting Assistant shows a recording prompt for each appointment +- **AND** does not require another Outlook calendar sync between the prompts + +#### Scenario: User accepts prompt while idle +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** no meeting recording is active +- **WHEN** the user accepts a Teams meeting recording prompt +- **THEN** Meeting Assistant starts recording normally + +#### Scenario: User accepts prompt while already recording +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** a meeting recording is active +- **WHEN** the user accepts a Teams meeting recording prompt +- **THEN** Meeting Assistant stops the active recording normally +- **AND** starts a new recording normally after the stop request + +#### Scenario: Accepted prompt supplies meeting metadata +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has cached two current Teams appointments with different metadata +- **WHEN** the user accepts the recording prompt for one appointment +- **THEN** Meeting Assistant starts the recording with the accepted appointment's metadata +- **AND** does not perform a standalone current-meeting metadata lookup for that recording +- **AND** runs `created` workflow rules before writing the cached appointment metadata +- **AND** runs `collecting metadata` to `transcribing` workflow rules after writing the cached appointment metadata + +#### Scenario: Prompt is disabled +- **GIVEN** scheduled Outlook recording prompts are disabled +- **WHEN** a Teams appointment reaches its scheduled start +- **THEN** Meeting Assistant does not query Outlook for recording prompt candidates +- **AND** does not show a recording prompt + +### Requirement: Windows Outlook enrichment is optional +Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target. + +When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter. + +Meeting Assistant SHALL exclude canceled Outlook appointments from current Teams appointment metadata lookup. + +Meeting Assistant SHALL copy the appointment attendees to the meeting note only when the raw appointment attendee count is less than or equal to the configured `Recording:MaxMetadataAttendeeImportCount`. The default maximum SHALL be 30 attendees. + +The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text. + +#### Scenario: Current Teams appointment enriches meeting artifacts +- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment +- **THEN** Meeting Assistant uses the appointment subject as the meeting title +- **AND** writes the appointment attendees into meeting note frontmatter when the raw attendee count is within the configured import limit +- **AND** writes the appointment agenda into assistant-context frontmatter +- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter + +#### Scenario: Canceled appointment is ignored during metadata lookup +- **GIVEN** Outlook Classic exposes one canceled current Teams appointment +- **AND** Outlook Classic exposes one active current Teams appointment at the same time +- **WHEN** a Windows build starts a meeting +- **THEN** Meeting Assistant selects the active appointment metadata + +#### Scenario: Oversized attendee list is not imported +- **GIVEN** the configured metadata attendee import limit is 30 +- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment with 31 attendees +- **THEN** Meeting Assistant uses the appointment subject as the meeting title +- **AND** does not write the appointment attendees into meeting note frontmatter +- **AND** writes the appointment agenda into assistant-context frontmatter +- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter + +#### Scenario: Outlook is unavailable or ambiguous +- **WHEN** Outlook Classic is unavailable or more than one current Teams appointment is found +- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda +- **AND** omits `scheduled_end` from assistant-context frontmatter diff --git a/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/tasks.md b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/tasks.md new file mode 100644 index 0000000..0e87382 --- /dev/null +++ b/openspec/changes/archive/2026-06-26-use-prompted-calendar-metadata/tasks.md @@ -0,0 +1,6 @@ +- [x] 1. Add OpenSpec scenario for prompt-accepted metadata. +- [x] 2. Add failing behavior coverage for prompted metadata starts. +- [x] 3. Pass prompted metadata through the scheduler/controller/coordinator start path. +- [x] 4. Populate cached Outlook prompt candidates with metadata. +- [x] 5. Exclude canceled Outlook appointments from recording prompts and metadata lookup. +- [x] 6. Run focused tests, full tests, and `openspec validate use-prompted-calendar-metadata --strict`. diff --git a/openspec/specs/meeting-recording/spec.md b/openspec/specs/meeting-recording/spec.md index 72910e4..e1267fa 100644 --- a/openspec/specs/meeting-recording/spec.md +++ b/openspec/specs/meeting-recording/spec.md @@ -55,6 +55,18 @@ Meeting Assistant SHALL clamp mixed samples after gain is applied. Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization. +Meeting Assistant SHALL allow `Recording:MicrophoneDeviceId` to select a Windows microphone capture endpoint. + +When `Recording:MicrophoneDeviceId` is blank or absent, Meeting Assistant SHALL use the Windows default capture endpoint. + +When a microphone is selected from the tray icon menu, Meeting Assistant SHALL use that selected microphone for later recording starts until another microphone is selected or the process exits. + +The tray icon right-click menu SHALL expose a `Microphone` submenu listing active microphone capture endpoints. + +The `Microphone` submenu SHALL mark exactly one effective microphone as checked. + +When no runtime microphone override is selected, the checked microphone SHALL be the configured microphone when it is available, otherwise the Windows default capture endpoint. + #### Scenario: Both sources produce audio - **WHEN** microphone and computer output audio chunks are available - **THEN** Meeting Assistant mixes them into one PCM stream before transcription @@ -101,6 +113,29 @@ Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by - **WHEN** automated tests run without live audio devices - **THEN** Meeting Assistant verifies the audio mixer through deterministic source abstractions rather than depending on physical microphone or speaker devices +#### Scenario: Configured microphone is used for capture +- **GIVEN** `Recording:MicrophoneDeviceId` identifies an active microphone endpoint +- **WHEN** Meeting Assistant starts microphone capture +- **THEN** it captures from that endpoint + +#### Scenario: Blank microphone setting uses Windows default +- **GIVEN** `Recording:MicrophoneDeviceId` is blank +- **WHEN** Meeting Assistant starts microphone capture +- **THEN** it captures from the Windows default capture endpoint + +#### Scenario: Tray menu lists microphones with current selection checked +- **GIVEN** active microphone endpoints `integrated microphone` and `other microphone` +- **AND** `integrated microphone` is the effective microphone +- **WHEN** the taskbar menu is opened +- **THEN** it shows a `Microphone` submenu +- **AND** the `integrated microphone` item is checked +- **AND** the `other microphone` item is unchecked + +#### Scenario: Tray microphone selection changes later capture +- **GIVEN** active microphone endpoints `integrated microphone` and `other microphone` +- **WHEN** the user selects `other microphone` from the taskbar microphone submenu +- **THEN** later recording starts capture from `other microphone` + ### Requirement: Stopping recording drains captured audio through transcription Meeting Assistant SHALL stop capturing new audio when recording mode is stopped, but it SHALL allow already captured audio to finish running through the configured speech recognition pipeline before the recording session completes. @@ -180,7 +215,7 @@ When switching profiles during an active meeting, Meeting Assistant SHALL keep t When switching profiles during an active meeting, Meeting Assistant SHALL NOT create new meeting artifacts, SHALL NOT collect calendar metadata again, and SHALL NOT run the summary pipeline for the old profile segment. -When switching profiles during an active meeting, Meeting Assistant SHALL stop and drain the current speech recognition pipeline, append a transcript marker indicating the target profile, start a new speech recognition pipeline using the target profile's resolved options, and continue writing live transcription to the same transcript file. +When switching profiles during an active meeting, Meeting Assistant SHALL stop and drain the current speech recognition pipeline, append a transcript marker indicating the target profile and that speaker recognition and identities were reset, start a new speech recognition pipeline using the target profile's resolved options, and continue writing live transcription to the same transcript file. When audio is captured while the old speech recognition pipeline is draining, Meeting Assistant SHALL buffer that audio and send it to the new profile's speech recognition pipeline after it is ready. @@ -192,6 +227,7 @@ When switching profiles during an active meeting, Meeting Assistant SHALL clear - **THEN** Meeting Assistant keeps the existing meeting artifacts - **AND** drains the old speech recognition pipeline without running summary generation - **AND** appends a transcript marker for the profile switch +- **AND** the transcript marker states that speaker recognition and identities were reset - **AND** starts transcription with the resolved `english` profile - **AND** continues writing transcript segments to the same transcript file @@ -199,7 +235,7 @@ When switching profiles during an active meeting, Meeting Assistant SHALL clear - **GIVEN** a meeting is switching from one launch profile to another - **WHEN** audio chunks are captured before the new speech recognition pipeline is ready - **THEN** Meeting Assistant buffers those chunks -- **AND** sends the buffered chunks to the new speech recognition pipeline after it starts +- **AND** sends the buffered chunks to the new profile's speech recognition pipeline after it starts #### Scenario: Switching clears future speaker mappings only - **GIVEN** a live speaker label was mapped to a named speaker before switching profiles @@ -216,6 +252,8 @@ When a new meeting is actively recording while an older stopped meeting is still The taskbar icon right-click menu SHALL expose recording controls based on the current state and configured launch profiles. +The taskbar icon right-click menu SHALL expose an Exit action in every recording state. + When Meeting Assistant is idle or only processing older stopped meetings, the menu SHALL allow starting a meeting recording for each configured launch profile. When a meeting is actively recording, the menu SHALL allow stopping the recording and continuing transcription/summary generation. @@ -224,6 +262,10 @@ When a meeting is actively recording, the menu SHALL allow canceling the recordi When a meeting is actively recording, the menu SHALL allow switching to each configured launch profile other than the current active profile. +Selecting Exit while Meeting Assistant is idle SHALL stop the application without an additional confirmation prompt. + +Selecting Exit while Meeting Assistant is recording, transcribing, recognizing speakers, or summarizing SHALL show a confirmation dialog before stopping the application. + #### Scenario: Idle tray menu can start configured profiles - **GIVEN** launch profiles `default` and `english` are configured - **AND** no meeting recording is active @@ -243,6 +285,21 @@ When a meeting is actively recording, the menu SHALL allow switching to each con - **WHEN** a newer meeting is actively recording - **THEN** the taskbar icon indicates recording +#### Scenario: Tray menu always exposes Exit +- **GIVEN** Meeting Assistant is running +- **WHEN** the taskbar menu is opened +- **THEN** it offers an Exit action + +#### Scenario: Idle Exit stops immediately +- **GIVEN** no recording, transcription, speaker recognition, or summary work is running +- **WHEN** the user selects Exit from the taskbar menu +- **THEN** Meeting Assistant stops the application without an additional confirmation prompt + +#### Scenario: In-progress Exit asks for confirmation +- **GIVEN** Meeting Assistant is recording, transcribing, recognizing speakers, or summarizing +- **WHEN** the user selects Exit from the taskbar menu +- **THEN** Meeting Assistant asks for confirmation before stopping the application + ### Requirement: Recording inactivity safeguard stops forgotten meetings Meeting Assistant SHALL track transcript inactivity during an active recording from the later of meeting start or the most recent live transcript segment that contains text. @@ -305,3 +362,4 @@ When a recording stops normally and its meeting note, transcript, and assistant - **WHEN** the recording stops normally - **THEN** Meeting Assistant deletes the run artifacts - **AND** does not run summary generation + diff --git a/openspec/specs/meeting-session/spec.md b/openspec/specs/meeting-session/spec.md index 34dd4c3..f7fcb37 100644 --- a/openspec/specs/meeting-session/spec.md +++ b/openspec/specs/meeting-session/spec.md @@ -80,6 +80,8 @@ Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows c When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter. +Meeting Assistant SHALL exclude canceled Outlook appointments from current Teams appointment metadata lookup. + Meeting Assistant SHALL copy the appointment attendees to the meeting note only when the raw appointment attendee count is less than or equal to the configured `Recording:MaxMetadataAttendeeImportCount`. The default maximum SHALL be 30 attendees. The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text. @@ -91,6 +93,12 @@ The agenda SHALL be extracted from the appointment body content before the Teams - **AND** writes the appointment agenda into assistant-context frontmatter - **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter +#### Scenario: Canceled appointment is ignored during metadata lookup +- **GIVEN** Outlook Classic exposes one canceled current Teams appointment +- **AND** Outlook Classic exposes one active current Teams appointment at the same time +- **WHEN** a Windows build starts a meeting +- **THEN** Meeting Assistant selects the active appointment metadata + #### Scenario: Oversized attendee list is not imported - **GIVEN** the configured metadata attendee import limit is 30 - **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment with 31 attendees @@ -115,6 +123,8 @@ Meeting Assistant SHALL schedule recording-start prompts from the cached calenda Meeting Assistant SHALL consider Teams appointments from Outlook calendar data as initial prompt candidates. The detection MAY be extended later for other meeting providers. +Meeting Assistant SHALL exclude canceled Outlook appointments from recording-start prompt candidates. + When a Teams appointment reaches its scheduled start window, Meeting Assistant SHALL show a native Windows app notification asking whether to record the meeting, with affirmative and negative actions. On Windows, the recording-start notification SHALL request reminder-style toast behavior and remain actionable for 5 minutes. @@ -127,6 +137,14 @@ If the user accepts the recording prompt while another recording is active, Meet When stopping an active recording for an accepted prompt, Meeting Assistant SHALL use the normal stop path so empty or too-short recordings are removed according to existing settings and other completed recordings continue normal transcription, speaker recognition, and summary processing. +When the user accepts a recording prompt, Meeting Assistant SHALL start the recording with the accepted cached appointment's metadata and SHALL NOT perform a separate current-meeting metadata lookup for that prompted start. + +Prompted-start metadata SHALL include the accepted appointment title, attendees, agenda, and scheduled end when those values are available from the cached appointment. + +Prompted starts SHALL run the normal meeting workflow `created` rules before applying the accepted appointment metadata. + +After `created` rules run, prompted starts SHALL apply the accepted appointment metadata and then run the normal `collecting metadata` to `transcribing` state-transition workflow rules with that metadata available in the meeting note. + #### Scenario: Teams meeting start prompts the user - **GIVEN** scheduled Outlook recording prompts are enabled - **AND** Meeting Assistant has synced Outlook Classic Teams appointments for today @@ -135,6 +153,12 @@ When stopping an active recording for an accepted prompt, Meeting Assistant SHAL - **AND** the notification remains actionable for 5 minutes - **AND** marks that appointment as prompted for the day +#### Scenario: Canceled Teams meeting does not prompt recording +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has synced a canceled Outlook Classic Teams appointment for today +- **WHEN** the canceled appointment reaches its scheduled start window +- **THEN** Meeting Assistant does not show a recording prompt for that appointment + #### Scenario: Back-to-back cached Teams meetings prompt without another Outlook sync - **GIVEN** scheduled Outlook recording prompts are enabled - **AND** Meeting Assistant has synced two Teams appointments for today that start ten minutes apart @@ -155,6 +179,15 @@ When stopping an active recording for an accepted prompt, Meeting Assistant SHAL - **THEN** Meeting Assistant stops the active recording normally - **AND** starts a new recording normally after the stop request +#### Scenario: Accepted prompt supplies meeting metadata +- **GIVEN** scheduled Outlook recording prompts are enabled +- **AND** Meeting Assistant has cached two current Teams appointments with different metadata +- **WHEN** the user accepts the recording prompt for one appointment +- **THEN** Meeting Assistant starts the recording with the accepted appointment's metadata +- **AND** does not perform a standalone current-meeting metadata lookup for that recording +- **AND** runs `created` workflow rules before writing the cached appointment metadata +- **AND** runs `collecting metadata` to `transcribing` workflow rules after writing the cached appointment metadata + #### Scenario: Prompt is disabled - **GIVEN** scheduled Outlook recording prompts are disabled - **WHEN** a Teams appointment reaches its scheduled start @@ -366,3 +399,4 @@ Meeting Assistant SHALL provide a diagnostic endpoint that opens the workflow ru - **GIVEN** the rules editor chat window content fits without scrolling or is already near the bottom - **WHEN** a new user or assistant message is appended - **THEN** the conversation scrolls to the bottom of the newly rendered message content + diff --git a/openspec/specs/meeting-summary/spec.md b/openspec/specs/meeting-summary/spec.md index b42c93c..762a2c9 100644 --- a/openspec/specs/meeting-summary/spec.md +++ b/openspec/specs/meeting-summary/spec.md @@ -4,32 +4,22 @@ TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. ## Requirements ### Requirement: Meeting Assistant generates meeting outputs -The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, `end_time`, `attendees`, and `projects` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary. +The settings/logs agent SHALL instruct meeting-note correction workflows to keep project knowledge consistent when the corrected meeting note has project references. -When the summary agent calls `write_summary` with a non-empty `title` parameter, Meeting Assistant SHALL update the current meeting note title and write that title to the summary note frontmatter. +When a corrected meeting note adds a new project reference, the settings/logs agent SHALL inspect the corrected note broadly enough to find project-relevant information that should be written to that project. -When the summary agent calls `write_summary` without a `title` parameter or with a blank `title` parameter, Meeting Assistant SHALL keep the existing meeting note title and summary title behavior. +When a corrected meeting note changes specific content for an existing project reference, the settings/logs agent SHALL check whether corresponding project knowledge needs the same correction. -The summary instructions SHALL tell the summary agent to provide a concise `title` parameter to `write_summary` when the meeting note still has a generated default title and the meeting purpose is clear from context. +For each affected project, when an `AGENTS.md` file exists directly in the project folder, the settings/logs agent SHALL follow those project instructions before updating the project. When no project `AGENTS.md` exists, the settings/logs agent SHALL use judgment to decide what project file updates are warranted by the note correction. -#### Scenario: Summary agent renames default-titled meeting -- **GIVEN** the current meeting note has a generated default title -- **AND** the meeting purpose is clear from transcript, user notes, or assistant context -- **WHEN** the summary agent writes the summary -- **THEN** the summary instructions tell the agent to provide a concise `title` parameter -- **AND** Meeting Assistant updates the meeting note title and summary note title to the provided title - -#### Scenario: Summary title parameter is omitted -- **GIVEN** the current meeting note has an existing title -- **WHEN** the summary agent writes the summary without a title parameter -- **THEN** Meeting Assistant keeps the existing meeting note title -- **AND** writes the existing title to the summary note frontmatter - -#### Scenario: Summary title parameter is blank -- **GIVEN** the current meeting note has an existing title -- **WHEN** the summary agent writes the summary with a blank title parameter -- **THEN** Meeting Assistant keeps the existing meeting note title -- **AND** writes the existing title to the summary note frontmatter +#### Scenario: Settings/logs agent syncs corrected meeting notes to projects +- **GIVEN** the settings/logs agent corrects a meeting note with project references +- **WHEN** a referenced project has an `AGENTS.md` +- **THEN** the settings/logs instructions tell the agent to follow that project's instructions before updating the project +- **WHEN** a corrected note adds a new project reference +- **THEN** the instructions tell the agent to review the corrected note for project-relevant information to add to that project +- **WHEN** a corrected note changes specific information for an existing project reference +- **THEN** the instructions tell the agent to check whether project knowledge needs a matching correction ### Requirement: Meeting screenshots are captured into assistant context Meeting Assistant SHALL expose a configurable screenshot hotkey. @@ -50,6 +40,14 @@ When OCR returns valid crop coordinates within the original image bounds, Meetin When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image. +After transcription finishes and before summarization starts, Meeting Assistant SHALL scan the meeting note for user-authored Obsidian image embeds and Markdown image embeds. + +When configured screenshot OCR is enabled and the meeting note contains image embeds, Meeting Assistant SHALL append each resolvable image to the assistant context note, state that the image came from the meeting note, preserve the original embed text for cross-reference, and run OCR for the linked image. + +Meeting-note image OCR SHALL NOT copy the image file, SHALL NOT write crop images, SHALL NOT add attendees from OCR metadata, and SHALL NOT modify the meeting note. + +Meeting Assistant SHALL wait for meeting-note image OCR to finish or time out before transitioning the assistant context to summarizing. + When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link. The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, indicate whether visible people are clearly the exact meeting participants or only a partial result, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene. @@ -73,6 +71,16 @@ The default OCR prompt SHALL explain that the image is from a meeting and ask th - **THEN** Meeting Assistant saves a cropped screenshot beside the original image - **AND** links the cropped screenshot before the OCR text in assistant context +#### Scenario: Meeting note image embeds are OCRed before summarization +- **GIVEN** screenshot OCR is configured +- **AND** the meeting note contains `![[whiteboard.png]]` +- **AND** the meeting note contains `![Diagram](attachments/diagram.png)` +- **WHEN** transcription finishes +- **THEN** Meeting Assistant appends both images to the assistant context as images from the meeting note +- **AND** includes the original embed text for each image +- **AND** runs OCR for each image without copying files, writing crop images, adding attendees, or modifying the meeting note +- **AND** waits for this OCR to finish or time out before transitioning to summarizing + #### Scenario: OCR is skipped when not configured - **GIVEN** screenshot OCR is not configured - **WHEN** the user captures a screenshot @@ -131,3 +139,4 @@ The summary-agent instructions SHALL tell the agent to keep the one-line summary - **WHEN** the summary agent calls `write_summary` with a one-line summary containing a line break - **THEN** Meeting Assistant refuses the write - **AND** does not mark the summary as written + diff --git a/openspec/specs/meeting-transcription/spec.md b/openspec/specs/meeting-transcription/spec.md index c3b472a..de2211b 100644 --- a/openspec/specs/meeting-transcription/spec.md +++ b/openspec/specs/meeting-transcription/spec.md @@ -125,6 +125,24 @@ When Azure Speech is the configured speech recognition pipeline, Meeting Assista When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL use the live Azure conversation transcription segments as the finished transcript without running pyannote finalization. +When Azure Speech reports a transient connectivity interruption, Meeting Assistant SHALL keep accepting captured audio into the active in-process pipeline buffer. + +When Azure Speech is reconnecting, Meeting Assistant SHALL write a transcript marker in the form ``. + +When Azure Speech emits the next real transcript segment after reconnecting, Meeting Assistant SHALL replace the latest reconnect marker with that transcript segment. + +When Azure Speech cannot reconnect after the configured immediate retry attempts, Meeting Assistant SHALL write a transcript marker explaining that Azure Speech is disconnected, recording can continue, and transcription/summarization will continue after Azure reconnects. + +After Azure Speech reconnects through a new SDK session, Meeting Assistant SHALL clear live speaker-label assumptions for future Azure speaker labels. + +When Azure Speech is still unavailable after recording stops and transcription cannot drain before the configured stop-processing timeout, Meeting Assistant SHALL persist the stopped meeting as a durable transcription backlog item that references the completed mixed WAV and meeting artifacts. + +When a stopped meeting is persisted to the durable transcription backlog, Meeting Assistant SHALL release the active recording slot so another meeting can be recorded while the stopped meeting waits for Azure Speech to become available. + +When Azure Speech becomes available again, Meeting Assistant SHALL retry durable backlog items, rewrite the transcript from the recorded WAV, run the normal post-transcription meeting completion and summary flow, and remove the backlog item after successful completion. + +When Meeting Assistant starts, it SHALL preserve WAV files that are referenced by durable transcription backlog items instead of deleting them as stale temporary recordings. + #### Scenario: Azure Speech pipeline is configured - **WHEN** the configured provider is `azure-speech` - **THEN** Meeting Assistant streams captured PCM chunks to Azure AI Speech conversation transcription through the configured speech recognition pipeline without changing recording control or transcript persistence code @@ -145,6 +163,32 @@ When Azure Speech is the configured speech recognition pipeline, Meeting Assista - **WHEN** the configured provider is `azure-speech` and live Azure transcript segments exist - **THEN** Meeting Assistant uses the live Azure conversation transcript segments as the finished transcript +#### Scenario: Azure reconnect marker is replaced by next transcript line +- **GIVEN** Azure Speech reports a transient connectivity interruption +- **WHEN** Meeting Assistant writes `` and Azure later emits a transcript segment +- **THEN** Meeting Assistant replaces the reconnect marker with the transcript segment +- **AND** keeps later transcript lines intact + +#### Scenario: Azure disconnected marker keeps meeting recording alive +- **GIVEN** Azure Speech cannot reconnect after the configured immediate retry attempts +- **WHEN** meeting audio continues to be captured +- **THEN** Meeting Assistant writes a transcript marker explaining that Azure Speech is disconnected +- **AND** keeps buffering captured audio in the active process for later transcription + +#### Scenario: Stopped Azure meeting is queued durably while offline +- **GIVEN** Azure Speech cannot finish transcription before the recording stop-processing timeout +- **WHEN** the user stops the meeting +- **THEN** Meeting Assistant persists a durable backlog item for the stopped meeting +- **AND** keeps the completed mixed WAV referenced by that backlog item +- **AND** returns to an idle recording state so another meeting can start + +#### Scenario: Durable Azure backlog resumes after connectivity returns +- **GIVEN** a stopped Azure meeting exists in the durable transcription backlog +- **WHEN** Azure Speech can transcribe the recorded WAV +- **THEN** Meeting Assistant rewrites the transcript from the recorded WAV +- **AND** runs meeting completion and summarization +- **AND** removes the durable backlog item and its temporary WAV after successful completion + ### Requirement: FunASR can provide speaker-attributed streaming transcription Meeting Assistant SHALL provide a FunASR speech recognition pipeline that streams PCM audio to a configured FunASR WebSocket endpoint. @@ -521,3 +565,4 @@ When pyannote secondary validation is enabled, Meeting Assistant SHALL start a n - **WHEN** Meeting Assistant starts - **THEN** it begins preparing the configured pyannote runtime image and model cache without waiting for the first validation request - **AND** application startup is not blocked by the warm-up task +