From b774ccc3755db4ba9a88980aa2ad3130d46d4f91 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Mon, 15 Jun 2026 17:26:34 +0200 Subject: [PATCH] Add resilient Azure offline transcription backlog --- .../MeetingWorkflowEngineTests.cs | 106 ++- .../RecordingCoordinatorTests.cs | 613 +++++++++++++++++- MeetingAssistant.Tests/TaskbarIconTests.cs | 28 + MeetingAssistant/MeetingAssistantOptions.cs | 4 + MeetingAssistant/Program.cs | 3 + .../FileOfflineTranscriptionBacklog.cs | 134 ++++ .../Recording/ITranscriptStore.cs | 28 +- .../Recording/MeetingRecordingCoordinator.cs | 244 ++++++- .../Recording/OfflineTranscriptionBacklog.cs | 45 ++ ...fflineTranscriptionBacklogHostedService.cs | 53 ++ .../OfflineTranscriptionBacklogProcessor.cs | 243 +++++++ ...cordingSpeechRecognitionPipelineOptions.cs | 20 + .../Recording/TemporaryRecordedAudioStore.cs | 20 +- .../Recording/VaultTranscriptStore.cs | 186 ++++-- .../Taskbar/MeetingTaskbarMenu.cs | 15 +- .../Taskbar/UnoTaskbarIconService.Windows.cs | 38 +- ...ureSpeechStreamingTranscriptionProvider.cs | 359 +++++++--- .../Transcription/TranscriptionSegment.cs | 15 +- .../Workflow/MeetingWorkflowEngine.cs | 61 +- .../MeetingWorkflowTemplateRenderer.cs | 4 +- docs/meeting-assistant-configuration.md | 8 +- docs/meeting-workflow-engine.md | 12 +- openspec/changes/add-tray-exit/.openspec.yaml | 2 + openspec/changes/add-tray-exit/design.md | 18 + openspec/changes/add-tray-exit/proposal.md | 21 + .../specs/meeting-recording/spec.md | 58 ++ openspec/changes/add-tray-exit/tasks.md | 7 + .../azure-speech-offline-resilience/design.md | 27 + .../proposal.md | 25 + .../specs/meeting-transcription/spec.md | 78 +++ .../azure-speech-offline-resilience/tasks.md | 11 + .../resilient-transcript-workflow/design.md | 32 + .../resilient-transcript-workflow/proposal.md | 25 + .../specs/meeting-session/spec.md | 102 +++ .../resilient-transcript-workflow/tasks.md | 9 + 35 files changed, 2456 insertions(+), 198 deletions(-) create mode 100644 MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs create mode 100644 MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs create mode 100644 MeetingAssistant/Recording/OfflineTranscriptionBacklogHostedService.cs create mode 100644 MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs create mode 100644 MeetingAssistant/Recording/RecordingSpeechRecognitionPipelineOptions.cs create mode 100644 openspec/changes/add-tray-exit/.openspec.yaml create mode 100644 openspec/changes/add-tray-exit/design.md create mode 100644 openspec/changes/add-tray-exit/proposal.md create mode 100644 openspec/changes/add-tray-exit/specs/meeting-recording/spec.md create mode 100644 openspec/changes/add-tray-exit/tasks.md create mode 100644 openspec/changes/azure-speech-offline-resilience/design.md create mode 100644 openspec/changes/azure-speech-offline-resilience/proposal.md create mode 100644 openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md create mode 100644 openspec/changes/azure-speech-offline-resilience/tasks.md create mode 100644 openspec/changes/resilient-transcript-workflow/design.md create mode 100644 openspec/changes/resilient-transcript-workflow/proposal.md create mode 100644 openspec/changes/resilient-transcript-workflow/specs/meeting-session/spec.md create mode 100644 openspec/changes/resilient-transcript-workflow/tasks.md diff --git a/MeetingAssistant.Tests/MeetingWorkflowEngineTests.cs b/MeetingAssistant.Tests/MeetingWorkflowEngineTests.cs index d51c9f3..df64ad1 100644 --- a/MeetingAssistant.Tests/MeetingWorkflowEngineTests.cs +++ b/MeetingAssistant.Tests/MeetingWorkflowEngineTests.cs @@ -1,5 +1,6 @@ using MeetingAssistant.MeetingNotes; using MeetingAssistant.Workflow; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -203,6 +204,71 @@ public sealed class MeetingWorkflowEngineTests Assert.Equal("[00:00:04] Guest-1: Azure returned [redacted] here.", line); } + [Fact] + public async Task TranscriptLineRulePreservesUtf8CharactersWhenRenderingRazor() + { + var fixture = await WorkflowFixture.CreateAsync(MeetingWorkflowTestRules.MaskedProfanityRedactionYaml); + + var line = await fixture.Engine.TransformTranscriptLineAsync( + MeetingWorkflowEvent.TranscriptLine( + fixture.Artifacts, + "[00:00:57] Guest-1: Das heißt, Schimpfwörter wie ***** müssen als nächstes richtig bleiben.", + "Guest-1"), + fixture.Options, + CancellationToken.None); + + Assert.Equal( + "[00:00:57] Guest-1: Das heißt, Schimpfwörter wie [redacted] müssen als nächstes richtig bleiben.", + line); + } + + [Fact] + public async Task TriggeredWorkflowRuleLogsStartAndCompletion() + { + var logger = new ListLogger(); + var fixture = await WorkflowFixture.CreateAsync( + """ + rules: + - name: logged-rule + on: + - created: {} + steps: + - uses: add_project + value: Diagnostics + """, + logger: logger); + + await RunCreatedAsync(fixture); + + Assert.Contains(logger.Messages, message => + message.Contains("Triggered meeting workflow rule logged-rule for event Created", StringComparison.Ordinal)); + Assert.Contains(logger.Messages, message => + message.Contains("Completed meeting workflow rule logged-rule for event Created", StringComparison.Ordinal)); + } + + [Fact] + public async Task WorkflowRuleFailureLogsRuleNameAndEventType() + { + var logger = new ListLogger(); + var fixture = await WorkflowFixture.CreateAsync( + """ + rules: + - name: broken-rule + on: + - created: {} + steps: + - uses: set_property + property: unsupported + value: Diagnostics + """, + logger: logger); + + await Assert.ThrowsAsync(() => RunCreatedAsync(fixture)); + + Assert.Contains(logger.Messages, message => + message.Contains("Meeting workflow rule broken-rule failed for event Created", StringComparison.Ordinal)); + } + [Fact] public async Task RuleCanRemoveAttendeeWhenNestedConditionMatches() { @@ -775,7 +841,8 @@ public sealed class MeetingWorkflowEngineTests string title = "Planning Sync", IReadOnlyList? attendees = null, IReadOnlyList? projects = null, - string? rulesPath = null) + string? rulesPath = null, + ILogger? logger = null) { var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(root); @@ -837,7 +904,7 @@ public sealed class MeetingWorkflowEngineTests NullLogger.Instance), noteStore, artifactStore, - NullLogger.Instance); + logger ?? NullLogger.Instance); return new WorkflowFixture(options, noteStore, artifacts, engine); } @@ -851,4 +918,39 @@ public sealed class MeetingWorkflowEngineTests return File.ReadAllTextAsync(Artifacts.AssistantContextPath); } } + + private sealed class ListLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull + { + return NullScope.Instance; + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static NullScope Instance { get; } = new(); + + public void Dispose() + { + } + } + } } diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs index 3242b09..d9718bf 100644 --- a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -172,7 +172,7 @@ public sealed class RecordingCoordinatorTests var started = await coordinator.StartAsync(CancellationToken.None); await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); - await WaitUntilAsync(() => FileContainsText(started.TranscriptPath!, "[redacted]")); + await WaitUntilAsync(() => FileContainsText(started.TranscriptPath!, "Azure returned")); await coordinator.StopAsync(CancellationToken.None); var content = await File.ReadAllTextAsync(started.TranscriptPath!); @@ -180,6 +180,224 @@ public sealed class RecordingCoordinatorTests Assert.DoesNotContain("*****", content); } + [Fact] + public async Task TranscriptLineWorkflowFailureKeepsWritingLiveTranscriptSegments() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new OrderedChunkProvider()), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + meetingWorkflowEngine: new FailingFirstTranscriptLineWorkflowEngine()); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([2, 0], 16000, 1), CancellationToken.None); + + await transcriptStore.WaitForTextAsync("first has enough words for identification."); + await transcriptStore.WaitForTextAsync("second has enough words for identification."); + await coordinator.StopAsync(CancellationToken.None); + + Assert.Contains(transcriptStore.Segments, segment => + segment.Text.Contains("first has enough words for identification.", StringComparison.Ordinal)); + Assert.Contains(transcriptStore.Segments, segment => + segment.Text.Contains("second has enough words for identification.", StringComparison.Ordinal)); + } + + [Fact] + public async Task TranscriptLineWorkflowRewriteRunsAfterOriginalLineIsWritten() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory( + new FixedSegmentStreamingTranscriptionProvider( + new TranscriptionSegment( + TimeSpan.FromSeconds(4), + TimeSpan.FromSeconds(5), + "Guest-1", + "Azure returned ***** here."))), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + meetingWorkflowEngine: new ObservingTranscriptRewriteWorkflowEngine(transcriptStore)); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + + await transcriptStore.WaitForTextAsync("Azure returned [redacted] here."); + await coordinator.StopAsync(CancellationToken.None); + + Assert.DoesNotContain(transcriptStore.Segments, segment => + segment.Text.Contains("*****", StringComparison.Ordinal)); + Assert.Contains(transcriptStore.Segments, segment => + segment.Text.Contains("Azure returned [redacted] here.", StringComparison.Ordinal)); + } + + [Fact] + public async Task PendingTranscriptLineWorkflowRewriteDoesNotRemoveFollowingLine() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var workflowEngine = new DelayedFirstTranscriptLineWorkflowEngine(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new OrderedChunkProvider()), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + meetingWorkflowEngine: workflowEngine); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await transcriptStore.WaitForTextAsync("first has enough words for identification."); + await workflowEngine.WaitUntilFirstLineWorkflowStartedAsync(); + + await audioSource.WriteAsync(new AudioChunk([2, 0], 16000, 1), CancellationToken.None); + await transcriptStore.WaitForTextAsync("second has enough words for identification."); + + workflowEngine.CompleteFirstLineWorkflow(); + await transcriptStore.WaitForTextAsync("rewritten first has enough words for identification."); + await coordinator.StopAsync(CancellationToken.None); + + var segments = transcriptStore.Segments; + Assert.Equal( + ["rewritten first has enough words for identification.", "second has enough words for identification."], + segments.Select(segment => segment.Text).ToArray()); + } + + [Fact] + public async Task TranscriptReconnectMarkerIsReplacedByNextAzureTranscriptLine() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new StaticOnAudioCompletionProvider( + [ + new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + "", + TranscriptionSegmentKind.Marker, + MarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.FromSeconds(1), + TimeSpan.FromSeconds(2), + "Guest-1", + "speech after reconnect", + ReplacesMarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.FromSeconds(3), + TimeSpan.FromSeconds(4), + "Guest-2", + "following speech") + ])), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await coordinator.StopAsync(CancellationToken.None); + + var segments = transcriptStore.Segments; + Assert.Equal( + ["speech after reconnect", "following speech"], + segments.Select(segment => segment.Text).ToArray()); + Assert.DoesNotContain(segments, segment => + segment.Text.Contains("Reconnecting", StringComparison.Ordinal)); + } + + [Fact] + public async Task TranscriptReconnectMarkerUpdatesReuseSameLineUntilAzureTranscriptResumes() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new StaticOnAudioCompletionProvider( + [ + new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + "", + TranscriptionSegmentKind.Marker, + MarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + "", + TranscriptionSegmentKind.Marker, + MarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + "", + TranscriptionSegmentKind.Marker, + MarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.FromSeconds(1), + TimeSpan.FromSeconds(2), + "Guest-1", + "speech after reconnect", + ReplacesMarkerId: "azure-reconnect"), + new TranscriptionSegment( + TimeSpan.FromSeconds(3), + TimeSpan.FromSeconds(4), + "Guest-2", + "following speech") + ])), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await coordinator.StopAsync(CancellationToken.None); + + var segments = transcriptStore.Segments; + Assert.Equal( + ["speech after reconnect", "following speech"], + segments.Select(segment => segment.Text).ToArray()); + Assert.DoesNotContain(segments, segment => + segment.Text.Contains("Reconnecting", StringComparison.Ordinal) + || segment.Text.Contains("disconnected", StringComparison.OrdinalIgnoreCase)); + } + [Fact] public async Task StartCreatesMeetingNoteLinkedToTranscriptAndOpensIt() { @@ -822,6 +1040,124 @@ public sealed class RecordingCoordinatorTests await store.WaitForTextAsync("final:2"); } + [Fact] + public async Task StopQueuesAzureBacklogAndAllowsNextMeetingWhenTranscriptionCannotDrain() + { + var audioSource = new ControlledAudioSource(); + var provider = new BlockingBeforeTranscriptionProvider(); + var transcriptStore = new InMemoryTranscriptStore(); + var audioArchive = new InMemoryRecordedAudioStore(); + var backlog = new InMemoryOfflineTranscriptionBacklog(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions + { + TranscriptionProvider = "azure-speech", + StopProcessingTimeout = TimeSpan.FromMilliseconds(20) + } + }), + NullLogger.Instance, + offlineTranscriptionBacklog: backlog); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await provider.WaitUntilWaitingForBackendAsync(); + + var stopped = await coordinator.StopAsync(CancellationToken.None); + var restarted = await coordinator.StartAsync(CancellationToken.None); + + Assert.False(stopped.IsRecording); + Assert.True(restarted.IsRecording); + var item = Assert.Single(backlog.Items); + Assert.Equal("memory-recording.wav", item.AudioPath); + Assert.Equal(transcriptStore.CreatedSessions[0].TranscriptPath, item.TranscriptPath); + Assert.True(audioArchive.Completed); + Assert.False(audioArchive.Deleted); + } + + [Fact] + public async Task OfflineBacklogReplaysQueuedRecordingAndCompletesMeetingArtifacts() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var audioPath = Path.Combine(root, "queued.wav"); + Directory.CreateDirectory(root); + using (var writer = new NAudio.Wave.WaveFileWriter(audioPath, new NAudio.Wave.WaveFormat(16000, 16, 1))) + { + writer.Write([1, 0, 2, 0], 0, 4); + } + + var transcriptStore = new InMemoryTranscriptStore(Path.Combine(root, "transcript.md")); + var noteStore = new InMemoryMeetingNoteStore(Path.Combine(root, "meeting.md")); + var artifactStore = new InMemoryMeetingArtifactStore(); + var summaryPipeline = new CapturingMeetingSummaryPipeline(); + var workflowEngine = new CapturingMeetingWorkflowEngine(); + var backlog = new InMemoryOfflineTranscriptionBacklog(); + var pipelineFactory = new ProfileSwitchSpeechRecognitionPipelineFactory(); + var startedAt = DateTimeOffset.Now.AddMinutes(-5); + var stoppedAt = DateTimeOffset.Now.AddMinutes(-1); + var note = new MeetingNote( + Path.Combine(root, "meeting.md"), + new MeetingNoteFrontmatter + { + Title = "Queued meeting", + StartTime = startedAt, + Attendees = ["Ada", "Grace"], + Transcript = Path.Combine(root, "transcript.md"), + AssistantContext = Path.Combine(root, "context.md"), + Summary = Path.Combine(root, "summary.md") + }, + ""); + noteStore.UpdateSavedNote(note); + await backlog.EnqueueAsync(new OfflineTranscriptionBacklogItem( + "queued", + audioPath, + Path.Combine(root, "transcript.md"), + Path.Combine(root, "meeting.md"), + Path.Combine(root, "context.md"), + Path.Combine(root, "summary.md"), + startedAt, + stoppedAt, + "english"), CancellationToken.None); + var launchProfiles = CreateLaunchProfiles(root, root); + var processor = new OfflineTranscriptionBacklogProcessor( + backlog, + pipelineFactory, + transcriptStore, + noteStore, + artifactStore, + summaryPipeline, + workflowEngine, + CreateRecordingDictationWordProvider(new FixedDictationWordStore(["codex"])), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + launchProfiles); + + var processed = await processor.ProcessPendingAsync(CancellationToken.None); + + Assert.Equal(1, processed); + Assert.Empty(backlog.Items); + Assert.Equal(["english"], pipelineFactory.ProfileNames); + var segment = Assert.Single(transcriptStore.ReplacedSegments); + Assert.Contains("english chunk:4", segment.Text, StringComparison.Ordinal); + Assert.Equal(stoppedAt, noteStore.SavedNote?.Frontmatter.EndTime); + Assert.True(summaryPipeline.WasRun); + Assert.Equal( + new[] { AssistantContextState.Summarizing, AssistantContextState.Finished }, + artifactStore.States); + Assert.Contains(workflowEngine.Events, workflowEvent => + workflowEvent.Type == MeetingWorkflowEventType.TranscriptLine && + workflowEvent.TranscriptLineText?.Contains("english chunk:4", StringComparison.Ordinal) == true); + } + [Fact] public async Task AbortStopsRecordingDeletesArtifactsAndSkipsSummary() { @@ -2031,6 +2367,46 @@ public sealed class RecordingCoordinatorTests Assert.DoesNotContain("transcript:", content); } + [Fact] + public async Task VaultTranscriptStoreRewritesOneLineWithoutRemovingFollowingLine() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var vaultFolder = Path.Combine(root, "Transcripts"); + var store = new VaultTranscriptStore( + Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions { TranscriptsFolder = vaultFolder } + }), + NullLogger.Instance); + var session = await store.CreateSessionAsync(CancellationToken.None); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Notes", "meeting.md"), + TranscriptPath: session.TranscriptPath, + AssistantContextPath: Path.Combine(root, "Context", "context.md"), + SummaryPath: Path.Combine(root, "Summaries", "summary.md")); + await store.UpdateMetadataAsync( + session, + artifacts, + MeetingNoteTemplate.Create("Transcript Rewrite", transcriptPath: session.TranscriptPath), + CancellationToken.None); + var original = "[00:00:01] Guest-1: original workflow text"; + var rewritten = "[00:00:01] Guest-1: rewritten workflow text"; + var following = "[00:00:02] Guest-2: following transcript text"; + + var originalReference = await store.AppendLineAsync(session, original, CancellationToken.None); + await store.AppendLineAsync(session, following, CancellationToken.None); + await store.ReplaceLineAsync(session, originalReference, rewritten, CancellationToken.None); + + var content = await File.ReadAllTextAsync(session.TranscriptPath); + Assert.Contains(rewritten, content); + Assert.Contains(following, content); + Assert.DoesNotContain(original, content); + Assert.True( + content.IndexOf(rewritten, StringComparison.Ordinal) < + content.IndexOf(following, StringComparison.Ordinal)); + Assert.Contains("meeting:", content); + } + [Fact] public async Task VaultTranscriptStoreNamesGeneratedTranscriptWithMinutePrecision() { @@ -2129,6 +2505,40 @@ public sealed class RecordingCoordinatorTests Assert.True(File.Exists(unrelatedFile)); } + [Fact] + public async Task TemporaryRecordedAudioStoreKeepsQueuedOfflineRecordingsOnStartup() + { + var recordingFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(recordingFolder); + var queuedRecording = Path.Combine(recordingFolder, "queued.wav"); + var staleRecording = Path.Combine(recordingFolder, "stale.wav"); + await File.WriteAllTextAsync(queuedRecording, "queued"); + await File.WriteAllTextAsync(staleRecording, "stale"); + var backlog = new InMemoryOfflineTranscriptionBacklog(); + await backlog.EnqueueAsync(new OfflineTranscriptionBacklogItem( + Id: "queued", + AudioPath: queuedRecording, + TranscriptPath: "transcript.md", + MeetingNotePath: "meeting.md", + AssistantContextPath: "context.md", + SummaryPath: "summary.md", + StartedAt: DateTimeOffset.Parse("2026-06-15T10:00:00+02:00"), + InferredEndTime: null, + LaunchProfileName: "default"), CancellationToken.None); + var store = new TemporaryRecordedAudioStore( + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions { TemporaryRecordingsFolder = recordingFolder } + }), + NullLogger.Instance, + backlog); + + await store.DeleteStaleRecordingsAsync(CancellationToken.None); + + Assert.True(File.Exists(queuedRecording)); + Assert.False(File.Exists(staleRecording)); + } + private static MeetingAssistantOptions CreateOptionsWithoutFinalizer() { return new MeetingAssistantOptions @@ -2190,9 +2600,13 @@ public sealed class RecordingCoordinatorTests this.transcriptPath = transcriptPath; } + public List CreatedSessions { get; } = []; + public Task CreateSessionAsync(CancellationToken cancellationToken) { - return Task.FromResult(new TranscriptSession(transcriptPath)); + var session = new TranscriptSession(transcriptPath); + CreatedSessions.Add(session); + return Task.FromResult(session); } public Task CreateSessionAsync( @@ -2203,13 +2617,34 @@ public sealed class RecordingCoordinatorTests return CreateSessionAsync(cancellationToken); } - public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) + public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) { + int index; lock (gate) { + index = segments.Count; segments.Add(ParseTranscriptLine(line)); } + return Task.FromResult(new TranscriptLineReference(session.TranscriptPath, index, line)); + } + + public Task ReplaceLineAsync( + TranscriptSession session, + TranscriptLineReference lineReference, + string replacementLine, + CancellationToken cancellationToken) + { + var replacement = ParseTranscriptLine(replacementLine); + lock (gate) + { + if (lineReference.BodyLineIndex >= 0 && + lineReference.BodyLineIndex < segments.Count) + { + segments[lineReference.BodyLineIndex] = replacement; + } + } + return Task.CompletedTask; } @@ -2280,11 +2715,12 @@ public sealed class RecordingCoordinatorTests return CreateSessionAsync(cancellationToken); } - public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) + public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) { + var index = AppendHistory.Count(entry => entry.Session.TranscriptPath == session.TranscriptPath); AppendHistory.Add(new TranscriptWrite(session, ParseTranscriptLine(line))); Interlocked.Increment(ref appendCount); - return Task.CompletedTask; + return Task.FromResult(new TranscriptLineReference(session.TranscriptPath, index, line)); } public Task WaitForAppendCountAsync(int expectedCount) @@ -2294,6 +2730,16 @@ public sealed class RecordingCoordinatorTests $"Expected {expectedCount} transcript append(s)."); } + public Task ReplaceLineAsync( + TranscriptSession session, + TranscriptLineReference lineReference, + string replacementLine, + CancellationToken cancellationToken) + { + ReplacementHistory.Add(new TranscriptReplacement(session, [ParseTranscriptLine(replacementLine)])); + return Task.CompletedTask; + } + public Task ReplaceLinesAsync( TranscriptSession session, IReadOnlyList replacementLines, @@ -2420,6 +2866,105 @@ public sealed class RecordingCoordinatorTests } } + private sealed class FailingFirstTranscriptLineWorkflowEngine : IMeetingWorkflowEngine + { + public Task RunAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public Task TransformTranscriptLineAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + if (workflowEvent.TranscriptLineText?.Contains("first", StringComparison.Ordinal) == true) + { + throw new InvalidOperationException("Workflow rule failed."); + } + + return Task.FromResult(workflowEvent.TranscriptLineText ?? ""); + } + } + + private sealed class ObservingTranscriptRewriteWorkflowEngine : IMeetingWorkflowEngine + { + private readonly InMemoryTranscriptStore transcriptStore; + + public ObservingTranscriptRewriteWorkflowEngine(InMemoryTranscriptStore transcriptStore) + { + this.transcriptStore = transcriptStore; + } + + public Task RunAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public Task TransformTranscriptLineAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + if (!transcriptStore.Segments.Any(segment => + segment.Text.Contains("Azure returned ***** here.", StringComparison.Ordinal))) + { + throw new InvalidOperationException("Original transcript line was not written before workflow processing."); + } + + return Task.FromResult( + (workflowEvent.TranscriptLineText ?? "").Replace("*****", "[redacted]", StringComparison.Ordinal)); + } + } + + private sealed class DelayedFirstTranscriptLineWorkflowEngine : IMeetingWorkflowEngine + { + private readonly TaskCompletionSource firstLineWorkflowStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource completeFirstLineWorkflow = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task RunAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task TransformTranscriptLineAsync( + MeetingWorkflowEvent workflowEvent, + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + var line = workflowEvent.TranscriptLineText ?? ""; + if (!line.Contains("first has enough words for identification.", StringComparison.Ordinal)) + { + return line; + } + + firstLineWorkflowStarted.TrySetResult(); + await completeFirstLineWorkflow.Task.WaitAsync(cancellationToken); + return line.Replace("first", "rewritten first", StringComparison.Ordinal); + } + + public Task WaitUntilFirstLineWorkflowStartedAsync() + { + return firstLineWorkflowStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public void CompleteFirstLineWorkflow() + { + completeFirstLineWorkflow.TrySetResult(); + } + } + private sealed class CapturingMeetingScreenshotService : IMeetingScreenshotService { public MeetingSessionArtifacts? Artifacts { get; private set; } @@ -2935,7 +3480,16 @@ public sealed class RecordingCoordinatorTests MeetingArtifactFileNames.Create(startedAt, MeetingArtifactFileNames.Transcript)))); } - public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) + public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) + { + return Task.FromResult(new TranscriptLineReference(session.TranscriptPath, 0, line)); + } + + public Task ReplaceLineAsync( + TranscriptSession session, + TranscriptLineReference lineReference, + string replacementLine, + CancellationToken cancellationToken) { return Task.CompletedTask; } @@ -3114,6 +3668,28 @@ public sealed class RecordingCoordinatorTests } } + private sealed class InMemoryOfflineTranscriptionBacklog : IOfflineTranscriptionBacklog + { + public List Items { get; } = []; + + public Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) + { + Items.Add(item); + return Task.CompletedTask; + } + + public Task> ListAsync(CancellationToken cancellationToken) + { + return Task.FromResult>(Items.ToList()); + } + + public Task CompleteAsync(string id, CancellationToken cancellationToken) + { + Items.RemoveAll(item => string.Equals(item.Id, id, StringComparison.Ordinal)); + return Task.CompletedTask; + } + } + private sealed class TestSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory { private readonly IStreamingTranscriptionProvider provider; @@ -3946,6 +4522,31 @@ public sealed class RecordingCoordinatorTests } } + private sealed class StaticOnAudioCompletionProvider : IStreamingTranscriptionProvider + { + private readonly IReadOnlyList segments; + + public StaticOnAudioCompletionProvider(IReadOnlyList segments) + { + this.segments = segments; + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + } + + foreach (var segment in segments) + { + yield return segment; + } + } + } + private sealed class BlockingBeforeTranscriptionProvider : IStreamingTranscriptionProvider { private readonly TaskCompletionSource waitingForBackend = diff --git a/MeetingAssistant.Tests/TaskbarIconTests.cs b/MeetingAssistant.Tests/TaskbarIconTests.cs index 68a67d0..8b55c99 100644 --- a/MeetingAssistant.Tests/TaskbarIconTests.cs +++ b/MeetingAssistant.Tests/TaskbarIconTests.cs @@ -83,6 +83,34 @@ public sealed class TaskbarIconTests Assert.Equal(RecordingProcessState.Recording, menu.State); } + [Theory] + [InlineData(RecordingProcessState.Idle, false)] + [InlineData(RecordingProcessState.Recording, true)] + [InlineData(RecordingProcessState.Summarizing, false)] + public void MenuAlwaysOffersExit(RecordingProcessState state, bool isRecording) + { + var menu = MeetingTaskbarMenuBuilder.Build( + Status(isRecording: isRecording, state: state, profile: "default"), + [Profile("default"), Profile("english")]); + + Assert.Contains(menu.Items, item => + item.Action == MeetingTaskbarAction.Exit && + item.Text == "Exit"); + } + + [Theory] + [InlineData(RecordingProcessState.Idle, false)] + [InlineData(RecordingProcessState.Recording, true)] + [InlineData(RecordingProcessState.Summarizing, true)] + public void ExitRequiresConfirmationWhileRecordingOrProcessing( + RecordingProcessState state, + bool requiresConfirmation) + { + Assert.Equal( + requiresConfirmation, + MeetingTaskbarExitPolicy.RequiresConfirmation(Status(state: state))); + } + [Fact] [SupportedOSPlatform("windows")] public void TaskbarIconGlyphsAreVisuallyCentered() diff --git a/MeetingAssistant/MeetingAssistantOptions.cs b/MeetingAssistant/MeetingAssistantOptions.cs index 5019736..4e909be 100644 --- a/MeetingAssistant/MeetingAssistantOptions.cs +++ b/MeetingAssistant/MeetingAssistantOptions.cs @@ -223,6 +223,10 @@ public sealed class AzureSpeechOptions public TimeSpan RecognitionStopTimeout { get; set; } = TimeSpan.FromMinutes(3); + public int ReconnectAttemptsBeforeDisconnectedMarker { get; set; } = 5; + + public TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5); + public bool DiarizeIntermediateResults { get; set; } = true; public string? PostProcessingOption { get; set; } diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index d66e8f2..c5ceec0 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -37,6 +37,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -116,9 +117,11 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); diff --git a/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs b/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs new file mode 100644 index 0000000..2d3e64f --- /dev/null +++ b/MeetingAssistant/Recording/FileOfflineTranscriptionBacklog.cs @@ -0,0 +1,134 @@ +using System.Text.Json; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Recording; + +public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBacklog +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public FileOfflineTranscriptionBacklog( + IOptions options, + ILogger logger) + { + this.options = options.Value; + this.logger = logger; + } + + public async Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) + { + var folder = GetBacklogFolder(options); + Directory.CreateDirectory(folder); + var path = GetItemPath(folder, item.Id); + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(item, JsonOptions), cancellationToken); + logger.LogInformation( + "Queued offline transcription backlog item {BacklogItemId} for recording {RecordingPath}", + item.Id, + item.AudioPath); + } + + public async Task> ListAsync(CancellationToken cancellationToken) + { + var folder = GetBacklogFolder(options); + if (!Directory.Exists(folder)) + { + return []; + } + + var items = new List(); + foreach (var path in Directory.EnumerateFiles(folder, "*.json", SearchOption.TopDirectoryOnly)) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var content = await File.ReadAllTextAsync(path, cancellationToken); + if (JsonSerializer.Deserialize(content, JsonOptions) is { } item) + { + items.Add(item); + } + } + catch (JsonException exception) + { + logger.LogWarning(exception, "Could not read offline transcription backlog item {BacklogItemPath}", path); + } + catch (IOException exception) + { + logger.LogWarning(exception, "Could not read offline transcription backlog item {BacklogItemPath}", path); + } + } + + return items; + } + + public async Task CompleteAsync(string id, CancellationToken cancellationToken) + { + var folder = GetBacklogFolder(options); + var path = GetItemPath(folder, id); + if (!File.Exists(path)) + { + return; + } + + OfflineTranscriptionBacklogItem? item = null; + try + { + item = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(path, cancellationToken), + JsonOptions); + } + catch (JsonException exception) + { + logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path); + } + + File.Delete(path); + if (item is not null && File.Exists(item.AudioPath)) + { + DeleteCompletedAudio(item.AudioPath); + } + } + + private static string GetBacklogFolder(MeetingAssistantOptions options) + { + return Path.Combine(VaultPath.Resolve(options.Recording.TemporaryRecordingsFolder), "offline-transcription-backlog"); + } + + private static string GetItemPath(string folder, string id) + { + return Path.Combine(folder, $"{id}.json"); + } + + private void DeleteCompletedAudio(string audioPath) + { + try + { + File.Delete(audioPath); + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + catch (IOException exception) + { + logger.LogWarning( + exception, + "Could not delete completed offline transcription recording file {RecordingPath}", + audioPath); + } + catch (UnauthorizedAccessException exception) + { + logger.LogWarning( + exception, + "Could not delete completed offline transcription recording file {RecordingPath}", + audioPath); + } + } +} diff --git a/MeetingAssistant/Recording/ITranscriptStore.cs b/MeetingAssistant/Recording/ITranscriptStore.cs index 98dfe20..229bc32 100644 --- a/MeetingAssistant/Recording/ITranscriptStore.cs +++ b/MeetingAssistant/Recording/ITranscriptStore.cs @@ -14,13 +14,30 @@ public interface ITranscriptStore Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken) { - return AppendLineAsync( + return AppendLineAndDiscardReferenceAsync( session, TranscriptLineFormatter.Format(segment).Line, cancellationToken); } - Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken); + private async Task AppendLineAndDiscardReferenceAsync( + TranscriptSession session, + string line, + CancellationToken cancellationToken) + { + _ = await AppendLineAsync(session, line, cancellationToken); + } + + Task AppendLineAsync( + TranscriptSession session, + string line, + CancellationToken cancellationToken); + + Task ReplaceLineAsync( + TranscriptSession session, + TranscriptLineReference lineReference, + string replacementLine, + CancellationToken cancellationToken); Task ReplaceAsync( TranscriptSession session, @@ -47,11 +64,18 @@ public interface ITranscriptStore public sealed record TranscriptSession(string TranscriptPath); +public sealed record TranscriptLineReference(string TranscriptPath, int BodyLineIndex, string OriginalLine); + public static class TranscriptLineFormatter { public static FormattedTranscriptLine Format(TranscriptionSegment segment) { var speaker = NormalizeSpeaker(segment.Speaker); + if (segment.Kind == TranscriptionSegmentKind.Marker) + { + return new FormattedTranscriptLine(segment.Text, speaker); + } + return new FormattedTranscriptLine( $"[{segment.Start:hh\\:mm\\:ss}] {speaker}: {segment.Text}", speaker); diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs index 8a436d2..261e2dc 100644 --- a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -31,6 +31,7 @@ public sealed class MeetingRecordingCoordinator private readonly IMeetingRunArtifactCleaner artifactCleaner; private readonly IMeetingInactivityPromptService inactivityPromptService; private readonly IMeetingInactivityClock inactivityClock; + private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog; private readonly MeetingAssistantOptions options; private readonly ILogger logger; private readonly SemaphoreSlim gate = new(1, 1); @@ -59,7 +60,8 @@ public sealed class MeetingRecordingCoordinator IMeetingScreenshotService? screenshotService = null, IMeetingRunArtifactCleaner? artifactCleaner = null, IMeetingInactivityPromptService? inactivityPromptService = null, - IMeetingInactivityClock? inactivityClock = null) + IMeetingInactivityClock? inactivityClock = null, + IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null) { this.audioSource = audioSource; this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory; @@ -79,6 +81,7 @@ public sealed class MeetingRecordingCoordinator this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner(); this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService(); this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock(); + this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance; this.options = options.Value; this.logger = logger; } @@ -301,7 +304,17 @@ public sealed class MeetingRecordingCoordinator } catch (TimeoutException) { - logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation"); + if (IsAzureSpeechRun(run)) + { + logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item"); + run.MarkQueuedForOfflineTranscription(); + await offlineTranscriptionBacklog.EnqueueAsync(CreateOfflineBacklogItem(run), cancellationToken); + } + else + { + logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation"); + } + run.CancelTranscription(); await run.Task.WaitAsync(cancellationToken); } @@ -424,6 +437,7 @@ public sealed class MeetingRecordingCoordinator await run.Pipeline.CompleteAsync(cancellationToken); await run.WaitForLiveTranscriptTasksAsync(cancellationToken); + await run.WaitForTranscriptWorkflowTasksAsync(cancellationToken); run.ResetSpeakerIdentification(); run.MarkProfileSwitched(); @@ -471,6 +485,7 @@ public sealed class MeetingRecordingCoordinator await captureTask; await run.Pipeline.CompleteAsync(run.TranscriptionCancellation); await run.WaitForLiveTranscriptTasksAsync(run.TranscriptionCancellation); + await run.WaitForTranscriptWorkflowTasksAsync(run.TranscriptionCancellation); run.CancelLiveIdentification(); await liveIdentificationTask; await run.RecordedAudio.DisposeAsync(); @@ -528,7 +543,15 @@ public sealed class MeetingRecordingCoordinator finally { run.CancelLiveIdentification(); - await run.RecordedAudio.DeleteAsync(CancellationToken.None); + if (run.IsQueuedForOfflineTranscription) + { + await run.RecordedAudio.DisposeAsync(); + } + else + { + await run.RecordedAudio.DeleteAsync(CancellationToken.None); + } + await run.DisposePipelinesAsync(); await CompleteRunAsync(run); } @@ -582,6 +605,17 @@ public sealed class MeetingRecordingCoordinator { await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken)) { + if (segment.Kind == TranscriptionSegmentKind.Marker) + { + await AppendTranscriptSegmentAsync(run, segment, cancellationToken); + continue; + } + + if (segment.ReplacesMarkerId is not null) + { + run.ResetSpeakerIdentification(); + } + run.RecordTranscriptActivity(segment, inactivityClock.Now); var sample = run.TryAddSpeakerSample(segment); if (sample is not null) @@ -607,8 +641,69 @@ public sealed class MeetingRecordingCoordinator TranscriptionSegment segment, CancellationToken cancellationToken) { - var line = await TransformTranscriptLineAsync(run, segment, cancellationToken); - await transcriptStore.AppendLineAsync(run.Session, line, cancellationToken); + var formatted = TranscriptLineFormatter.Format(segment); + var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId && + run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference) + ? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken) + : segment.Kind == TranscriptionSegmentKind.Marker && + segment.MarkerId is { } existingMarkerId && + run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference) + ? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken) + : await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken); + if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId) + { + run.SetTranscriptMarker(markerId, lineReference); + return; + } + + run.AddTranscriptWorkflowTask( + () => ApplyTranscriptLineWorkflowAsync(run, lineReference, formatted, run.TranscriptionCancellation)); + } + + private async Task ReplaceTranscriptMarkerAsync( + RecordingRun run, + TranscriptLineReference markerReference, + FormattedTranscriptLine formatted, + CancellationToken cancellationToken) + { + await transcriptStore.ReplaceLineAsync(run.Session, markerReference, formatted.Line, cancellationToken); + return markerReference with { OriginalLine = formatted.Line }; + } + + private async Task ApplyTranscriptLineWorkflowAsync( + RecordingRun run, + TranscriptLineReference lineReference, + FormattedTranscriptLine formatted, + CancellationToken cancellationToken) + { + var transformed = await TryTransformTranscriptLineAsync( + run, + formatted.Line, + formatted.Speaker, + cancellationToken); + if (!string.Equals(formatted.Line, transformed, StringComparison.Ordinal)) + { + try + { + await transcriptStore.ReplaceLineAsync( + run.Session, + lineReference, + transformed, + cancellationToken); + run.RecordTranscriptLineRewrite(formatted.Line, transformed); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError( + exception, + "Transcript line rewrite failed for speaker {Speaker}; keeping original transcript line", + formatted.Speaker); + } + } } private async Task ReplaceTranscriptSegmentsAsync( @@ -616,28 +711,41 @@ public sealed class MeetingRecordingCoordinator IReadOnlyList segments, CancellationToken cancellationToken) { - var lines = new List(segments.Count); - foreach (var segment in segments) - { - lines.Add(await TransformTranscriptLineAsync(run, segment, cancellationToken)); - } + var lines = segments + .Select(segment => run.ApplyTranscriptLineRewrite(TranscriptLineFormatter.Format(segment).Line)) + .ToList(); await transcriptStore.ReplaceLinesAsync(run.Session, lines, cancellationToken); } - private async Task TransformTranscriptLineAsync( + private async Task TryTransformTranscriptLineAsync( RecordingRun run, - TranscriptionSegment segment, + string formattedLine, + string speaker, CancellationToken cancellationToken) { - var formatted = TranscriptLineFormatter.Format(segment); - return await meetingWorkflowEngine.TransformTranscriptLineAsync( - MeetingWorkflowEvent.TranscriptLine( - run.Artifacts, - formatted.Line, - formatted.Speaker), - run.Options, - cancellationToken); + try + { + return await meetingWorkflowEngine.TransformTranscriptLineAsync( + MeetingWorkflowEvent.TranscriptLine( + run.Artifacts, + formattedLine, + speaker), + run.Options, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError( + exception, + "Transcript line workflow failed for speaker {Speaker}; keeping original transcript line", + speaker); + return formattedLine; + } } private async Task RunInactivitySafeguardAsync(RecordingRun run) @@ -1459,17 +1567,14 @@ public sealed class MeetingRecordingCoordinator string? meetingNotePath, CancellationToken cancellationToken) { - var dictationWords = await dictationWordProvider.ReadOptionalWordsAsync(runOptions, cancellationToken); - if (string.IsNullOrWhiteSpace(meetingNotePath)) - { - return new SpeechRecognitionPipelineOptions(DictationWords: dictationWords); - } - - var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken); - var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee)); - return attendeeCount > 1 - ? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords) - : new SpeechRecognitionPipelineOptions(DictationWords: dictationWords); + var meetingNote = string.IsNullOrWhiteSpace(meetingNotePath) + ? null + : await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken); + return await RecordingSpeechRecognitionPipelineOptions.BuildAsync( + dictationWordProvider, + runOptions, + meetingNote, + cancellationToken); } private LaunchProfile ResolveProfile(string? launchProfileName) @@ -1515,6 +1620,25 @@ public sealed class MeetingRecordingCoordinator return minimumDuration > TimeSpan.Zero && stoppedAt - run.StartedAt < minimumDuration; } + private static bool IsAzureSpeechRun(RecordingRun run) + { + return run.Options.Recording.TranscriptionProvider.Equals("azure-speech", StringComparison.OrdinalIgnoreCase); + } + + private static OfflineTranscriptionBacklogItem CreateOfflineBacklogItem(RecordingRun run) + { + return new OfflineTranscriptionBacklogItem( + Id: Guid.NewGuid().ToString("N"), + AudioPath: run.RecordedAudio.AudioPath, + TranscriptPath: run.Session.TranscriptPath, + MeetingNotePath: run.MeetingNotePath, + AssistantContextPath: run.Artifacts.AssistantContextPath, + SummaryPath: run.Artifacts.SummaryPath, + StartedAt: run.StartedAt, + InferredEndTime: run.InferredEndTime, + LaunchProfileName: run.LaunchProfileName); + } + private static IReadOnlyList GetInactivityPromptThresholds( RecordingInactivitySafeguardOptions options) { @@ -1566,10 +1690,14 @@ public sealed class MeetingRecordingCoordinator private readonly object stateGate = new(); private readonly object liveSegmentsGate = new(); private readonly object liveTranscriptTasksGate = new(); + private readonly object transcriptWorkflowTasksGate = new(); private readonly object liveIdentificationGate = new(); private readonly object transcriptActivityGate = new(); private readonly List liveSegments = []; private readonly List liveTranscriptTasks = []; + private Task transcriptWorkflowTail = Task.CompletedTask; + private readonly ConcurrentDictionary transcriptMarkers = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary transcriptLineRewrites = new(StringComparer.Ordinal); private readonly List pipelines = []; private readonly ConcurrentDictionary speakerMappings = new(StringComparer.OrdinalIgnoreCase); private readonly SpeakerAudioSampleCollector speakerSampleCollector; @@ -1656,6 +1784,8 @@ public sealed class MeetingRecordingCoordinator public bool IsAborted { get; private set; } + public bool IsQueuedForOfflineTranscription { get; private set; } + public bool HasSwitchedProfile { get; private set; } public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata; @@ -1689,6 +1819,11 @@ public sealed class MeetingRecordingCoordinator TranscriptionCancellationSource.Cancel(); } + public void MarkQueuedForOfflineTranscription() + { + IsQueuedForOfflineTranscription = true; + } + public void CancelLiveIdentification() { LiveIdentificationCancellationSource.Cancel(); @@ -1745,6 +1880,41 @@ public sealed class MeetingRecordingCoordinator return Task.WhenAll(tasks).WaitAsync(cancellationToken); } + public void AddTranscriptWorkflowTask(Func taskFactory) + { + lock (transcriptWorkflowTasksGate) + { + transcriptWorkflowTail = RunAfterAsync(transcriptWorkflowTail, taskFactory); + } + } + + public Task WaitForTranscriptWorkflowTasksAsync(CancellationToken cancellationToken) + { + Task task; + lock (transcriptWorkflowTasksGate) + { + task = transcriptWorkflowTail; + } + + return task.WaitAsync(cancellationToken); + } + + private static async Task RunAfterAsync(Task previous, Func next) + { + await previous.ConfigureAwait(false); + await next().ConfigureAwait(false); + } + + public void SetTranscriptMarker(string markerId, TranscriptLineReference lineReference) + { + transcriptMarkers[markerId] = lineReference; + } + + public bool TryTakeTranscriptMarker(string markerId, out TranscriptLineReference lineReference) + { + return transcriptMarkers.TryRemove(markerId, out lineReference!); + } + public async Task DisposePipelinesAsync() { foreach (var pipeline in pipelines.Distinct()) @@ -1817,6 +1987,18 @@ public sealed class MeetingRecordingCoordinator } } + public void RecordTranscriptLineRewrite(string originalLine, string replacementLine) + { + transcriptLineRewrites[originalLine] = replacementLine; + } + + public string ApplyTranscriptLineRewrite(string line) + { + return transcriptLineRewrites.TryGetValue(line, out var replacement) + ? replacement + : line; + } + public void RecordTranscriptActivity(TranscriptionSegment segment, DateTimeOffset arrivedAt) { if (string.IsNullOrWhiteSpace(segment.Text)) diff --git a/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs b/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs new file mode 100644 index 0000000..6c1534e --- /dev/null +++ b/MeetingAssistant/Recording/OfflineTranscriptionBacklog.cs @@ -0,0 +1,45 @@ +namespace MeetingAssistant.Recording; + +public interface IOfflineTranscriptionBacklog +{ + Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken); + + Task> ListAsync(CancellationToken cancellationToken); + + Task CompleteAsync(string id, CancellationToken cancellationToken); +} + +public sealed record OfflineTranscriptionBacklogItem( + string Id, + string AudioPath, + string TranscriptPath, + string MeetingNotePath, + string AssistantContextPath, + string SummaryPath, + DateTimeOffset StartedAt, + DateTimeOffset? InferredEndTime, + string LaunchProfileName); + +public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBacklog +{ + public static readonly NoopOfflineTranscriptionBacklog Instance = new(); + + private NoopOfflineTranscriptionBacklog() + { + } + + public Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public Task> ListAsync(CancellationToken cancellationToken) + { + return Task.FromResult>([]); + } + + public Task CompleteAsync(string id, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } +} diff --git a/MeetingAssistant/Recording/OfflineTranscriptionBacklogHostedService.cs b/MeetingAssistant/Recording/OfflineTranscriptionBacklogHostedService.cs new file mode 100644 index 0000000..14730ba --- /dev/null +++ b/MeetingAssistant/Recording/OfflineTranscriptionBacklogHostedService.cs @@ -0,0 +1,53 @@ +namespace MeetingAssistant.Recording; + +public sealed class OfflineTranscriptionBacklogHostedService : BackgroundService +{ + private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + private readonly OfflineTranscriptionBacklogProcessor processor; + private readonly ILogger logger; + + public OfflineTranscriptionBacklogHostedService( + OfflineTranscriptionBacklogProcessor processor, + ILogger logger) + { + this.processor = processor; + this.logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + var processed = await processor.ProcessPendingAsync(stoppingToken); + if (processed > 0) + { + logger.LogInformation( + "Processed {ProcessedCount} offline transcription backlog item(s)", + processed); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Offline transcription backlog processing failed; it will be retried later"); + } + + try + { + await Task.Delay(RetryDelay, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + } + } +} diff --git a/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs b/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs new file mode 100644 index 0000000..f8e8120 --- /dev/null +++ b/MeetingAssistant/Recording/OfflineTranscriptionBacklogProcessor.cs @@ -0,0 +1,243 @@ +using MeetingAssistant.LaunchProfiles; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using MeetingAssistant.Transcription; +using MeetingAssistant.Workflow; +using Microsoft.Extensions.Options; +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public sealed class OfflineTranscriptionBacklogProcessor +{ + private const int MaxChunkDurationMilliseconds = 1000; + + private readonly IOfflineTranscriptionBacklog backlog; + private readonly ISpeechRecognitionPipelineFactory pipelineFactory; + private readonly ITranscriptStore transcriptStore; + private readonly IMeetingNoteStore meetingNoteStore; + private readonly IMeetingArtifactStore meetingArtifactStore; + private readonly IMeetingSummaryPipeline summaryPipeline; + private readonly IMeetingWorkflowEngine workflowEngine; + private readonly IRecordingDictationWordProvider dictationWordProvider; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + private readonly ILaunchProfileOptionsProvider? launchProfiles; + + public OfflineTranscriptionBacklogProcessor( + IOfflineTranscriptionBacklog backlog, + ISpeechRecognitionPipelineFactory pipelineFactory, + ITranscriptStore transcriptStore, + IMeetingNoteStore meetingNoteStore, + IMeetingArtifactStore meetingArtifactStore, + IMeetingSummaryPipeline summaryPipeline, + IMeetingWorkflowEngine workflowEngine, + IRecordingDictationWordProvider dictationWordProvider, + IOptions options, + ILogger logger, + ILaunchProfileOptionsProvider? launchProfiles = null) + { + this.backlog = backlog; + this.pipelineFactory = pipelineFactory; + this.transcriptStore = transcriptStore; + this.meetingNoteStore = meetingNoteStore; + this.meetingArtifactStore = meetingArtifactStore; + this.summaryPipeline = summaryPipeline; + this.workflowEngine = workflowEngine; + this.dictationWordProvider = dictationWordProvider; + this.options = options.Value; + this.logger = logger; + this.launchProfiles = launchProfiles; + } + + public async Task ProcessPendingAsync(CancellationToken cancellationToken) + { + var processed = 0; + foreach (var item in await backlog.ListAsync(cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (await TryProcessAsync(item, cancellationToken)) + { + processed++; + } + } + + return processed; + } + + private async Task TryProcessAsync( + OfflineTranscriptionBacklogItem item, + CancellationToken cancellationToken) + { + try + { + await ProcessAsync(item, cancellationToken); + await backlog.CompleteAsync(item.Id, cancellationToken); + logger.LogInformation( + "Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}", + item.Id, + item.AudioPath); + return true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not process offline transcription backlog item {BacklogItemId}; it will be retried later", + item.Id); + return false; + } + } + + private async Task ProcessAsync( + OfflineTranscriptionBacklogItem item, + CancellationToken cancellationToken) + { + var runOptions = ResolveOptions(item.LaunchProfileName); + var artifacts = new MeetingSessionArtifacts( + item.MeetingNotePath, + item.TranscriptPath, + item.AssistantContextPath, + item.SummaryPath); + var session = new TranscriptSession(item.TranscriptPath); + var meetingNote = await meetingNoteStore.ReadAsync(item.MeetingNotePath, cancellationToken); + var pipelineOptions = await RecordingSpeechRecognitionPipelineOptions.BuildAsync( + dictationWordProvider, + runOptions, + meetingNote, + cancellationToken); + + await using var pipeline = pipelineFactory.Create(item.LaunchProfileName); + await pipeline.InitializeAsync(pipelineOptions, cancellationToken); + await pipeline.WaitUntilReadyAsync(cancellationToken); + await WriteRecordingToPipelineAsync(item.AudioPath, pipeline, cancellationToken); + var finishedSegments = await pipeline.ReadFinishedTranscriptAsync( + item.AudioPath, + pipelineOptions, + cancellationToken); + + var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken); + await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken); + + meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now; + var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken); + await transcriptStore.UpdateMetadataAsync(session, artifacts, completedMeetingNote, cancellationToken); + await meetingArtifactStore.UpdateAssistantContextMeetingAsync(artifacts, completedMeetingNote, cancellationToken); + + await TransitionMeetingAsync( + artifacts, + runOptions, + AssistantContextState.Transcribing, + AssistantContextState.Summarizing, + cancellationToken); + var summaryResult = await summaryPipeline.RunAsync(artifacts, runOptions, cancellationToken); + await TransitionMeetingAsync( + artifacts, + runOptions, + AssistantContextState.Summarizing, + summaryResult.Succeeded ? AssistantContextState.Finished : AssistantContextState.Error, + cancellationToken); + } + + private MeetingAssistantOptions ResolveOptions(string? launchProfileName) + { + if (launchProfiles is not null) + { + return launchProfiles.GetRequiredProfile(launchProfileName).Options; + } + + return options; + } + + private async Task> TransformTranscriptLinesAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions runOptions, + IReadOnlyList segments, + CancellationToken cancellationToken) + { + var lines = new List(segments.Count); + foreach (var segment in segments) + { + var formatted = TranscriptLineFormatter.Format(segment); + lines.Add(await TransformTranscriptLineAsync(artifacts, runOptions, formatted, cancellationToken)); + } + + return lines; + } + + private async Task TransformTranscriptLineAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions runOptions, + FormattedTranscriptLine formatted, + CancellationToken cancellationToken) + { + try + { + return await workflowEngine.TransformTranscriptLineAsync( + MeetingWorkflowEvent.TranscriptLine(artifacts, formatted.Line, formatted.Speaker), + runOptions, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError( + exception, + "Offline transcript line workflow failed for speaker {Speaker}; keeping original transcript line", + formatted.Speaker); + return formatted.Line; + } + } + + private async Task TransitionMeetingAsync( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions runOptions, + AssistantContextState from, + AssistantContextState to, + CancellationToken cancellationToken) + { + await meetingArtifactStore.UpdateAssistantContextStateAsync(artifacts, to, cancellationToken); + await workflowEngine.RunAsync( + MeetingWorkflowEvent.StateTransition(artifacts, from, to), + runOptions, + cancellationToken); + } + + private static async Task WriteRecordingToPipelineAsync( + string audioPath, + ISpeechRecognitionPipeline pipeline, + CancellationToken cancellationToken) + { + using var reader = new WaveFileReader(audioPath); + var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond; + var chunkSize = Math.Max( + reader.WaveFormat.BlockAlign, + bytesPerSecond * MaxChunkDurationMilliseconds / 1000); + chunkSize -= chunkSize % reader.WaveFormat.BlockAlign; + + var buffer = new byte[chunkSize]; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken); + if (read == 0) + { + break; + } + + await pipeline.WriteAsync( + new AudioChunk( + buffer[..read], + reader.WaveFormat.SampleRate, + reader.WaveFormat.Channels), + cancellationToken); + } + } +} diff --git a/MeetingAssistant/Recording/RecordingSpeechRecognitionPipelineOptions.cs b/MeetingAssistant/Recording/RecordingSpeechRecognitionPipelineOptions.cs new file mode 100644 index 0000000..a3a7c46 --- /dev/null +++ b/MeetingAssistant/Recording/RecordingSpeechRecognitionPipelineOptions.cs @@ -0,0 +1,20 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Recording; + +public static class RecordingSpeechRecognitionPipelineOptions +{ + public static async Task BuildAsync( + IRecordingDictationWordProvider dictationWordProvider, + MeetingAssistantOptions runOptions, + MeetingNote? meetingNote, + CancellationToken cancellationToken) + { + var dictationWords = await dictationWordProvider.ReadOptionalWordsAsync(runOptions, cancellationToken); + var attendeeCount = meetingNote?.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee)) ?? 0; + return attendeeCount > 1 + ? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords) + : new SpeechRecognitionPipelineOptions(DictationWords: dictationWords); + } +} diff --git a/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs b/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs index 8a2626d..55ad1b0 100644 --- a/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs +++ b/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs @@ -7,13 +7,16 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore { private readonly MeetingAssistantOptions options; private readonly ILogger logger; + private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog; public TemporaryRecordedAudioStore( IOptions options, - ILogger logger) + ILogger logger, + IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null) { this.options = options.Value; this.logger = logger; + this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance; } public Task CreateSessionAsync(CancellationToken cancellationToken) @@ -35,21 +38,28 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore return Task.FromResult(new TemporaryRecordedAudioSink(path, format, logger)); } - public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken) + public async Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken) { var folder = GetTemporaryRecordingsFolder(options); if (!Directory.Exists(folder)) { - return Task.CompletedTask; + return; } + var queuedAudioPaths = (await offlineTranscriptionBacklog.ListAsync(cancellationToken)) + .Select(item => item.AudioPath) + .ToHashSet(StringComparer.OrdinalIgnoreCase); foreach (var path in Directory.EnumerateFiles(folder, "*.wav", SearchOption.TopDirectoryOnly)) { cancellationToken.ThrowIfCancellationRequested(); + if (queuedAudioPaths.Contains(path)) + { + logger.LogInformation("Keeping queued offline transcription recording file {RecordingPath}", path); + continue; + } + DeleteFile(path); } - - return Task.CompletedTask; } private static string GetTemporaryRecordingsFolder(MeetingAssistantOptions options) diff --git a/MeetingAssistant/Recording/VaultTranscriptStore.cs b/MeetingAssistant/Recording/VaultTranscriptStore.cs index 2968999..c4e3816 100644 --- a/MeetingAssistant/Recording/VaultTranscriptStore.cs +++ b/MeetingAssistant/Recording/VaultTranscriptStore.cs @@ -1,6 +1,7 @@ using MeetingAssistant.Transcription; using MeetingAssistant.MeetingNotes; using Microsoft.Extensions.Options; +using System.Collections.Concurrent; namespace MeetingAssistant.Recording; @@ -8,6 +9,7 @@ public sealed class VaultTranscriptStore : ITranscriptStore { private readonly MeetingAssistantOptions options; private readonly ILogger logger; + private readonly ConcurrentDictionary fileGates = new(StringComparer.OrdinalIgnoreCase); public VaultTranscriptStore(IOptions options, ILogger logger) { @@ -36,9 +38,50 @@ public sealed class VaultTranscriptStore : ITranscriptStore return new TranscriptSession(path); } - public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken) + public async Task AppendLineAsync( + TranscriptSession session, + string line, + CancellationToken cancellationToken) { - return AppendToBodyAsync(session.TranscriptPath, line + Environment.NewLine, cancellationToken); + return await WithFileGateAsync( + session.TranscriptPath, + () => AppendToBodyAsync(session.TranscriptPath, line, cancellationToken), + cancellationToken); + } + + public async Task ReplaceLineAsync( + TranscriptSession session, + TranscriptLineReference lineReference, + string replacementLine, + CancellationToken cancellationToken) + { + await WithFileGateAsync( + session.TranscriptPath, + async () => + { + if (!File.Exists(session.TranscriptPath)) + { + return; + } + + var content = await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken); + var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content); + var lines = SplitLines(body); + if (lineReference.BodyLineIndex < 0 || + lineReference.BodyLineIndex >= lines.Count || + !string.Equals(lines[lineReference.BodyLineIndex], lineReference.OriginalLine, StringComparison.Ordinal)) + { + logger.LogWarning( + "Could not replace transcript line in {TranscriptPath} at body line {BodyLineIndex} because the original line no longer matched", + session.TranscriptPath, + lineReference.BodyLineIndex); + return; + } + + lines[lineReference.BodyLineIndex] = replacementLine; + await WriteBodyAsync(session.TranscriptPath, frontmatter, string.Join(Environment.NewLine, lines), cancellationToken); + }, + cancellationToken); } public Task ReplaceLinesAsync( @@ -46,18 +89,21 @@ public sealed class VaultTranscriptStore : ITranscriptStore IReadOnlyList replacementLines, CancellationToken cancellationToken) { - var existing = File.Exists(session.TranscriptPath) - ? File.ReadAllText(session.TranscriptPath) - : ""; - var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing); - var body = "# Meeting Transcript" - + Environment.NewLine - + Environment.NewLine - + string.Concat(replacementLines.Select(line => line + Environment.NewLine)); - var content = string.IsNullOrWhiteSpace(frontmatter) - ? body - : "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body; - return File.WriteAllTextAsync(session.TranscriptPath, content, cancellationToken); + return WithFileGateAsync( + session.TranscriptPath, + async () => + { + var existing = File.Exists(session.TranscriptPath) + ? File.ReadAllText(session.TranscriptPath) + : ""; + var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing); + var body = "# Meeting Transcript" + + Environment.NewLine + + Environment.NewLine + + string.Concat(replacementLines.Select(line => line + Environment.NewLine)); + await WriteBodyAsync(session.TranscriptPath, frontmatter, body, cancellationToken); + }, + cancellationToken); } public async Task UpdateMetadataAsync( @@ -66,50 +112,108 @@ public sealed class VaultTranscriptStore : ITranscriptStore MeetingNote meetingNote, CancellationToken cancellationToken) { - var body = File.Exists(session.TranscriptPath) - ? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body - : "# Meeting Transcript" + Environment.NewLine + Environment.NewLine; - var frontmatter = MeetingArtifactFrontmatterRenderer.Create( - artifacts, - meetingNote, - MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"), - session.TranscriptPath); - - await File.WriteAllTextAsync( + await WithFileGateAsync( session.TranscriptPath, - MeetingArtifactFrontmatterRenderer.Render(frontmatter, body), + async () => + { + var body = File.Exists(session.TranscriptPath) + ? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body + : "# Meeting Transcript" + Environment.NewLine + Environment.NewLine; + var frontmatter = MeetingArtifactFrontmatterRenderer.Create( + artifacts, + meetingNote, + MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"), + session.TranscriptPath); + + await File.WriteAllTextAsync( + session.TranscriptPath, + MeetingArtifactFrontmatterRenderer.Render(frontmatter, body), + cancellationToken); + }, cancellationToken); } - private static async Task AppendToBodyAsync( + private static async Task AppendToBodyAsync( string path, - string text, + string line, CancellationToken cancellationToken) { if (!File.Exists(path)) { - await File.AppendAllTextAsync(path, text, cancellationToken); - return; + await File.AppendAllTextAsync(path, line + Environment.NewLine, cancellationToken); + return new TranscriptLineReference(path, 0, line); } var content = await File.ReadAllTextAsync(path, cancellationToken); var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content); + var bodyLineIndex = GetAppendBodyLineIndex(body); if (string.IsNullOrWhiteSpace(frontmatter)) { - await File.AppendAllTextAsync(path, text, cancellationToken); - return; + await File.AppendAllTextAsync(path, line + Environment.NewLine, cancellationToken); + return new TranscriptLineReference(path, bodyLineIndex, line); } - var updated = "---" - + Environment.NewLine - + frontmatter - + Environment.NewLine - + "---" - + Environment.NewLine - + Environment.NewLine - + body - + text; - await File.WriteAllTextAsync(path, updated, cancellationToken); + await WriteBodyAsync(path, frontmatter, body + line + Environment.NewLine, cancellationToken); + return new TranscriptLineReference(path, bodyLineIndex, line); } + private async Task WithFileGateAsync( + string path, + Func action, + CancellationToken cancellationToken) + { + await WithFileGateAsync( + path, + async () => + { + await action(); + return true; + }, + cancellationToken); + } + + private async Task WithFileGateAsync( + string path, + Func> action, + CancellationToken cancellationToken) + { + var gate = fileGates.GetOrAdd(path, static _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken); + try + { + return await action(); + } + finally + { + gate.Release(); + } + } + + private static async Task WriteBodyAsync( + string path, + string frontmatter, + string body, + CancellationToken cancellationToken) + { + var content = string.IsNullOrWhiteSpace(frontmatter) + ? body + : "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body; + await File.WriteAllTextAsync(path, content, cancellationToken); + } + + private static int GetAppendBodyLineIndex(string body) + { + var lines = SplitLines(body); + return lines.Count > 0 && lines[^1].Length == 0 + ? lines.Count - 1 + : lines.Count; + } + + private static List SplitLines(string text) + { + return text + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Split('\n') + .ToList(); + } } diff --git a/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs b/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs index 1c60f37..e56b37e 100644 --- a/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs +++ b/MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs @@ -9,7 +9,8 @@ public enum MeetingTaskbarAction StartRecording, StopRecording, AbortRecording, - SwitchProfile + SwitchProfile, + Exit } public sealed record MeetingTaskbarMenu( @@ -61,6 +62,10 @@ public static class MeetingTaskbarMenuBuilder } } + items.Add(new MeetingTaskbarMenuItem( + "Exit", + MeetingTaskbarAction.Exit)); + return new MeetingTaskbarMenu( status.State, BuildTooltip(status), @@ -96,3 +101,11 @@ public static class MeetingTaskbarMenuBuilder : $"{text}\t{hotkey.Trim()}"; } } + +public static class MeetingTaskbarExitPolicy +{ + public static bool RequiresConfirmation(RecordingStatus status) + { + return status.State != RecordingProcessState.Idle; + } +} diff --git a/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs b/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs index 45a241d..93d1ab2 100644 --- a/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs +++ b/MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs @@ -1,5 +1,6 @@ #if WINDOWS using System.Drawing; +using System.Windows; using H.NotifyIcon.Core; using MeetingAssistant.LaunchProfiles; using MeetingAssistant.Recording; @@ -14,6 +15,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable private readonly MeetingRecordingCoordinator coordinator; private readonly ILaunchProfileOptionsProvider launchProfiles; private readonly IWorkflowRulesEditorWindowService rulesEditorWindow; + private readonly IHostApplicationLifetime applicationLifetime; private readonly ILogger logger; private readonly object sync = new(); private CancellationTokenSource? refreshCancellation; @@ -28,11 +30,13 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable MeetingRecordingCoordinator coordinator, ILaunchProfileOptionsProvider launchProfiles, IWorkflowRulesEditorWindowService rulesEditorWindow, + IHostApplicationLifetime applicationLifetime, ILogger logger) { this.coordinator = coordinator; this.launchProfiles = launchProfiles; this.rulesEditorWindow = rulesEditorWindow; + this.applicationLifetime = applicationLifetime; this.logger = logger; } @@ -182,7 +186,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable var popupMenu = new PopupMenu(); for (var index = 0; index < menu.Items.Count; index++) { - if (index == 1) + if (index == 1 || + (menu.Items[index].Action == MeetingTaskbarAction.Exit && + menu.Items[index - 1].Action != MeetingTaskbarAction.EditRules)) { popupMenu.Items.Add(new PopupMenuSeparator()); } @@ -222,6 +228,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable case MeetingTaskbarAction.SwitchProfile: await coordinator.ToggleAsync(item.ProfileName, CancellationToken.None); break; + case MeetingTaskbarAction.Exit: + ExitApplication(); + break; } RefreshVisualState(); @@ -247,5 +256,32 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable menu.Items.Select(item => $"{item.Action}:{item.ProfileName}:{item.Text}")); } + private void ExitApplication() + { + var status = coordinator.CurrentStatus; + if (MeetingTaskbarExitPolicy.RequiresConfirmation(status) && !ConfirmExitDuringProcessing(status.State)) + { + logger.LogInformation("Taskbar exit canceled while meeting state was {State}", status.State); + return; + } + + logger.LogInformation("Taskbar exit requested while meeting state was {State}", status.State); + applicationLifetime.StopApplication(); + } + + private static bool ConfirmExitDuringProcessing(RecordingProcessState state) + { + var activity = state == RecordingProcessState.Recording + ? "A meeting is still recording and transcribing." + : "A stopped meeting is still transcribing, recognizing speakers, or summarizing."; + var result = MessageBox.Show( + $"{activity}\n\nExit Meeting Assistant anyway?", + "Exit Meeting Assistant", + MessageBoxButton.YesNo, + MessageBoxImage.Warning, + MessageBoxResult.No); + + return result == MessageBoxResult.Yes; + } } #endif diff --git a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs index c8b8694..689e38b 100644 --- a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs @@ -33,69 +33,78 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc yield break; } - var firstChunk = enumerator.Current; - using var pushStream = AudioInputStream.CreatePushStream( - AudioStreamFormat.GetWaveFormatPCM( - (uint)firstChunk.SampleRate, - 16, - (byte)firstChunk.Channels)); - using var audioConfig = AudioConfig.FromStreamInput(pushStream); - var speechConfig = CreateSpeechConfig(); - using var recognizer = CreateTranscriber(speechConfig, audioConfig); - AddPhraseList(recognizer, pipelineOptions.DictationWords); - var segments = Channel.CreateUnbounded(); + var nextChunk = enumerator.Current; + var reconnectAttempt = 0; + var markerId = ""; + var disconnectedMarkerWritten = false; - recognizer.Transcribed += (_, args) => + while (true) { - if (args.Result.Reason != ResultReason.RecognizedSpeech - || string.IsNullOrWhiteSpace(args.Result.Text)) - { - return; - } - - var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); - var segment = new TranscriptionSegment( - start, - start + args.Result.Duration, - NormalizeSpeakerId(args.Result.SpeakerId), - args.Result.Text.Trim()); - logger.LogInformation( - "Azure Speech emitted segment at {Start} from {Speaker}: {Text}", - segment.Start, - segment.Speaker, - segment.Text); - segments.Writer.TryWrite(segment); - }; - recognizer.Canceled += (_, args) => - { - if (args.Reason == CancellationReason.Error) - { - segments.Writer.TryComplete(new InvalidOperationException( - $"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}")); - return; - } - - segments.Writer.TryComplete(); - }; - - await recognizer.StartTranscribingAsync(); - var pumpTask = Task.Run( - () => PumpAudioAsync( - firstChunk, + var session = StartSession( + nextChunk, enumerator, - pushStream, - recognizer, - segments.Writer, - options.AzureSpeech.RecognitionStopTimeout, - cancellationToken), - CancellationToken.None); + pipelineOptions, + markerId, + cancellationToken); - await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) - { - yield return segment; + await foreach (var segment in session.Segments.ReadAllAsync(cancellationToken)) + { + if (segment.ReplacesMarkerId is not null) + { + reconnectAttempt = 0; + disconnectedMarkerWritten = false; + } + + yield return segment; + } + + var result = await session.Completion.WaitAsync(cancellationToken); + if (result.IsCompleted) + { + yield break; + } + + reconnectAttempt++; + markerId = string.IsNullOrWhiteSpace(markerId) + ? $"azure-reconnect-{Guid.NewGuid():N}" + : markerId; + nextChunk = result.NextChunk ?? nextChunk; + if (reconnectAttempt <= Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker)) + { + logger.LogWarning( + "Azure Speech reconnecting after connection interruption, attempt {Attempt}/{MaxAttempts}: {ErrorDetails}", + reconnectAttempt, + Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker), + result.ErrorDetails); + yield return new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + $"", + TranscriptionSegmentKind.Marker, + MarkerId: markerId); + } + else if (!disconnectedMarkerWritten) + { + disconnectedMarkerWritten = true; + logger.LogWarning( + "Azure Speech remains disconnected after {AttemptCount} reconnect attempt(s); recording continues and transcription will retry: {ErrorDetails}", + reconnectAttempt - 1, + result.ErrorDetails); + yield return new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "System", + "", + TranscriptionSegmentKind.Marker, + MarkerId: markerId); + } + + if (options.AzureSpeech.ReconnectDelay > TimeSpan.Zero) + { + await Task.Delay(options.AzureSpeech.ReconnectDelay, cancellationToken); + } } - - await pumpTask.WaitAsync(cancellationToken); } finally { @@ -110,6 +119,117 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc } } + private AzureSpeechSession StartSession( + AudioChunk firstChunk, + IAsyncEnumerator audio, + SpeechRecognitionPipelineOptions pipelineOptions, + string reconnectMarkerId, + CancellationToken cancellationToken) + { + var segments = Channel.CreateUnbounded(); + var completion = Task.Run( + () => RunSessionAsync( + firstChunk, + audio, + pipelineOptions, + reconnectMarkerId, + segments.Writer, + cancellationToken), + CancellationToken.None); + return new AzureSpeechSession(segments.Reader, completion); + } + + private async Task RunSessionAsync( + AudioChunk firstChunk, + IAsyncEnumerator audio, + SpeechRecognitionPipelineOptions pipelineOptions, + string reconnectMarkerId, + ChannelWriter segments, + CancellationToken cancellationToken) + { + try + { + using var pushStream = AudioInputStream.CreatePushStream( + AudioStreamFormat.GetWaveFormatPCM( + (uint)firstChunk.SampleRate, + 16, + (byte)firstChunk.Channels)); + using var audioConfig = AudioConfig.FromStreamInput(pushStream); + var speechConfig = CreateSpeechConfig(); + using var recognizer = CreateTranscriber(speechConfig, audioConfig); + AddPhraseList(recognizer, pipelineOptions.DictationWords); + var sessionStop = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var pendingReconnectMarkerId = reconnectMarkerId; + + recognizer.Transcribed += (_, args) => + { + if (args.Result.Reason != ResultReason.RecognizedSpeech + || string.IsNullOrWhiteSpace(args.Result.Text)) + { + return; + } + + var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); + var replacesMarkerId = string.IsNullOrWhiteSpace(pendingReconnectMarkerId) + ? null + : Interlocked.Exchange(ref pendingReconnectMarkerId, null); + var segment = new TranscriptionSegment( + start, + start + args.Result.Duration, + NormalizeSpeakerId(args.Result.SpeakerId), + args.Result.Text.Trim(), + ReplacesMarkerId: replacesMarkerId); + logger.LogInformation( + "Azure Speech emitted segment at {Start} from {Speaker}: {Text}", + segment.Start, + segment.Speaker, + segment.Text); + segments.TryWrite(segment); + }; + recognizer.Canceled += (_, args) => + { + if (args.Reason == CancellationReason.Error) + { + logger.LogWarning( + "Azure Speech recognition canceled with {ErrorCode}: {ErrorDetails}", + args.ErrorCode, + args.ErrorDetails); + sessionStop.TrySetResult(AzureSpeechSessionResult.Reconnect( + $"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}")); + return; + } + + sessionStop.TrySetResult(AzureSpeechSessionResult.Completed()); + }; + recognizer.SessionStopped += (_, _) => sessionStop.TrySetResult(AzureSpeechSessionResult.Completed()); + + await recognizer.StartTranscribingAsync(); + return await PumpAudioAsync( + firstChunk, + audio, + pushStream, + recognizer, + sessionStop.Task, + options.AzureSpeech.RecognitionStopTimeout, + cancellationToken); + } + catch (Exception exception) when (IsTransientConnectionException(exception)) + { + logger.LogWarning(exception, "Azure Speech session failed transiently; reconnecting."); + return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message); + } + catch (Exception exception) + { + segments.TryComplete(exception); + throw; + } + finally + { + segments.TryComplete(); + } + } + internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig) { var languages = GetAutoDetectLanguages(options.AzureSpeech); @@ -234,47 +354,118 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc : "Continuous"; } - private static async Task PumpAudioAsync( + private static async Task PumpAudioAsync( AudioChunk firstChunk, IAsyncEnumerator audio, PushAudioInputStream pushStream, ConversationTranscriber recognizer, - ChannelWriter segments, + Task sessionStop, TimeSpan recognitionStopTimeout, CancellationToken cancellationToken) { + WriteChunk(pushStream, firstChunk, firstChunk); + while (true) + { + var moveNextTask = audio.MoveNextAsync().AsTask(); + var completedTask = await Task.WhenAny(moveNextTask, sessionStop); + if (completedTask == sessionStop) + { + var stop = await sessionStop; + if (stop.IsCompleted) + { + return AzureSpeechSessionResult.Completed(); + } + + if (!await moveNextTask) + { + return AzureSpeechSessionResult.Completed(); + } + + return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails); + } + + if (!await moveNextTask) + { + break; + } + + if (sessionStop.IsCompleted) + { + var stop = await sessionStop; + if (!stop.IsCompleted) + { + return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + WriteChunk(pushStream, firstChunk, audio.Current); + } + + pushStream.Close(); try { - WriteChunk(pushStream, firstChunk, firstChunk); - while (await audio.MoveNextAsync()) + var stopTask = recognizer.StopTranscribingAsync(); + if (recognitionStopTimeout > TimeSpan.Zero) { - cancellationToken.ThrowIfCancellationRequested(); - WriteChunk(pushStream, firstChunk, audio.Current); + await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken); } - - pushStream.Close(); - try + else { - var stopTask = recognizer.StopTranscribingAsync(); - if (recognitionStopTimeout > TimeSpan.Zero) - { - await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken); - } - else - { - await stopTask.WaitAsync(cancellationToken); - } + await stopTask.WaitAsync(cancellationToken); } - catch (TimeoutException) - { - // The SDK can outlive short diagnostic streams while waiting for final service events. - } - - segments.TryComplete(); } - catch (Exception exception) + catch (TimeoutException) { - segments.TryComplete(exception); + // The SDK can outlive short diagnostic streams while waiting for final service events. + } + catch (Exception exception) when (IsTransientConnectionException(exception)) + { + return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message); + } + + return AzureSpeechSessionResult.Completed(); + } + + private static bool IsTransientConnectionException(Exception exception) + { + if (exception is OperationCanceledException) + { + return false; + } + + return exception is TimeoutException + or IOException + or HttpRequestException + or System.Net.Sockets.SocketException + || exception.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) + || exception.Message.Contains("websocket", StringComparison.OrdinalIgnoreCase) + || exception.Message.Contains("network", StringComparison.OrdinalIgnoreCase) + || exception.Message.Contains("transport", StringComparison.OrdinalIgnoreCase); + } + + private sealed record AzureSpeechSession( + ChannelReader Segments, + Task Completion); + + private sealed record AzureSpeechSessionResult( + bool IsCompleted, + AudioChunk? NextChunk, + string? ErrorDetails) + { + public static AzureSpeechSessionResult Completed() + { + return new AzureSpeechSessionResult(true, null, null); + } + + public static AzureSpeechSessionResult Reconnect(AudioChunk nextChunk, string? errorDetails) + { + return new AzureSpeechSessionResult(false, nextChunk, errorDetails); + } + + public static AzureSpeechSessionResult Reconnect(string? errorDetails) + { + return new AzureSpeechSessionResult(false, null, errorDetails); } } diff --git a/MeetingAssistant/Transcription/TranscriptionSegment.cs b/MeetingAssistant/Transcription/TranscriptionSegment.cs index 73ce93d..fe600e2 100644 --- a/MeetingAssistant/Transcription/TranscriptionSegment.cs +++ b/MeetingAssistant/Transcription/TranscriptionSegment.cs @@ -1,3 +1,16 @@ namespace MeetingAssistant.Transcription; -public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text); +public sealed record TranscriptionSegment( + TimeSpan Start, + TimeSpan End, + string Speaker, + string Text, + TranscriptionSegmentKind Kind = TranscriptionSegmentKind.Speech, + string? MarkerId = null, + string? ReplacesMarkerId = null); + +public enum TranscriptionSegmentKind +{ + Speech, + Marker +} diff --git a/MeetingAssistant/Workflow/MeetingWorkflowEngine.cs b/MeetingAssistant/Workflow/MeetingWorkflowEngine.cs index bc3574f..c61f72e 100644 --- a/MeetingAssistant/Workflow/MeetingWorkflowEngine.cs +++ b/MeetingAssistant/Workflow/MeetingWorkflowEngine.cs @@ -117,26 +117,53 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine foreach (var rule in rules) { - if (!MatchesTrigger(rule, workflowEvent) || - !EvaluateConditions(rule.If, meeting, context)) + try { - continue; - } + if (!MatchesTrigger(rule, workflowEvent) || + !EvaluateConditions(rule.If, meeting, context)) + { + continue; + } - logger.LogInformation( - "Applying meeting workflow rule {RuleName} for event {EventType}", - rule.Name, - workflowEvent.Type); - var model = CreateTemplateModel(meeting, context); - foreach (var step in rule.Steps) + logger.LogInformation( + "Triggered meeting workflow rule {RuleName} for event {EventType}", + rule.Name, + workflowEvent.Type); + var ruleNoteChanged = false; + var originalTranscriptLine = context.TranscriptLine; + var model = CreateTemplateModel(meeting, context); + foreach (var step in rule.Steps) + { + var stepChanged = await ApplyStepAsync( + step, + meeting, + context, + model, + cancellationToken); + ruleNoteChanged |= stepChanged; + noteChanged |= stepChanged; + model = CreateTemplateModel(meeting, context); + } + + logger.LogInformation( + "Completed meeting workflow rule {RuleName} for event {EventType}; noteChanged={NoteChanged}, transcriptLineChanged={TranscriptLineChanged}", + rule.Name, + workflowEvent.Type, + ruleNoteChanged, + !string.Equals(originalTranscriptLine, context.TranscriptLine, StringComparison.Ordinal)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - noteChanged |= await ApplyStepAsync( - step, - meeting, - context, - model, - cancellationToken); - model = CreateTemplateModel(meeting, context); + throw; + } + catch (Exception exception) + { + logger.LogError( + exception, + "Meeting workflow rule {RuleName} failed for event {EventType}", + rule.Name, + workflowEvent.Type); + throw; } } diff --git a/MeetingAssistant/Workflow/MeetingWorkflowTemplateRenderer.cs b/MeetingAssistant/Workflow/MeetingWorkflowTemplateRenderer.cs index 249d5b5..e5b2ab6 100644 --- a/MeetingAssistant/Workflow/MeetingWorkflowTemplateRenderer.cs +++ b/MeetingAssistant/Workflow/MeetingWorkflowTemplateRenderer.cs @@ -1,4 +1,5 @@ using System.Text.RegularExpressions; +using System.Net; using RazorLight; namespace MeetingAssistant.Workflow; @@ -20,10 +21,11 @@ internal sealed class MeetingWorkflowTemplateRenderer return template; } - return await razorEngine.CompileRenderStringAsync( + var rendered = await razorEngine.CompileRenderStringAsync( Guid.NewGuid().ToString("N"), EscapeEmailAddressAtSigns(template), model); + return WebUtility.HtmlDecode(rendered); } public static bool ContainsRazorTemplateSyntax(string value) diff --git a/docs/meeting-assistant-configuration.md b/docs/meeting-assistant-configuration.md index 153f999..07ae7c1 100644 --- a/docs/meeting-assistant-configuration.md +++ b/docs/meeting-assistant-configuration.md @@ -129,7 +129,7 @@ The default profile is always named `default`. Non-default profile hotkeys are r During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription. -`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. +`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV. `Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings. @@ -234,10 +234,12 @@ Pyannote diarization settings are shared by local Whisper finalization and speak ## Azure Speech -`azure-speech` streams Meeting Assistant's captured PCM chunks to Azure AI Speech using the Speech SDK `ConversationTranscriber`; it does not use an Azure-owned microphone capture or MAS/AEC input path. +`azure-speech` streams Meeting Assistant's captured PCM chunks to Azure AI Speech using the Speech SDK `ConversationTranscriber`; it does not use MSAL, an Azure-owned microphone capture, or a separate MAS/AEC input path. Configure `AzureSpeech:Endpoint`, `Region`, `Language`, and either `Key` or `KeyEnv`; this repository uses `AZURE_SPEECH_KEY` by default and does not store the key in appsettings. `DiarizeIntermediateResults` enables diarization for intermediate and final conversation transcription results. +If Azure Speech reports a transient connection failure, Meeting Assistant starts a new SDK session instead of completing the transcript with an error. The transcript note shows a single status line such as ``, updates that line across retry attempts, and replaces it with the next real transcript line after Azure emits speech again. After the configured immediate attempts are exhausted, the same line changes to an Azure disconnected marker while retries continue. Reconnect starts a fresh Azure conversation session, so live speaker mappings are reset before processing the first resumed speech line. + | Setting | Purpose | | --- | --- | | `Endpoint` | Optional Azure Speech endpoint. When blank, `Region` plus subscription key is used. | @@ -248,6 +250,8 @@ Configure `AzureSpeech:Endpoint`, `Region`, `Language`, and either `Key` or `Key | `Key` | Optional inline Azure Speech key. Prefer `KeyEnv`. | | `KeyEnv` | Environment variable name that contains the Azure Speech key. | | `RecognitionStopTimeout` | Timeout while stopping the ConversationTranscriber after closing the audio stream. | +| `ReconnectAttemptsBeforeDisconnectedMarker` | Immediate reconnect-marker attempt count before the transcript status changes to the longer disconnected marker. | +| `ReconnectDelay` | Delay between Azure Speech SDK session reconnect attempts. | | `DiarizeIntermediateResults` | Requests diarized intermediate results from Azure. | | `PostProcessingOption` | Accepted by config but currently skipped for the live `ConversationTranscriber` path; the app logs a warning if set. | | `PhraseListWeight` | Weight applied to dictation-word phrase-list hints. | diff --git a/docs/meeting-workflow-engine.md b/docs/meeting-workflow-engine.md index 1e0a91f..9c8a95e 100644 --- a/docs/meeting-workflow-engine.md +++ b/docs/meeting-workflow-engine.md @@ -68,7 +68,7 @@ The engine runs when `MeetingRecordingCoordinator` emits a meeting workflow even - `created`: after the meeting note and assistant context artifact are created. - `state_transition`: after the assistant context state moves forward. - `speaker_identified`: when live or final speaker identification reports a display name. -- `transcript_line`: before a formatted transcript line is appended or used in a transcript rewrite. +- `transcript_line`: after a formatted live transcript line is durably appended, before any changed line is rewritten in place, and before lines are used in full transcript rewrites. For every event, the engine: @@ -80,9 +80,9 @@ For every event, the engine: 6. Saves the meeting note once if any meeting-note step changed it. 7. Updates assistant-context frontmatter links/title from the saved meeting note after note changes. -For `transcript_line` events, the engine returns the possibly updated transcript line to the caller. Live transcript appends, live speaker relabel rewrites, final diarization rewrites, and final speaker-identity rewrites all pass their formatted lines through this event before writing markdown. +For `transcript_line` events, the engine returns the possibly updated transcript line to the caller. Live transcript appends first write the original formatted line and keep a reference to that exact body line, then run transcript-line rules out of band. Later transcript segments can continue to be appended while the workflow runs. If rules return a changed line, Meeting Assistant rewrites the referenced line in the transcript file. If a transcript-line rule fails during live recording, Meeting Assistant logs the failure and keeps the original line so later transcript segments can continue to be written. -Rules are best-effort automation. The settings/logs write tool rejects invalid YAML, unknown steps, unsupported set-property fields, trigger or condition entries without a supported key, and step value Razor templates that fail against the workflow template model before writing the rules file. Invalid NCalc expressions can still fail at workflow runtime and should be covered by tests before the engine is broadened. +Rules are best-effort automation for live transcript persistence. The settings/logs write tool rejects invalid YAML, unknown steps, unsupported set-property fields, trigger or condition entries without a supported key, and step value Razor templates that fail against the workflow template model before writing the rules file. Invalid NCalc expressions can still fail at workflow runtime. Workflow execution logs every triggered rule with its rule name and event type, logs completion with whether the note or transcript line changed, and logs rule failures with the rule name and event type. ## YAML Shape @@ -227,6 +227,10 @@ valid email address token. Literal email addresses such as `Support@contoso.com` unchanged; if the same value also contains a Razor expression, the email `@` is escaped before rendering so the address still appears normally in the rendered result. +Rendered values are plain text for markdown artifacts, not HTML. UTF-8 characters such as +`ß`, `ö`, and `ä` are preserved after Razor rendering instead of being persisted as HTML +entities. + ## Steps ### `add_attendee` @@ -251,7 +255,7 @@ steps: ### `set_property` -Sets a supported property. Supported meeting properties are `title` and `meeting.title`. During `transcript_line` events, `transcript.line` is also supported and replaces the single formatted line that will be written. +Sets a supported property. Supported meeting properties are `title` and `meeting.title`. During `transcript_line` events, `transcript.line` is also supported. For live transcription, changing `transcript.line` rewrites the referenced formatted line after the workflow task completes. ```yaml steps: diff --git a/openspec/changes/add-tray-exit/.openspec.yaml b/openspec/changes/add-tray-exit/.openspec.yaml new file mode 100644 index 0000000..8fe2055 --- /dev/null +++ b/openspec/changes/add-tray-exit/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-06-12 diff --git a/openspec/changes/add-tray-exit/design.md b/openspec/changes/add-tray-exit/design.md new file mode 100644 index 0000000..d96d3e7 --- /dev/null +++ b/openspec/changes/add-tray-exit/design.md @@ -0,0 +1,18 @@ +# Design + +## Tray Menu + +`MeetingTaskbarMenuBuilder` will add a stable `Exit` action after the existing recording controls. Keeping this in the menu builder lets tests assert the visible menu behavior without depending on the Windows tray implementation. + +## Exit Confirmation + +The Windows tray service will handle `Exit` by checking the current `RecordingStatus.State`: + +- `Idle` exits immediately. +- `Recording` or `Summarizing` prompts for confirmation before exiting. + +The existing `Summarizing` state represents stopped-run finalization, including transcript drain, speaker recognition, and summary generation. `Recording` represents active capture plus live transcription. If the user confirms, the tray service asks `IHostApplicationLifetime` to stop the application. + +## Windows UI Boundary + +The confirmation dialog is only needed in the Windows tray implementation. A small method local to `UnoTaskbarIconService` can use WPF `MessageBox` so tests can continue covering platform-independent menu construction through `MeetingTaskbarMenuBuilder`. diff --git a/openspec/changes/add-tray-exit/proposal.md b/openspec/changes/add-tray-exit/proposal.md new file mode 100644 index 0000000..a472700 --- /dev/null +++ b/openspec/changes/add-tray-exit/proposal.md @@ -0,0 +1,21 @@ +# Add Tray Exit + +## Summary + +Add an Exit action to the Windows taskbar icon menu so Meeting Assistant can be shut down from the tray. If Meeting Assistant is recording or still processing a stopped meeting's transcript or summary, Exit should ask for confirmation before stopping the application. + +## Motivation + +The tray icon is the normal local control surface for Meeting Assistant, but it currently cannot close the application. Because the app may be doing live transcription or post-recording summary work after capture stops, an accidental exit can interrupt useful meeting processing. + +## Scope + +- Add an Exit item to the tray context menu in all recording states. +- Stop the application when Exit is selected and Meeting Assistant is idle. +- Show a confirmation dialog before exiting while recording, transcribing, speaker recognition, or summarization is still in progress. + +## Out Of Scope + +- Changing shutdown semantics for the recording coordinator. +- Adding background resume of interrupted transcription or summary work. +- Restarting or otherwise managing the local service process. diff --git a/openspec/changes/add-tray-exit/specs/meeting-recording/spec.md b/openspec/changes/add-tray-exit/specs/meeting-recording/spec.md new file mode 100644 index 0000000..2bbf66b --- /dev/null +++ b/openspec/changes/add-tray-exit/specs/meeting-recording/spec.md @@ -0,0 +1,58 @@ +## MODIFIED Requirements + +### Requirement: Windows taskbar icon controls recording +Meeting Assistant SHALL show a Windows taskbar notification icon when running on Windows. + +The taskbar icon SHALL indicate whether the newest meeting process is idle, actively recording, or post-recording processing/summarizing. + +When a new meeting is actively recording while an older stopped meeting is still transcribing, recognizing speakers, or summarizing, the taskbar icon SHALL show the new active recording state. + +The taskbar icon right-click menu SHALL expose recording controls based on the current state and configured launch profiles. + +The taskbar icon right-click menu SHALL expose an Exit action in every recording state. + +When Meeting Assistant is idle or only processing older stopped meetings, the menu SHALL allow starting a meeting recording for each configured launch profile. + +When a meeting is actively recording, the menu SHALL allow stopping the recording and continuing transcription/summary generation. + +When a meeting is actively recording, the menu SHALL allow canceling the recording and discarding that run's artifacts. + +When a meeting is actively recording, the menu SHALL allow switching to each configured launch profile other than the current active profile. + +Selecting Exit while Meeting Assistant is idle SHALL stop the application without an additional confirmation prompt. + +Selecting Exit while Meeting Assistant is recording, transcribing, recognizing speakers, or summarizing SHALL show a confirmation dialog before stopping the application. + +#### Scenario: Idle tray menu can start configured profiles +- **GIVEN** launch profiles `default` and `english` are configured +- **AND** no meeting recording is active +- **WHEN** the taskbar menu is opened +- **THEN** it offers start recording actions for `default` and `english` + +#### Scenario: Recording tray menu exposes stop, cancel, and profile switches +- **GIVEN** launch profiles `default` and `english` are configured +- **AND** a meeting is actively recording with profile `default` +- **WHEN** the taskbar menu is opened +- **THEN** it offers stop and cancel actions +- **AND** it offers switching to `english` +- **AND** it does not offer switching to `default` + +#### Scenario: Active recording has priority over older summarizing runs +- **GIVEN** an older meeting is still summarizing +- **WHEN** a newer meeting is actively recording +- **THEN** the taskbar icon indicates recording + +#### Scenario: Tray menu always exposes Exit +- **GIVEN** Meeting Assistant is running +- **WHEN** the taskbar menu is opened +- **THEN** it offers an Exit action + +#### Scenario: Idle Exit stops immediately +- **GIVEN** no recording, transcription, speaker recognition, or summary work is running +- **WHEN** the user selects Exit from the taskbar menu +- **THEN** Meeting Assistant stops the application without an additional confirmation prompt + +#### Scenario: In-progress Exit asks for confirmation +- **GIVEN** Meeting Assistant is recording, transcribing, recognizing speakers, or summarizing +- **WHEN** the user selects Exit from the taskbar menu +- **THEN** Meeting Assistant asks for confirmation before stopping the application diff --git a/openspec/changes/add-tray-exit/tasks.md b/openspec/changes/add-tray-exit/tasks.md new file mode 100644 index 0000000..68e5145 --- /dev/null +++ b/openspec/changes/add-tray-exit/tasks.md @@ -0,0 +1,7 @@ +# Tasks + +- [x] Add requirement scenarios for tray Exit and in-progress confirmation. +- [x] Add a failing behavior test proving the tray menu exposes Exit. +- [x] Add the Exit tray menu item. +- [x] Implement Windows tray Exit handling with confirmation for non-idle states. +- [x] Run focused taskbar tests and `openspec validate add-tray-exit --strict`. diff --git a/openspec/changes/azure-speech-offline-resilience/design.md b/openspec/changes/azure-speech-offline-resilience/design.md new file mode 100644 index 0000000..efc80e3 --- /dev/null +++ b/openspec/changes/azure-speech-offline-resilience/design.md @@ -0,0 +1,27 @@ +# Design + +## Current Azure Input Path + +The live Azure Speech path does not use MSAL and does not record audio independently. `AzureSpeechStreamingTranscriptionProvider` receives `AudioChunk` values from `StreamingSpeechRecognitionPipeline`, writes them into a Speech SDK `PushAudioInputStream`, and emits SDK transcript events back as `TranscriptionSegment` values. The recording coordinator separately writes the same mixed chunks to the temporary WAV. + +## Transcript Markers + +Add lightweight transcript marker semantics to `TranscriptionSegment`: + +- a marker segment writes its text directly, for example ``; +- a later real transcript segment can identify the marker it replaces; +- the coordinator rewrites that exact marker line using the existing transcript-line reference path. + +This keeps markers independent from markdown string matching and prevents a later append from being lost when the marker is replaced. + +## Azure Reconnect Loop + +When Azure Speech reports a transient connection cancellation, the provider should stop the current Speech SDK session, leave the upstream audio channel unconsumed while disconnected so it naturally buffers, emit reconnect markers, then create a new SDK session and continue reading the buffered audio. Once the next real transcript segment arrives, it replaces the latest reconnect marker. + +After configured reconnect attempts are exhausted, Azure should emit a longer disconnect marker that explains recording can continue and transcription will drain when Azure reconnects. The provider should continue retrying. + +## Durable Offline Backlog + +If a stopped Azure Speech meeting cannot finish draining before `Recording:StopProcessingTimeout`, the coordinator queues the completed temporary WAV and artifact paths in a persisted offline backlog, releases the active recording slot, and keeps the WAV from startup cleanup. This lets the user start more meetings while Azure or the network is still unavailable. + +The backlog processor retries queued items in the background. Each item creates a fresh speech recognition pipeline for the original launch profile, streams the queued WAV into that pipeline, rewrites the original transcript file with the finished lines, updates meeting-note and transcript metadata, runs transcript-line workflow transformations, transitions the assistant context through summarizing to finished/error, runs the normal summary pipeline, then removes the backlog item and deletes the temporary WAV. Failed processing leaves the backlog item and WAV in place for a later retry. diff --git a/openspec/changes/azure-speech-offline-resilience/proposal.md b/openspec/changes/azure-speech-offline-resilience/proposal.md new file mode 100644 index 0000000..83d8a09 --- /dev/null +++ b/openspec/changes/azure-speech-offline-resilience/proposal.md @@ -0,0 +1,25 @@ +# Azure Speech Offline Resilience + +## Summary + +Make Azure Speech transcription resilient to transient and longer network loss by keeping meeting audio capture alive, surfacing reconnect/disconnect state in the transcript note, and draining buffered audio once Azure Speech can be reached again. + +## Motivation + +When IPv4 connectivity was lost while IPv6 still worked, Azure Speech stopped producing transcript lines but recovered after connectivity returned. The app continued running, which points to Azure Speech connectivity rather than local audio capture. Users need visible status and continued recording instead of silent transcript stalls. + +## Scope + +- Confirm Azure Speech live transcription consumes only Meeting Assistant's audio channel. +- Add transcript marker support for reconnect/disconnect status lines. +- Add Azure Speech transient reconnect markers like ``. +- Replace a reconnect marker with the next real transcript line when Azure resumes. +- Keep audio capture and buffering alive while Azure Speech is unavailable. +- Reset live speaker identity assumptions after Azure reconnects because speaker IDs may change across SDK sessions. +- Queue stopped Azure meetings durably when the SDK cannot drain before the stop timeout, then replay the recorded WAV and complete transcription/summary after connectivity returns. + +## Out Of Scope + +- Replacing Azure Speech with local models. +- Changing the temporary mixed WAV format. +- Replacing the summary or workflow engines used after an offline replay. diff --git a/openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md b/openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md new file mode 100644 index 0000000..35f2f4f --- /dev/null +++ b/openspec/changes/azure-speech-offline-resilience/specs/meeting-transcription/spec.md @@ -0,0 +1,78 @@ +## MODIFIED Requirements + +### Requirement: Azure Speech can provide streaming transcription +Meeting Assistant SHALL provide an Azure Speech speech recognition pipeline that streams captured PCM audio to Azure AI Speech through the Azure Speech SDK conversation transcription API. + +The Azure Speech adapter SHALL use configurable endpoint, region, language, key, and key environment variable settings. + +The Azure Speech adapter SHALL support configurable diarization of intermediate conversation transcription results. + +The Azure Speech adapter SHALL bound recognizer shutdown after the input audio stream is closed. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL emit live transcript segments from Azure conversation transcription events with Azure speaker IDs when available. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL use the live Azure conversation transcription segments as the finished transcript without running pyannote finalization. + +When Azure Speech reports a transient connectivity interruption, Meeting Assistant SHALL keep accepting captured audio into the active in-process pipeline buffer. + +When Azure Speech is reconnecting, Meeting Assistant SHALL write a transcript marker in the form ``. + +When Azure Speech emits the next real transcript segment after reconnecting, Meeting Assistant SHALL replace the latest reconnect marker with that transcript segment. + +When Azure Speech cannot reconnect after the configured immediate retry attempts, Meeting Assistant SHALL write a transcript marker explaining that Azure Speech is disconnected, recording can continue, and transcription/summarization will continue after Azure reconnects. + +After Azure Speech reconnects through a new SDK session, Meeting Assistant SHALL clear live speaker-label assumptions for future Azure speaker labels. + +When Azure Speech is still unavailable after recording stops and transcription cannot drain before the configured stop-processing timeout, Meeting Assistant SHALL persist the stopped meeting as a durable transcription backlog item that references the completed mixed WAV and meeting artifacts. + +When a stopped meeting is persisted to the durable transcription backlog, Meeting Assistant SHALL release the active recording slot so another meeting can be recorded while the stopped meeting waits for Azure Speech to become available. + +When Azure Speech becomes available again, Meeting Assistant SHALL retry durable backlog items, rewrite the transcript from the recorded WAV, run the normal post-transcription meeting completion and summary flow, and remove the backlog item after successful completion. + +When Meeting Assistant starts, it SHALL preserve WAV files that are referenced by durable transcription backlog items instead of deleting them as stale temporary recordings. + +#### Scenario: Azure Speech pipeline is configured +- **WHEN** the configured provider is `azure-speech` +- **THEN** Meeting Assistant streams captured PCM chunks to Azure AI Speech conversation transcription through the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: Azure Speech returns speaker IDs +- **WHEN** Azure Speech conversation transcription returns transcript text with speaker identity +- **THEN** Meeting Assistant emits ordered transcript segments with those Azure speaker identities + +#### Scenario: Azure Speech key is resolved from environment +- **WHEN** Azure Speech is configured with `KeyEnv` +- **THEN** Meeting Assistant reads the Azure Speech key from that environment variable + +#### Scenario: Azure Speech stream shutdown is bounded +- **WHEN** the captured audio stream has ended +- **THEN** Meeting Assistant stops Azure Speech continuous recognition within the configured stop timeout instead of waiting indefinitely + +#### Scenario: Azure Speech transcript is finalized from live conversation segments +- **WHEN** the configured provider is `azure-speech` and live Azure transcript segments exist +- **THEN** Meeting Assistant uses the live Azure conversation transcript segments as the finished transcript + +#### Scenario: Azure reconnect marker is replaced by next transcript line +- **GIVEN** Azure Speech reports a transient connectivity interruption +- **WHEN** Meeting Assistant writes `` and Azure later emits a transcript segment +- **THEN** Meeting Assistant replaces the reconnect marker with the transcript segment +- **AND** keeps later transcript lines intact + +#### Scenario: Azure disconnected marker keeps meeting recording alive +- **GIVEN** Azure Speech cannot reconnect after the configured immediate retry attempts +- **WHEN** meeting audio continues to be captured +- **THEN** Meeting Assistant writes a transcript marker explaining that Azure Speech is disconnected +- **AND** keeps buffering captured audio in the active process for later transcription + +#### Scenario: Stopped Azure meeting is queued durably while offline +- **GIVEN** Azure Speech cannot finish transcription before the recording stop-processing timeout +- **WHEN** the user stops the meeting +- **THEN** Meeting Assistant persists a durable backlog item for the stopped meeting +- **AND** keeps the completed mixed WAV referenced by that backlog item +- **AND** returns to an idle recording state so another meeting can start + +#### Scenario: Durable Azure backlog resumes after connectivity returns +- **GIVEN** a stopped Azure meeting exists in the durable transcription backlog +- **WHEN** Azure Speech can transcribe the recorded WAV +- **THEN** Meeting Assistant rewrites the transcript from the recorded WAV +- **AND** runs meeting completion and summarization +- **AND** removes the durable backlog item and its temporary WAV after successful completion diff --git a/openspec/changes/azure-speech-offline-resilience/tasks.md b/openspec/changes/azure-speech-offline-resilience/tasks.md new file mode 100644 index 0000000..28246bc --- /dev/null +++ b/openspec/changes/azure-speech-offline-resilience/tasks.md @@ -0,0 +1,11 @@ +# Tasks + +- [x] Inspect Azure Speech live input path and confirm whether it uses separate recording/MSAL. +- [x] Add OpenSpec requirements for Azure reconnect/disconnect transcript markers and buffered audio. +- [x] Add transcript marker replacement tests through the recording coordinator. +- [x] Add Azure reconnect marker emission and SDK-session restart behavior. +- [x] Reset live speaker assumptions after Azure reconnect markers. +- [x] Document Azure resilience behavior. +- [x] Run focused tests, full solution tests, and `openspec validate azure-speech-offline-resilience --strict`. +- [x] Add a durable offline backlog for stopped Azure meetings across process restarts. +- [x] Replay queued WAV files through a fresh speech pipeline and finish transcript metadata, meeting context state, summary generation, and backlog cleanup. diff --git a/openspec/changes/resilient-transcript-workflow/design.md b/openspec/changes/resilient-transcript-workflow/design.md new file mode 100644 index 0000000..dbdc3e0 --- /dev/null +++ b/openspec/changes/resilient-transcript-workflow/design.md @@ -0,0 +1,32 @@ +# Design + +## Live Transcript Ordering + +`MeetingRecordingCoordinator` will treat the formatted transcript line as the durable fallback. For each live segment: + +1. Format and relabel the segment. +2. Append the formatted line to the transcript store immediately. +3. Keep the returned transcript-line reference for that exact appended line. +4. Queue `transcript_line` workflow processing out of band so later live segments can continue to be appended. +5. If workflow processing returns a different line, ask the transcript store to replace the referenced line. +6. If workflow processing fails, log the failure and keep the original appended line. + +This preserves the observable transcript even when local automation rules are invalid, slow, or unexpectedly fail after validation. + +## Transcript Store Contract + +Add a transcript-store operation for replacing one previously written line by stable line reference. The vault-backed implementation serializes transcript-file edits per file and rewrites the referenced body line while preserving frontmatter. Existing bulk replacement remains in place for final diarization and speaker relabeling rewrites. + +## Workflow Logging + +The workflow engine already logs when a rule is applied. Extend this so logs show: + +- rule start with rule name and event type, +- rule completion with whether the meeting note changed and whether the transcript line changed, +- rule errors with rule name and event type. + +The engine will still throw for general workflow events so non-transcript lifecycle automation failures remain visible to callers. The recording coordinator catches transcript-line workflow failures during live transcript writes so transcript persistence continues. + +## Disk-Full Follow-Up + +The temporary WAV disk-full exception is tracked as a follow-up task. The expected one-hour file size for the configured `16 kHz / mono / 16-bit` stream is about 110 MiB, so a one-off disk-full error with ample later free space needs separate investigation rather than speculative cleanup behavior. diff --git a/openspec/changes/resilient-transcript-workflow/proposal.md b/openspec/changes/resilient-transcript-workflow/proposal.md new file mode 100644 index 0000000..3a71822 --- /dev/null +++ b/openspec/changes/resilient-transcript-workflow/proposal.md @@ -0,0 +1,25 @@ +# Resilient Transcript Workflow + +## Summary + +Make live transcript persistence independent from workflow-rule success. Transcript lines should be written before optional workflow transformations run, changed lines should be rewritten in place, and workflow rule activity/errors should be logged clearly enough to diagnose broken local rules. + +## Motivation + +A running meeting showed Azure Speech continuing to emit live segments while the transcript markdown stopped advancing. The current append path waits for workflow transformation before writing each line, so a failing or stalled workflow rule can make transcription appear dead even when ASR is still producing output. + +A separate earlier run logged a disk-full `IOException` while writing the temporary WAV, despite the expected WAV size being small. That anomaly should be tracked separately because the current evidence does not identify a safe corrective behavior beyond better diagnostics. + +## Scope + +- Write the formatted live transcript line before transcript-line workflow processing. +- Rewrite the just-written transcript line if workflow rules change it. +- Keep transcript writes going when transcript-line workflow rules fail. +- Log triggered workflow rules, workflow rule completion, and workflow rule errors with event context. +- Track follow-up investigation for anomalous temporary-recording disk-full failures. + +## Out Of Scope + +- Changing ASR providers or audio capture format. +- Restarting or interrupting an active meeting. +- Automatically deleting files or freeing disk space after a disk-full error. diff --git a/openspec/changes/resilient-transcript-workflow/specs/meeting-session/spec.md b/openspec/changes/resilient-transcript-workflow/specs/meeting-session/spec.md new file mode 100644 index 0000000..ade18ba --- /dev/null +++ b/openspec/changes/resilient-transcript-workflow/specs/meeting-session/spec.md @@ -0,0 +1,102 @@ +## MODIFIED Requirements + +### Requirement: Meeting automation rules support lifecycle triggers +Meeting Assistant SHALL support rule triggers for `created`, `state_transition`, `speaker_identified`, and `transcript_line`. + +A `state_transition` trigger MAY filter by `from`, `to`, or both state values. + +A `speaker_identified` trigger MAY filter by speaker name. + +A `transcript_line` trigger MAY filter by speaker name. + +Workflow rule execution SHALL log each triggered rule with its rule name and event type. + +Workflow rule execution SHALL log rule failures with the rule name and event type. + +#### Scenario: State transition rule matches from and to +- **GIVEN** a configured rule that triggers on a state transition from `collecting metadata` to `transcribing` +- **WHEN** Meeting Assistant transitions that meeting from `collecting metadata` to `transcribing` +- **THEN** it applies the rule steps + +#### Scenario: Speaker identified rule filters by name +- **GIVEN** a configured rule that triggers when speaker `Ada` is identified +- **WHEN** Meeting Assistant identifies speaker `Grace` +- **THEN** it does not apply the rule +- **WHEN** Meeting Assistant identifies speaker `Ada` +- **THEN** it applies the rule steps + +#### Scenario: Transcript line rule rewrites masked profanity after durable append +- **GIVEN** a configured rule that triggers on transcript line writes and sets `transcript.line` by replacing `*****` with `[redacted]` +- **WHEN** Meeting Assistant writes a transcript line for speaker `Guest-1` containing `*****` +- **THEN** Meeting Assistant first appends the original formatted line to the transcript file +- **AND** rewrites that written line to contain `[redacted]` +- **AND** the final written transcript line does not contain `*****` + +#### Scenario: Transcript line workflow failure keeps transcript writing +- **GIVEN** a configured transcript line workflow rule fails while processing a transcript line +- **WHEN** Meeting Assistant receives that live transcript segment +- **THEN** Meeting Assistant keeps the original formatted line in the transcript file +- **AND** logs the workflow rule failure +- **AND** continues processing later transcript segments + +### Requirement: Meeting automation rules support conditions and steps +Meeting Assistant SHALL support rule conditions using an expression engine. + +Rules SHALL support nested `and`, `or`, and `not` condition groups. + +Step values SHALL support Razor syntax against the current meeting event model. + +Rendered step values SHALL be treated as plain UTF-8 text for markdown artifacts and SHALL NOT persist Razor HTML entity encoding. + +Step values SHALL treat `@` characters inside valid email address tokens as literal text rather than Razor transitions. + +Meeting Assistant SHALL expose the formatted transcript line and transcript speaker to `transcript_line` rule conditions and Razor step templates. + +Meeting Assistant SHALL support these initial rule steps: + +- `add_attendee` +- `remove_attendee` +- `set_property` +- `add_context` +- `add_project` + +The `set_property` step SHALL support setting `transcript.line` during `transcript_line` events. + +#### Scenario: Nested conditions choose a matching rule +- **GIVEN** a configured rule with nested `and`, `or`, and `not` conditions over meeting title, attendees, and event data +- **WHEN** the condition evaluates to true +- **THEN** Meeting Assistant applies the rule +- **WHEN** the condition evaluates to false +- **THEN** Meeting Assistant skips the rule + +#### Scenario: Templated context mentions identified speaker +- **GIVEN** a configured `speaker_identified` rule with an `add_context` step using Razor syntax +- **WHEN** Meeting Assistant identifies matching speaker `Ada` +- **THEN** it appends rendered context text containing `Ada` to the assistant context note + +#### Scenario: Email addresses do not trigger Razor templating +- **GIVEN** a configured rule step value containing `Support@contoso.com` +- **WHEN** the rule runs +- **THEN** Meeting Assistant preserves the email address as literal text + +#### Scenario: Email addresses can appear beside Razor templating +- **GIVEN** a configured rule step value containing both `Support@contoso.com` and a Razor expression +- **WHEN** the rule runs +- **THEN** Meeting Assistant renders the Razor expression and preserves the email address as literal text + +#### Scenario: Razor-rendered transcript line preserves UTF-8 text +- **GIVEN** a configured `transcript_line` rule uses Razor to replace part of a transcript line containing `heißt`, `Schimpfwörter`, and `nächstes` +- **WHEN** the rule changes `transcript.line` +- **THEN** the resulting transcript line contains the original UTF-8 characters +- **AND** does not contain HTML entities such as `ß`, `ö`, or `ä` + +#### Scenario: Rule can clean a meeting title +- **GIVEN** a configured state-transition rule that matches a title containing a configured marker +- **WHEN** the rule runs +- **THEN** Meeting Assistant can update the meeting title through `set_property` + +#### Scenario: Transcript line conditions can use the written line and speaker +- **GIVEN** a configured `transcript_line` rule with conditions over `transcript.line` and `transcript.speaker` +- **WHEN** Meeting Assistant writes a transcript line that matches both conditions +- **THEN** Meeting Assistant applies the rule steps after the original formatted line is durably appended +- **AND** rewrites the written line if a rule changes `transcript.line` diff --git a/openspec/changes/resilient-transcript-workflow/tasks.md b/openspec/changes/resilient-transcript-workflow/tasks.md new file mode 100644 index 0000000..a8acb71 --- /dev/null +++ b/openspec/changes/resilient-transcript-workflow/tasks.md @@ -0,0 +1,9 @@ +# Tasks + +- [x] Add requirement scenarios for resilient transcript workflow writes and workflow diagnostics. +- [x] Add a failing behavior test proving transcript text is written when transcript-line workflow rules fail. +- [x] Append live transcript lines before workflow transformation and rewrite changed lines afterward. +- [x] Add workflow rule completion/error logging. +- [x] Update workflow engine documentation. +- [x] Run focused tests and `openspec validate resilient-transcript-workflow --strict`. +- [ ] Follow up later: investigate anomalous temporary-recording disk-full `IOException` despite small expected WAV size.