diff --git a/.gitignore b/.gitignore index 42f39ea..227d078 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ MeetingAssistant/appsettings.*.local.json MeetingAssistant/**/appsettings.Local.json MeetingAssistant/**/appsettings.*.local.json .meeting-assistant/ +tmp/ recordings/ Recordings/ transcripts/ diff --git a/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs b/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs index f70c945..97b8142 100644 --- a/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs +++ b/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs @@ -1,6 +1,7 @@ using System.Net; using System.Net.Http.Json; using MeetingAssistant.Recording; +using MeetingAssistant.Speakers; using MeetingAssistant.Transcription; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; @@ -68,6 +69,36 @@ public sealed class AsrDiagnosticEndpointTests Assert.StartsWith("diarized:endpoint-bytes:", segment.Text, StringComparison.Ordinal); } + [Fact] + public async Task SpeakerIdentityMergeDiagnosticEndpointUsesRecentDaysParameter() + { + var mergeService = new CapturingSpeakerIdentityMergeService(); + await using var factory = new WebApplicationFactory() + .WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new Dictionary + { + ["MeetingAssistant:FunAsr:Backend:Enabled"] = "false" + }); + }); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.AddSingleton(mergeService); + }); + }); + using var client = factory.CreateClient(); + + using var response = await client.PostAsJsonAsync( + "/diagnostics/speaker-identities/merge", + new { recentDays = 7 }); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(TimeSpan.FromDays(7), mergeService.RecentIdentityAge); + } + private static WebApplicationFactory CreateFactory() where TProvider : class, IStreamingTranscriptionProvider { @@ -101,6 +132,7 @@ public sealed class AsrDiagnosticEndpointTests { public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { var bytes = 0; @@ -117,6 +149,7 @@ public sealed class AsrDiagnosticEndpointTests { public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); @@ -171,4 +204,18 @@ public sealed class AsrDiagnosticEndpointTests private sealed record AsrDiagnosticResponse(string Path, IReadOnlyList Segments); private sealed record AsrDiagnosticSegment(string Speaker, string Text); + + private sealed class CapturingSpeakerIdentityMergeService : ISpeakerIdentityMergeService + { + public TimeSpan? RecentIdentityAge { get; private set; } + + public Task MergeRecentIdentitiesAsync( + TimeSpan? recentIdentityAge, + CancellationToken cancellationToken) + { + RecentIdentityAge = recentIdentityAge; + return Task.FromResult(new SpeakerIdentityMergeResult(0, 0, 0, 0)); + } + } } + diff --git a/MeetingAssistant.Tests/AsrDiagnosticTests.cs b/MeetingAssistant.Tests/AsrDiagnosticTests.cs index 26c69b9..94938c5 100644 --- a/MeetingAssistant.Tests/AsrDiagnosticTests.cs +++ b/MeetingAssistant.Tests/AsrDiagnosticTests.cs @@ -30,6 +30,7 @@ public sealed class AsrDiagnosticTests public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (var chunk in audio.WithCancellation(cancellationToken)) @@ -77,3 +78,4 @@ public sealed class AsrDiagnosticTests } } } + diff --git a/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs b/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs index 789460f..185400c 100644 --- a/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs +++ b/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs @@ -47,6 +47,7 @@ public sealed class AzureSpeechRecognitionPipelineTests public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (var _ in audio.WithCancellation(cancellationToken)) @@ -60,3 +61,4 @@ public sealed class AzureSpeechRecognitionPipelineTests } } } + diff --git a/MeetingAssistant.Tests/AzureSpeechSpeakerIdentityMatcherTests.cs b/MeetingAssistant.Tests/AzureSpeechSpeakerIdentityMatcherTests.cs new file mode 100644 index 0000000..31c433d --- /dev/null +++ b/MeetingAssistant.Tests/AzureSpeechSpeakerIdentityMatcherTests.cs @@ -0,0 +1,148 @@ +using MeetingAssistant; +using MeetingAssistant.Speakers; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NAudio.Wave; + +namespace MeetingAssistant.Tests; + +public sealed class AzureSpeechSpeakerIdentityMatcherTests +{ + [Fact] + public void IdentityDiarizationLanguageUsesFirstAutoDetectLanguageWhenConfiguredLanguageIsAuto() + { + Assert.Equal( + "de-DE", + AzureSpeechSpeakerIdentityDiarizationClient.ResolveRecognitionLanguage(new AzureSpeechOptions + { + Language = "auto", + AutoDetectLanguages = ["de-DE", "en-US"] + })); + } + + [Fact] + public void IdentityDiarizationLanguageUsesConfiguredConcreteLanguage() + { + Assert.Equal( + "en-US", + AzureSpeechSpeakerIdentityDiarizationClient.ResolveRecognitionLanguage(new AzureSpeechOptions + { + Language = "en-US", + AutoDetectLanguages = ["de-DE"] + })); + } + + [Fact] + public async Task MatcherReturnsIdentityWhenAzureDiarizationAssignsUnknownSnippetToKnownSpeaker() + { + var diarizationClient = new FakeSpeakerIdentityDiarizationClient( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest-1", "known"), + new TranscriptionSegment(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), "Guest-1", "unknown") + ]); + var matcher = CreateMatcher(diarizationClient); + var snippet = CreateWavSnippet(); + + var match = await matcher.MatchAsync( + new SpeakerIdentityMatchRequest( + "Guest03", + snippet, + [new SpeakerIdentityMatchCandidate(42, "Chris", 5, [snippet])]), + CancellationToken.None); + + Assert.NotNull(match); + Assert.Equal(42, match.IdentityId); + Assert.NotNull(diarizationClient.WavPath); + Assert.False(File.Exists(diarizationClient.WavPath)); + } + + [Fact] + public async Task MatcherRequiresTwoMatchingKnownSnippetsWhenIdentityHasMultipleSnippets() + { + var diarizationClient = new FakeSpeakerIdentityDiarizationClient( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest-1", "known one"), + new TranscriptionSegment(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), "Guest-2", "known two"), + new TranscriptionSegment(TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(5), "Guest-1", "unknown") + ]); + var matcher = CreateMatcher(diarizationClient); + var snippet = CreateWavSnippet(); + + var match = await matcher.MatchAsync( + new SpeakerIdentityMatchRequest( + "Guest03", + snippet, + [new SpeakerIdentityMatchCandidate(42, "Chris", 5, [snippet, snippet])]), + CancellationToken.None); + + Assert.Null(match); + } + + [Fact] + public async Task MatcherReturnsNullWhenAzureDiarizationAssignsDifferentSpeakers() + { + var diarizationClient = new FakeSpeakerIdentityDiarizationClient( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest-1", "known"), + new TranscriptionSegment(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), "Guest-2", "unknown") + ]); + var matcher = CreateMatcher(diarizationClient); + var snippet = CreateWavSnippet(); + + var match = await matcher.MatchAsync( + new SpeakerIdentityMatchRequest( + "Guest03", + snippet, + [new SpeakerIdentityMatchCandidate(42, "Chris", 5, [snippet])]), + CancellationToken.None); + + Assert.Null(match); + } + + private static AzureSpeechSpeakerIdentityMatcher CreateMatcher(ISpeakerIdentityDiarizationClient diarizationClient) + { + return new AzureSpeechSpeakerIdentityMatcher( + diarizationClient, + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + SilenceBetweenSnippetsSeconds = 1 + } + }), + NullLogger.Instance); + } + + private static byte[] CreateWavSnippet() + { + using var stream = new MemoryStream(); + using (var writer = new WaveFileWriter(stream, new WaveFormat(16000, 16, 1))) + { + writer.Write(new byte[16000 * 2], 0, 16000 * 2); + } + + return stream.ToArray(); + } + + private sealed class FakeSpeakerIdentityDiarizationClient : ISpeakerIdentityDiarizationClient + { + private readonly IReadOnlyList segments; + + public FakeSpeakerIdentityDiarizationClient(IReadOnlyList segments) + { + this.segments = segments; + } + + public string? WavPath { get; private set; } + + public Task> DiarizeAsync( + string wavPath, + CancellationToken cancellationToken) + { + WavPath = wavPath; + Assert.True(File.Exists(wavPath)); + return Task.FromResult(segments); + } + } +} diff --git a/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs index 261112a..2d06d1d 100644 --- a/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs +++ b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs @@ -23,7 +23,7 @@ public sealed class AzureSpeechStreamingTranscriptionProviderTests var exception = await Assert.ThrowsAsync(async () => { - await foreach (var _ in provider.TranscribeAsync(ReadSingleChunk(), CancellationToken.None)) + await foreach (var _ in provider.TranscribeAsync(ReadSingleChunk(), SpeechRecognitionPipelineOptions.Default, CancellationToken.None)) { } }); @@ -31,6 +31,78 @@ public sealed class AzureSpeechStreamingTranscriptionProviderTests Assert.Contains("Azure Speech key is not configured", exception.Message); } + [Fact] + public void AutoDetectLanguagesAreUsedOnlyWhenLanguageIsAuto() + { + Assert.Equal( + ["de-DE", "en-US"], + AzureSpeechStreamingTranscriptionProvider.GetAutoDetectLanguages(new AzureSpeechOptions + { + Language = "auto", + AutoDetectLanguages = ["de-DE", "en-US", "de-DE", " "] + })); + + Assert.Empty(AzureSpeechStreamingTranscriptionProvider.GetAutoDetectLanguages(new AzureSpeechOptions + { + Language = "de-DE", + AutoDetectLanguages = ["de-DE", "en-US"] + })); + } + + [Fact] + public void NormalizeDictationWordsTrimsAndDeduplicatesPhrases() + { + Assert.Equal( + ["PBI", "Product Backlog Item"], + AzureSpeechStreamingTranscriptionProvider.NormalizeDictationWords([ + " PBI ", + "pbi", + "", + "Product Backlog Item" + ])); + } + + [Fact] + public void ResolveKeyFallsBackToUserEnvironment() + { + var keyEnv = $"MEETING_ASSISTANT_AZURE_KEY_{Guid.NewGuid():N}"; + try + { + Environment.SetEnvironmentVariable(keyEnv, "user-key", EnvironmentVariableTarget.User); + + Assert.Equal("user-key", AzureSpeechStreamingTranscriptionProvider.ResolveKey(new AzureSpeechOptions + { + KeyEnv = keyEnv + })); + } + finally + { + Environment.SetEnvironmentVariable(keyEnv, null, EnvironmentVariableTarget.User); + } + } + + [Fact] + public void EndpointDefaultsToUniversalV2SpeechEndpointForRegion() + { + Assert.Equal( + new Uri("wss://germanywestcentral.stt.speech.microsoft.com/speech/universal/v2"), + AzureSpeechStreamingTranscriptionProvider.GetEndpoint(new AzureSpeechOptions + { + Endpoint = "", + Region = "germanywestcentral" + })); + } + + [Theory] + [InlineData(null, "Continuous")] + [InlineData("", "Continuous")] + [InlineData("continuous", "Continuous")] + [InlineData("AtStart", "AtStart")] + public void LanguageIdModeDefaultsToContinuous(string? configured, string expected) + { + Assert.Equal(expected, AzureSpeechStreamingTranscriptionProvider.NormalizeLanguageIdMode(configured)); + } + private static async IAsyncEnumerable ReadSingleChunk() { await Task.Yield(); diff --git a/MeetingAssistant.Tests/DictationWordStoreTests.cs b/MeetingAssistant.Tests/DictationWordStoreTests.cs new file mode 100644 index 0000000..82893c8 --- /dev/null +++ b/MeetingAssistant.Tests/DictationWordStoreTests.cs @@ -0,0 +1,26 @@ +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Tests; + +public sealed class DictationWordStoreTests +{ + [Fact] + public void NormalizeParsesMarkdownBulletsAndDeduplicatesWords() + { + var words = MarkdownDictationWordStore.Normalize([ + "- PBI", + "* Product Backlog Item", + "pbi", + "`Velocity Vanguards`", + "", + " \"SEW\" " + ]); + + Assert.Equal([ + "PBI", + "Product Backlog Item", + "SEW", + "Velocity Vanguards" + ], words); + } +} diff --git a/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs b/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs index 5569f47..3ee2591 100644 --- a/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs +++ b/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs @@ -36,7 +36,7 @@ public sealed class FunAsrStreamingTranscriptionProviderTests new AudioChunk([3, 0, 4, 0], 16000, 1) }; - var segments = await CollectAsync(provider.TranscribeAsync(ToAsyncEnumerable(chunks), CancellationToken.None)); + var segments = await CollectAsync(provider.TranscribeAsync(ToAsyncEnumerable(chunks), SpeechRecognitionPipelineOptions.Default, CancellationToken.None)); Assert.Equal(new Uri("ws://localhost:10095"), connection.ConnectedEndpoint); Assert.Equal(2, connection.BinaryMessages.Count); @@ -75,6 +75,7 @@ public sealed class FunAsrStreamingTranscriptionProviderTests var segments = await CollectAsync(provider.TranscribeAsync( ToAsyncEnumerable([new AudioChunk([1, 0], 16000, 1)]), + SpeechRecognitionPipelineOptions.Default, CancellationToken.None)); Assert.Collection( @@ -95,6 +96,22 @@ public sealed class FunAsrStreamingTranscriptionProviderTests }); } + [Fact] + public void BuildHotwordsSerializesDictationWordsWithDeduplication() + { + var hotwords = FunAsrStreamingTranscriptionProvider.BuildHotwords([ + "PBI", + "pbi", + "Product Backlog Item", + " " + ]); + + using var document = JsonDocument.Parse(hotwords); + Assert.Equal(20, document.RootElement.GetProperty("PBI").GetInt32()); + Assert.Equal(20, document.RootElement.GetProperty("Product Backlog Item").GetInt32()); + Assert.Equal(2, document.RootElement.EnumerateObject().Count()); + } + private static async IAsyncEnumerable ToAsyncEnumerable(IEnumerable chunks) { foreach (var chunk in chunks) diff --git a/MeetingAssistant.Tests/MeetingSummaryInstructionBuilderTests.cs b/MeetingAssistant.Tests/MeetingSummaryInstructionBuilderTests.cs new file mode 100644 index 0000000..bbf7c9b --- /dev/null +++ b/MeetingAssistant.Tests/MeetingSummaryInstructionBuilderTests.cs @@ -0,0 +1,111 @@ +using MeetingAssistant; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingSummaryInstructionBuilderTests +{ + [Fact] + public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = CreateArtifacts(root); + await WriteMeetingNoteAsync(artifacts.MeetingNotePath, []); + var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions + { + Agent = new AgentOptions { InitialPrompt = " " }, + Vault = new VaultOptions { ProjectsFolder = Path.Combine(root, "Projects") } + })); + + var instructions = await builder.BuildAsync(artifacts, CancellationToken.None); + + Assert.Contains("You are the Meeting Assistant summary agent.", instructions); + } + + [Fact] + public async Task BuilderUsesConfiguredPrompt() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = CreateArtifacts(root); + await WriteMeetingNoteAsync(artifacts.MeetingNotePath, []); + var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions + { + Agent = new AgentOptions { InitialPrompt = "Custom summarizer instructions." }, + Vault = new VaultOptions { ProjectsFolder = Path.Combine(root, "Projects") } + })); + + var instructions = await builder.BuildAsync(artifacts, CancellationToken.None); + + Assert.Equal("Custom summarizer instructions.", instructions); + } + + [Fact] + public async Task BuilderAppendsAgentsFilesForBoundProjects() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var projectsRoot = Path.Combine(root, "Projects"); + var artifacts = CreateArtifacts(root); + await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Beta", "Alpha", "Missing"]); + await WriteProjectAgentsAsync(projectsRoot, "Alpha", "Alpha instructions."); + await WriteProjectAgentsAsync(projectsRoot, "Beta", "Beta instructions."); + Directory.CreateDirectory(Path.Combine(projectsRoot, "Missing")); + var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions + { + Agent = new AgentOptions { InitialPrompt = "Base." }, + Vault = new VaultOptions { ProjectsFolder = projectsRoot } + })); + + var instructions = await builder.BuildAsync(artifacts, CancellationToken.None); + + Assert.Equal( + """ + Base. + + --- + projects: + + # Alpha + + Alpha instructions. + + # Beta + + Beta instructions. + """, + instructions); + } + + private static MeetingSessionArtifacts CreateArtifacts(string root) + { + return new MeetingSessionArtifacts( + Path.Combine(root, "Meetings", "Notes", "meeting.md"), + Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), + Path.Combine(root, "Meetings", "Assistant Context", "context.md"), + Path.Combine(root, "Meetings", "Summaries", "summary.md")); + } + + private static async Task WriteMeetingNoteAsync(string path, IReadOnlyList projects) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var projectLines = projects.Count == 0 + ? "projects: []" + : "projects:\n" + string.Join('\n', projects.Select(project => $"- {project}")); + await File.WriteAllTextAsync( + path, + $""" + --- + title: Meeting + {projectLines} + --- + """); + } + + private static async Task WriteProjectAgentsAsync(string projectsRoot, string project, string content) + { + var folder = Path.Combine(projectsRoot, project); + Directory.CreateDirectory(folder); + await File.WriteAllTextAsync(Path.Combine(folder, "AGENTS.md"), content); + } +} diff --git a/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs index 307369e..5c9454b 100644 --- a/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs +++ b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs @@ -21,5 +21,21 @@ public sealed class OutlookClassicMeetingMetadataProviderTests Assert.Equal("Review current prototype\nDecide next backend", agenda.Replace("\r\n", "\n", StringComparison.Ordinal)); } + + [Fact] + public void NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail() + { + var attendees = OutlookClassicMeetingMetadataProvider.NormalizeAttendees([ + "Marcus.Altmann@sew-eurodrive.de", + "Marcus.Altmann@sew-eurodrive.de ", + "Schweigert, Manuel", + " " + ]); + + Assert.Equal([ + "Marcus.Altmann@sew-eurodrive.de", + "Schweigert, Manuel" + ], attendees); + } } #endif diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs index fe1f3ac..04b1900 100644 --- a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -1,4 +1,5 @@ using MeetingAssistant.Recording; +using MeetingAssistant.Speakers; using MeetingAssistant.Transcription; using MeetingAssistant.MeetingNotes; using MeetingAssistant.Summary; @@ -10,6 +11,22 @@ namespace MeetingAssistant.Tests; public sealed class RecordingCoordinatorTests { + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(5); + while (DateTimeOffset.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(25); + } + + throw new TimeoutException("Condition was not met."); + } + [Fact] public async Task ToggleStartsStreamingTranscriptionAndSecondToggleStopsIt() { @@ -116,6 +133,7 @@ public sealed class RecordingCoordinatorTests DateTimeOffset.Parse("2026-05-19T11:00:00+02:00")))); await coordinator.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => artifactStore.Agenda == "Review API shape"); Assert.Equal("Architecture Sync", noteStore.SavedNote?.Frontmatter.Title); Assert.Equal(["Ada ", "Grace"], noteStore.SavedNote?.Frontmatter.Attendees); @@ -125,6 +143,101 @@ public sealed class RecordingCoordinatorTests await coordinator.StopAsync(CancellationToken.None); } + [Fact] + public async Task StartDoesNotWaitForSlowMeetingMetadataButAppliesItLater() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\metadata-meeting.md"); + var artifactStore = new InMemoryMeetingArtifactStore(); + var provider = new BlockingMeetingMetadataProvider(new MeetingMetadata( + "Late Calendar Match", + ["Ada"], + "Late agenda", + DateTimeOffset.Parse("2026-05-19T12:00:00+02:00"))); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + provider); + + await coordinator.StartAsync(CancellationToken.None); + + Assert.StartsWith("Meeting ", noteStore.SavedNote?.Frontmatter.Title, StringComparison.Ordinal); + provider.Release(); + await WaitUntilAsync(() => + noteStore.SavedNote?.Frontmatter.Title == "Late Calendar Match" && + artifactStore.Agenda == "Late agenda"); + + Assert.Equal(["Ada"], noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Equal("Late agenda", artifactStore.Agenda); + Assert.Equal(DateTimeOffset.Parse("2026-05-19T12:00:00+02:00"), artifactStore.ScheduledEnd); + + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartCreatesMeetingNoteWhenMetadataProviderFails() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\fallback-meeting.md"); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore("C:\\Vault\\Meetings\\Transcripts\\fallback-transcript.md"), + noteStore, + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + new ThrowingMeetingMetadataProvider()); + + var status = await coordinator.StartAsync(CancellationToken.None); + + Assert.True(status.IsRecording); + Assert.Equal("C:\\Vault\\Meetings\\Notes\\fallback-meeting.md", status.MeetingNotePath); + Assert.NotNull(artifactStore.CreatedArtifacts); + Assert.StartsWith("Meeting ", noteStore.SavedNote?.Frontmatter.Title, StringComparison.Ordinal); + + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartContinuesWhenObsidianOpenFails() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\open-fails.md"); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore("C:\\Vault\\Meetings\\Transcripts\\open-fails-transcript.md"), + noteStore, + new ThrowingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + var status = await coordinator.StartAsync(CancellationToken.None); + + Assert.True(status.IsRecording); + Assert.Equal("C:\\Vault\\Meetings\\Notes\\open-fails.md", status.MeetingNotePath); + Assert.NotNull(artifactStore.CreatedArtifacts); + + await coordinator.StopAsync(CancellationToken.None); + } + [Fact] public async Task StopCompletesAudioCaptureAndDrainsFinalTranscriptionWindow() { @@ -187,6 +300,33 @@ public sealed class RecordingCoordinatorTests await coordinator.StopAsync(CancellationToken.None); } + [Fact] + public async Task StartPassesDictationWordsToSpeechPipeline() + { + var audioSource = new ControlledAudioSource(); + var provider = new CapturingPipelineOptionsProvider(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + dictationWordStore: new FixedDictationWordStore(["PBI", "Product Backlog Item"])); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await provider.OptionsObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(["PBI", "Product Backlog Item"], provider.Options?.DictationWords); + + await coordinator.StopAsync(CancellationToken.None); + } + [Fact] public async Task StopRewritesTranscriptWithFinalDiarizedSegmentsWhenAvailable() { @@ -236,6 +376,272 @@ public sealed class RecordingCoordinatorTests }); } + [Fact] + public async Task StopRelabelsFinishedTranscriptWithLearnedSpeakerIdentity() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk(new byte[] { 1, 0, 2, 0 }, 16000, 1)); + var transcriptStore = new InMemoryTranscriptStore(); + var noteStore = new InMemoryMeetingNoteStore(); + noteStore.UpdateSavedNote(MeetingNoteTemplate.Create( + "Identity Test", + DateTimeOffset.Parse("2026-05-20T12:00:00+02:00"), + attendees: ["Chris"], + transcriptPath: "memory-transcript.md", + assistantContextPath: "memory-context.md", + summaryPath: "memory-summary.md")); + var finalizer = new CapturingTranscriptFinalizer( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest03", "hello") + ]); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider(), finalizer.FinalizeAsync), + transcriptStore, + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + speakerIdentificationService: new FixedSpeakerIdentificationService("Guest03", "Chris")); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal("Chris", transcriptStore.ReplacedSegments.Single().Speaker); + } + + [Fact] + public async Task StopAddsIdentifiedSpeakerToMeetingAttendees() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk(new byte[] { 1, 0, 2, 0 }, 16000, 1)); + var noteStore = new InMemoryMeetingNoteStore(); + var finalizer = new CapturingTranscriptFinalizer( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest03", "hello") + ]); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider(), finalizer.FinalizeAsync), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + speakerIdentificationService: new FixedSpeakerIdentificationService( + "Guest03", + "Chris", + ["Chris", "Christopher"])); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + var attendees = Assert.IsType>(noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Contains("Chris", attendees); + } + + [Fact] + public async Task StopDoesNotDuplicateIdentifiedSpeakerWhenAliasIsAlreadyAttendee() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk(new byte[] { 1, 0, 2, 0 }, 16000, 1)); + var noteStore = new InMemoryMeetingNoteStore(); + var finalizer = new CapturingTranscriptFinalizer( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest03", "hello") + ]); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider(), finalizer.FinalizeAsync), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + speakerIdentificationService: new FixedSpeakerIdentificationService( + "Guest03", + "Christopher", + ["Christopher", "Chris"])); + + await coordinator.StartAsync(CancellationToken.None); + noteStore.UpdateAttendees(["Chris "]); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + var attendees = Assert.IsType>(noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Equal(["Chris "], attendees); + } + + [Fact] + public async Task LiveSpeakerIdentityMatchRelabelsCurrentAndFutureTranscriptSegments() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore(); + var speakerIdentification = new BlockingSpeakerIdentificationService("Guest03", "Chris"); + var provider = new OrderedChunkProvider(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + InitialDelay = TimeSpan.Zero, + Interval = TimeSpan.FromMilliseconds(10) + } + }), + NullLogger.Instance, + speakerIdentificationService: speakerIdentification); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk(Samples(0, 1, 2, 3, 4, 5, 6, 7), 4, 1), CancellationToken.None); + await WaitUntilAsync(() => transcriptStore.Segments.Any(segment => segment.Text.Contains("first", StringComparison.Ordinal))); + await speakerIdentification.IdentificationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => transcriptStore.ReplacedSegments.Any(segment => segment.Text.Contains("first", StringComparison.Ordinal))); + Assert.Equal("Chris", transcriptStore.ReplacedSegments.Single().Speaker); + await Task.Delay(50); + await audioSource.WriteAsync(new AudioChunk(Samples(8, 9, 10, 11, 12, 13, 14, 15), 4, 1), CancellationToken.None); + + await transcriptStore.WaitForTextAsync("second"); + await coordinator.StopAsync(CancellationToken.None); + + Assert.Collection( + transcriptStore.Segments, + first => Assert.Equal("Guest03", first.Speaker), + second => Assert.Equal("Chris", second.Speaker)); + Assert.Contains( + transcriptStore.ReplacedSegments, + segment => segment.Text.Contains("first", StringComparison.Ordinal) && segment.Speaker == "Chris"); + } + + [Fact] + public async Task LiveSpeakerIdentityMatchAddsIdentifiedSpeakerToMeetingAttendees() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore(); + var speakerIdentification = new BlockingSpeakerIdentificationService("Guest03", "Chris"); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new OrderedChunkProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + InitialDelay = TimeSpan.Zero, + Interval = TimeSpan.FromMilliseconds(10) + } + }), + NullLogger.Instance, + speakerIdentificationService: speakerIdentification); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk(Samples(0, 1, 2, 3, 4, 5, 6, 7), 4, 1), CancellationToken.None); + + await WaitUntilAsync(() => noteStore.SavedNote?.Frontmatter.Attendees.Contains("Chris") == true); + var attendees = Assert.IsType>(noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Contains("Chris", attendees); + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task LiveSpeakerIdentificationRetriesWhenAttendeesChange() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore(); + var speakerIdentification = new CountingSpeakerIdentificationService(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new OrderedChunkProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + InitialDelay = TimeSpan.Zero, + Interval = TimeSpan.FromMilliseconds(20) + } + }), + NullLogger.Instance, + speakerIdentificationService: speakerIdentification); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk(Samples(0, 1, 2, 3, 4, 5, 6, 7), 4, 1), CancellationToken.None); + await WaitUntilAsync(() => speakerIdentification.Requests.Count == 1); + + noteStore.UpdateAttendees(["Chris"]); + + await WaitUntilAsync(() => speakerIdentification.Requests.Count == 2); + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal(["Chris"], speakerIdentification.Requests.Last().MeetingNote.Frontmatter.Attendees); + } + + [Fact] + public async Task LiveSpeakerIdentificationRetriesWhenNewUnmappedSpeakerAppears() + { + var audioSource = new ControlledAudioSource(); + var speakerIdentification = new CountingSpeakerIdentificationService(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new ChangingSpeakerOrderedChunkProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + InitialDelay = TimeSpan.Zero, + Interval = TimeSpan.FromMilliseconds(20) + } + }), + NullLogger.Instance, + speakerIdentificationService: speakerIdentification); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk(Samples(0, 1, 2, 3, 4, 5, 6, 7), 4, 1), CancellationToken.None); + await WaitUntilAsync(() => speakerIdentification.Requests.Count == 1); + await audioSource.WriteAsync(new AudioChunk(Samples(8, 9, 10, 11, 12, 13, 14, 15), 4, 1), CancellationToken.None); + + await WaitUntilAsync(() => speakerIdentification.Requests.Count == 2); + await coordinator.StopAsync(CancellationToken.None); + + var samples = Assert.IsAssignableFrom>( + speakerIdentification.Requests.Last().Samples); + Assert.Contains(samples, sample => sample.Speaker == "Guest04"); + } + [Theory] [InlineData(0, null)] [InlineData(1, null)] @@ -538,6 +944,8 @@ public sealed class RecordingCoordinatorTests public IReadOnlyList ReplacedSegments { get; private set; } = []; + public IReadOnlyList Segments => segments; + public MeetingNote? MetadataMeetingNote { get; private set; } public Task ReplaceAsync( @@ -591,6 +999,11 @@ public sealed class RecordingCoordinatorTests SavedNote.Frontmatter.Attendees = attendees.ToList(); } + + public void UpdateSavedNote(MeetingNote note) + { + SavedNote = note with { Path = notePath }; + } } private sealed class CapturingMeetingNoteOpener : IMeetingNoteOpener @@ -634,6 +1047,17 @@ public sealed class RecordingCoordinatorTests States.Add(state); return Task.CompletedTask; } + + public Task UpdateAssistantContextMetadataAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken) + { + Agenda = agenda; + ScheduledEnd = scheduledEnd; + return Task.CompletedTask; + } } private sealed class FixedMeetingMetadataProvider : IMeetingMetadataProvider @@ -653,6 +1077,48 @@ public sealed class RecordingCoordinatorTests } } + private sealed class ThrowingMeetingMetadataProvider : IMeetingMetadataProvider + { + public Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + throw new InvalidOperationException("metadata unavailable"); + } + } + + private sealed class BlockingMeetingMetadataProvider : IMeetingMetadataProvider + { + private readonly MeetingMetadata metadata; + private readonly TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public BlockingMeetingMetadataProvider(MeetingMetadata metadata) + { + this.metadata = metadata; + } + + public void Release() + { + release.TrySetResult(); + } + + public async Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + await release.Task.WaitAsync(cancellationToken); + return metadata; + } + } + + private sealed class ThrowingMeetingNoteOpener : IMeetingNoteOpener + { + public Task OpenAsync(string notePath, CancellationToken cancellationToken) + { + throw new InvalidOperationException("obsidian unavailable"); + } + } + private sealed class CapturingMeetingSummaryPipeline : IMeetingSummaryPipeline { private readonly bool succeeded; @@ -817,6 +1283,181 @@ public sealed class RecordingCoordinatorTests } } + private sealed class FixedSpeakerIdentificationService : ISpeakerIdentificationService + { + private readonly string sourceSpeaker; + private readonly string targetSpeaker; + private readonly IReadOnlyList acceptedNames; + + public FixedSpeakerIdentificationService( + string sourceSpeaker, + string targetSpeaker, + IReadOnlyList? acceptedNames = null) + { + this.sourceSpeaker = sourceSpeaker; + this.targetSpeaker = targetSpeaker; + this.acceptedNames = acceptedNames ?? [targetSpeaker]; + } + + public Task ProcessFinishedTranscriptAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return Task.FromResult(new SpeakerIdentificationResult( + request.Segments.Select(segment => segment.Speaker == sourceSpeaker + ? segment with { Speaker = targetSpeaker } + : segment).ToList(), + new Dictionary { [sourceSpeaker] = targetSpeaker }, + [new SpeakerIdentityAttendeeMatch(targetSpeaker, acceptedNames)])); + } + + public Task IdentifyKnownSpeakersAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return ProcessFinishedTranscriptAsync(request, cancellationToken); + } + } + + private sealed class BlockingSpeakerIdentificationService : ISpeakerIdentificationService + { + private readonly string sourceSpeaker; + private readonly string targetSpeaker; + private readonly IReadOnlyList acceptedNames; + + public BlockingSpeakerIdentificationService( + string sourceSpeaker, + string targetSpeaker, + IReadOnlyList? acceptedNames = null) + { + this.sourceSpeaker = sourceSpeaker; + this.targetSpeaker = targetSpeaker; + this.acceptedNames = acceptedNames ?? [targetSpeaker]; + } + + public TaskCompletionSource IdentificationObserved { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task IdentifyKnownSpeakersAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + IdentificationObserved.TrySetResult(); + return Task.FromResult(new SpeakerIdentificationResult( + request.Segments, + new Dictionary { [sourceSpeaker] = targetSpeaker }, + [new SpeakerIdentityAttendeeMatch(targetSpeaker, acceptedNames)])); + } + + public Task ProcessFinishedTranscriptAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return Task.FromResult(new SpeakerIdentificationResult(request.Segments, new Dictionary())); + } + } + + private sealed class CountingSpeakerIdentificationService : ISpeakerIdentificationService + { + public List Requests { get; } = []; + + public Task IdentifyKnownSpeakersAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(new SpeakerIdentificationResult(request.Segments, new Dictionary())); + } + + public Task ProcessFinishedTranscriptAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return Task.FromResult(new SpeakerIdentificationResult(request.Segments, new Dictionary())); + } + } + + private sealed class FixedDictationWordStore : IDictationWordStore + { + private readonly IReadOnlyList words; + + public FixedDictationWordStore(IReadOnlyList words) + { + this.words = words; + } + + public Task> ReadWordsAsync(CancellationToken cancellationToken) + { + return Task.FromResult(words); + } + + public Task> AddWordAsync(string word, CancellationToken cancellationToken) + { + return Task.FromResult(words); + } + } + + private sealed class CapturingPipelineOptionsProvider : IStreamingTranscriptionProvider + { + public TaskCompletionSource OptionsObserved { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public SpeechRecognitionPipelineOptions? Options { get; private set; } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + Options = options; + OptionsObserved.TrySetResult(); + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + yield return new TranscriptionSegment(TimeSpan.Zero, TimeSpan.Zero, "Unknown", "ok"); + } + } + } + + private sealed class OrderedChunkProvider : IStreamingTranscriptionProvider + { + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var index = 0; + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + var text = index++ == 0 ? "first" : "second"; + yield return new TranscriptionSegment( + TimeSpan.FromSeconds(index - 1), + TimeSpan.FromSeconds(index + 1), + "Guest03", + $"{text} has enough words for identification."); + } + } + } + + private sealed class ChangingSpeakerOrderedChunkProvider : IStreamingTranscriptionProvider + { + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var index = 0; + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + var speaker = index++ == 0 ? "Guest03" : "Guest04"; + yield return new TranscriptionSegment( + TimeSpan.FromSeconds(index - 1), + TimeSpan.FromSeconds(index + 1), + speaker, + "This segment has enough words for identification."); + } + } + } + private sealed class ControlledAudioSource : IMeetingAudioSource { private readonly Channel chunks = Channel.CreateUnbounded(); @@ -870,6 +1511,7 @@ public sealed class RecordingCoordinatorTests public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (var chunk in audio.WithCancellation(cancellationToken)) @@ -884,6 +1526,7 @@ public sealed class RecordingCoordinatorTests { public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { var byteCount = 0; @@ -915,6 +1558,7 @@ public sealed class RecordingCoordinatorTests public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { waitingForBackend.TrySetResult(); @@ -925,6 +1569,14 @@ public sealed class RecordingCoordinatorTests } } } + + private static byte[] Samples(params short[] samples) + { + var bytes = new byte[samples.Length * sizeof(short)]; + Buffer.BlockCopy(samples, 0, bytes, 0, bytes.Length); + return bytes; + } } + diff --git a/MeetingAssistant.Tests/RollingAudioBufferTests.cs b/MeetingAssistant.Tests/RollingAudioBufferTests.cs new file mode 100644 index 0000000..1cb8b1f --- /dev/null +++ b/MeetingAssistant.Tests/RollingAudioBufferTests.cs @@ -0,0 +1,42 @@ +using MeetingAssistant.Recording; +using NAudio.Wave; + +namespace MeetingAssistant.Tests; + +public sealed class RollingAudioBufferTests +{ + [Fact] + public void TryExtractWavReturnsRequestedRangeFromBufferedPcm() + { + var buffer = new RollingAudioBuffer(TimeSpan.FromSeconds(10)); + buffer.Append(new AudioChunk(Samples(0, 1, 2, 3), SampleRate: 2, Channels: 1)); + buffer.Append(new AudioChunk(Samples(4, 5, 6, 7), SampleRate: 2, Channels: 1)); + + var wav = buffer.TryExtractWav(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3)); + + using var reader = new WaveFileReader(new MemoryStream(wav)); + var bytes = new byte[reader.Length]; + var read = reader.Read(bytes, 0, bytes.Length); + Assert.Equal(bytes.Length, read); + Assert.Equal(Samples(2, 3, 4, 5), bytes); + } + + [Fact] + public void TryExtractWavDropsAudioOutsideRetentionWindow() + { + var buffer = new RollingAudioBuffer(TimeSpan.FromSeconds(1)); + buffer.Append(new AudioChunk(Samples(0, 1), SampleRate: 1, Channels: 1)); + buffer.Append(new AudioChunk(Samples(2, 3), SampleRate: 1, Channels: 1)); + + var wav = buffer.TryExtractWav(TimeSpan.Zero, TimeSpan.FromSeconds(1)); + + Assert.Empty(wav); + } + + private static byte[] Samples(params short[] samples) + { + var bytes = new byte[samples.Length * sizeof(short)]; + Buffer.BlockCopy(samples, 0, bytes, 0, bytes.Length); + return bytes; + } +} diff --git a/MeetingAssistant.Tests/SpeakerIdentityMergeServiceTests.cs b/MeetingAssistant.Tests/SpeakerIdentityMergeServiceTests.cs new file mode 100644 index 0000000..7e5d87f --- /dev/null +++ b/MeetingAssistant.Tests/SpeakerIdentityMergeServiceTests.cs @@ -0,0 +1,198 @@ +using MeetingAssistant; +using MeetingAssistant.Speakers; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class SpeakerIdentityMergeServiceTests +{ + [Fact] + public async Task MergeRecentIdentitiesConfirmsMatchTwiceAndMergesAliasesAndSamples() + { + await using var fixture = await SpeakerIdentityMergeFixture.CreateAsync(); + var older = await fixture.AddIdentityAsync( + canonicalName: "Manuel", + aliases: ["M. Schweigert"], + snippets: [[1], [2], [3]], + createdAt: DateTimeOffset.UtcNow.AddMonths(-2), + transcriptionCount: 5); + var recent = await fixture.AddIdentityAsync( + canonicalName: "Guest Manuel", + aliases: [], + snippets: [[4], [5]], + createdAt: DateTimeOffset.UtcNow.AddDays(-1), + transcriptionCount: 2); + fixture.Matcher.MatchIdentityIds.Enqueue(older.Id); + fixture.Matcher.MatchIdentityIds.Enqueue(older.Id); + var service = fixture.CreateService(); + + var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None); + + Assert.Equal(1, result.MergedPairs); + Assert.Equal(2, fixture.Matcher.Requests.Count); + Assert.Equal([4], fixture.Matcher.Requests[0].UnknownSnippet); + Assert.Equal([5], fixture.Matcher.Requests[1].UnknownSnippet); + var savedOlder = await fixture.LoadIdentityAsync(older.Id); + Assert.Equal("Manuel", savedOlder.CanonicalName); + Assert.Equal(7, savedOlder.TranscriptionCount); + Assert.True(savedOlder.UpdatedAt > older.UpdatedAt); + Assert.Equal(["Guest Manuel", "M. Schweigert"], savedOlder.Aliases.Select(alias => alias.Name).Order()); + Assert.Equal(3, savedOlder.Snippets.Count); + Assert.False(await fixture.Context.SpeakerIdentities.AnyAsync(identity => identity.Id == recent.Id)); + } + + [Fact] + public async Task MergeRecentIdentitiesDoesNotMergeWithoutSecondValidation() + { + await using var fixture = await SpeakerIdentityMergeFixture.CreateAsync(); + var older = await fixture.AddIdentityAsync("Manuel", [], [[1]], DateTimeOffset.UtcNow.AddMonths(-2), 5); + var recent = await fixture.AddIdentityAsync("Guest Manuel", [], [[4], [5]], DateTimeOffset.UtcNow.AddDays(-1), 2); + fixture.Matcher.MatchIdentityIds.Enqueue(older.Id); + var service = fixture.CreateService(); + + var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None); + + Assert.Equal(0, result.MergedPairs); + Assert.True(await fixture.Context.SpeakerIdentities.AnyAsync(identity => identity.Id == recent.Id)); + } + + [Fact] + public async Task MergeRecentIdentitiesIgnoresOldSourceIdentities() + { + await using var fixture = await SpeakerIdentityMergeFixture.CreateAsync(); + await fixture.AddIdentityAsync("Older A", [], [[1], [2]], DateTimeOffset.UtcNow.AddMonths(-2), 5); + await fixture.AddIdentityAsync("Older B", [], [[4], [5]], DateTimeOffset.UtcNow.AddMonths(-1), 2); + var service = fixture.CreateService(); + + var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None); + + Assert.Equal(0, result.MergedPairs); + Assert.Empty(fixture.Matcher.Requests); + } + + private sealed class SpeakerIdentityMergeFixture : IAsyncDisposable + { + private readonly string dbPath; + + private SpeakerIdentityMergeFixture(string dbPath, SpeakerIdentityDbContext context) + { + this.dbPath = dbPath; + Context = context; + } + + public SpeakerIdentityDbContext Context { get; } + + public QueueingSpeakerIdentityMatcher Matcher { get; } = new(); + + public static async Task CreateAsync() + { + var dbPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", $"{Guid.NewGuid():N}.db"); + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); + var context = new SpeakerIdentityDbContext(new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbPath};Pooling=False") + .Options); + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None); + return new SpeakerIdentityMergeFixture(dbPath, context); + } + + public SpeakerIdentityMergeService CreateService() + { + return new SpeakerIdentityMergeService( + new TestSpeakerIdentityDbContextFactory(dbPath), + Matcher, + Options.Create(new MeetingAssistantOptions + { + SpeakerIdentification = new SpeakerIdentificationOptions + { + MatchBatchSize = 6, + MaxSnippetsPerSpeaker = 3 + } + }), + NullLogger.Instance); + } + + public async Task AddIdentityAsync( + string? canonicalName, + IReadOnlyList aliases, + IReadOnlyList snippets, + DateTimeOffset createdAt, + int transcriptionCount) + { + var identity = new SpeakerIdentity + { + CanonicalName = canonicalName, + CreatedAt = createdAt, + UpdatedAt = createdAt, + TranscriptionCount = transcriptionCount, + Aliases = aliases.Select(alias => new SpeakerAlias { Name = alias }).ToList(), + Snippets = snippets + .Select((snippet, index) => new SpeakerSnippet + { + WavBytes = snippet, + CreatedAt = createdAt.AddMinutes(index) + }) + .ToList() + }; + Context.SpeakerIdentities.Add(identity); + await Context.SaveChangesAsync(); + return identity; + } + + public Task LoadIdentityAsync(int id) + { + Context.ChangeTracker.Clear(); + return Context.SpeakerIdentities + .Include(identity => identity.Aliases) + .Include(identity => identity.Snippets) + .SingleAsync(identity => identity.Id == id); + } + + public async ValueTask DisposeAsync() + { + await Context.DisposeAsync(); + if (File.Exists(dbPath)) + { + File.Delete(dbPath); + } + } + } + + private sealed class TestSpeakerIdentityDbContextFactory : IDbContextFactory + { + private readonly string dbPath; + + public TestSpeakerIdentityDbContextFactory(string dbPath) + { + this.dbPath = dbPath; + } + + public SpeakerIdentityDbContext CreateDbContext() + { + return new SpeakerIdentityDbContext(new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbPath};Pooling=False") + .Options); + } + } + + private sealed class QueueingSpeakerIdentityMatcher : ISpeakerIdentityMatcher + { + public Queue MatchIdentityIds { get; } = new(); + + public List Requests { get; } = []; + + public Task MatchAsync( + SpeakerIdentityMatchRequest request, + CancellationToken cancellationToken) + { + Requests.Add(request); + if (!MatchIdentityIds.TryDequeue(out var identityId)) + { + return Task.FromResult(null); + } + + return Task.FromResult(new SpeakerIdentityMatch(identityId)); + } + } +} diff --git a/MeetingAssistant.Tests/SpeakerIdentityServiceTests.cs b/MeetingAssistant.Tests/SpeakerIdentityServiceTests.cs new file mode 100644 index 0000000..df3cdbc --- /dev/null +++ b/MeetingAssistant.Tests/SpeakerIdentityServiceTests.cs @@ -0,0 +1,451 @@ +using MeetingAssistant; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Speakers; +using MeetingAssistant.Transcription; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class SpeakerIdentityServiceTests +{ + [Fact] + public async Task MatchedUnnamedIdentityEliminatesCandidatesAndPromotesCanonicalName() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var identity = await fixture.AddIdentityAsync(["John", "Mike"], [1, 2, 3]); + fixture.Matcher.MatchIdentityId = identity.Id; + var service = fixture.CreateService(); + + var result = await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["Jane", "John", "Chris"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var saved = await fixture.LoadIdentityAsync(identity.Id); + Assert.Equal("John", saved.CanonicalName); + Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name)); + Assert.Equal(1, saved.TranscriptionCount); + Assert.Equal("John", result.Segments.Single().Speaker); + Assert.Equal("John", result.SpeakerMappings["Guest01"]); + } + + [Fact] + public async Task MatchedIdentityUpdatesLastModifiedTimestamp() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var oldTimestamp = DateTimeOffset.UtcNow.AddDays(-30); + var identity = await fixture.AddIdentityAsync( + [], + [1, 2, 3], + "Chris", + transcriptionCount: 5, + updatedAt: oldTimestamp); + fixture.Matcher.MatchIdentityId = identity.Id; + var service = fixture.CreateService(); + + await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["Chris"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var saved = await fixture.LoadIdentityAsync(identity.Id); + Assert.True(saved.UpdatedAt > oldTimestamp); + } + + [Fact] + public async Task MatchedIdentityWithEmptyCandidateIntersectionResetsCandidatesAndReplacesOldestSnippet() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(options => options.MaxSnippetsPerSpeaker = 1); + var identity = await fixture.AddIdentityAsync(["John", "Mike"], [9, 9, 9]); + fixture.Matcher.MatchIdentityId = identity.Id; + var service = fixture.CreateService(); + + await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["Jane", "Chris"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var saved = await fixture.LoadIdentityAsync(identity.Id); + Assert.Null(saved.CanonicalName); + Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order()); + Assert.Equal([7, 8, 9], saved.Snippets.Single().WavBytes); + } + + [Fact] + public async Task MatchedUnnamedIdentityTreatsAliasesAsCandidateMatches() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var identity = await fixture.AddIdentityAsync(["Michael"], [1, 2, 3], aliases: ["Mike"]); + fixture.Matcher.MatchIdentityId = identity.Id; + var service = fixture.CreateService(); + + await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["Mike", "Jane"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var saved = await fixture.LoadIdentityAsync(identity.Id); + Assert.Equal("Michael", saved.CanonicalName); + Assert.Equal(["Michael"], saved.CandidateNames.Select(candidate => candidate.Name)); + } + + [Fact] + public async Task UnmatchedDiarizedSpeakersCreateCandidateIdentitiesFromUnmatchedAttendees() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var chris = await fixture.AddIdentityAsync([], [1, 2, 3], "Chris", transcriptionCount: 5); + fixture.Matcher.MatchIdentityId = chris.Id; + var service = fixture.CreateService(); + + await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["John", "Mike", "Chris"], + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest03", "known"), + new TranscriptionSegment(TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8), "Guest01", "unknown one"), + new TranscriptionSegment(TimeSpan.FromSeconds(8), TimeSpan.FromSeconds(12), "Guest02", "unknown two") + ]), + CancellationToken.None); + + var learned = await fixture.Context.SpeakerIdentities + .Include(identity => identity.CandidateNames) + .Where(identity => identity.CanonicalName == null) + .OrderBy(identity => identity.Id) + .ToListAsync(); + + Assert.Equal(2, learned.Count); + Assert.All(learned, identity => + { + Assert.NotEqual(default, identity.CreatedAt); + Assert.NotEqual(default, identity.UpdatedAt); + Assert.Equal(["John", "Mike"], identity.CandidateNames.Select(candidate => candidate.Name).Order()); + }); + } + + [Fact] + public async Task UnmatchedSpeakersExcludeAliasesOfMatchedIdentitiesFromCandidates() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var michael = await fixture.AddIdentityAsync([], [1, 2, 3], "Michael", transcriptionCount: 5, aliases: ["Mike"]); + fixture.Matcher.MatchIdentityId = michael.Id; + var service = fixture.CreateService(); + + await service.ProcessFinishedTranscriptAsync( + fixture.CreateRequest( + ["Mike", "Jane"], + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest03", "known"), + new TranscriptionSegment(TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8), "Guest01", "unknown") + ]), + CancellationToken.None); + + var learned = await fixture.Context.SpeakerIdentities + .Include(identity => identity.CandidateNames) + .Where(identity => identity.CanonicalName == null) + .SingleAsync(); + Assert.Equal(["Jane"], learned.CandidateNames.Select(candidate => candidate.Name)); + } + + [Fact] + public async Task SuppliedSpeakerSamplesAreUsedBeforeExtractingFromAudioFile() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var chris = await fixture.AddIdentityAsync([], [1, 2, 3], "Chris", transcriptionCount: 5); + fixture.Matcher.MatchIdentityId = chris.Id; + var service = fixture.CreateService(); + var segment = new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(5), "Guest03", "hello from the live buffer"); + + var result = await service.IdentifyKnownSpeakersAsync( + fixture.CreateRequest( + ["Chris"], + [segment], + [new SpeakerAudioSample("Guest03", segment, [4, 5, 6], 90)]), + CancellationToken.None); + + Assert.Equal("Chris", result.Segments.Single().Speaker); + Assert.Equal([4, 5, 6], fixture.Matcher.LastUnknownSnippet); + Assert.Equal(0, fixture.SnippetExtractor.ExtractCalls); + } + + [Fact] + public async Task LiveKnownSpeakerIdentificationDoesNotMutateIdentityDatabase() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var identity = await fixture.AddIdentityAsync(["John", "Mike"], [1, 2, 3], transcriptionCount: 5); + fixture.Matcher.MatchIdentityId = identity.Id; + var service = fixture.CreateService(); + + var result = await service.IdentifyKnownSpeakersAsync( + fixture.CreateRequest( + ["Jane", "John", "Chris"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var saved = await fixture.LoadIdentityAsync(identity.Id); + Assert.Null(saved.CanonicalName); + Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order()); + Assert.Equal(5, saved.TranscriptionCount); + Assert.Single(saved.Snippets); + Assert.Equal("Guest01", result.Segments.Single().Speaker); + Assert.Empty(result.SpeakerMappings); + } + + [Fact] + public async Task SpeakerMatchingPrioritizesAttendeeDisplayNamesAndAliases() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(); + var unrelated = await fixture.AddIdentityAsync([], [1], "Frequent", transcriptionCount: 99); + var aliasMatch = await fixture.AddIdentityAsync([], [2], "Michael", transcriptionCount: 1, aliases: ["Mike"]); + var displayNameMatch = await fixture.AddIdentityAsync([], [3], "Jane", transcriptionCount: 2); + var service = fixture.CreateService(); + + await service.IdentifyKnownSpeakersAsync( + fixture.CreateRequest( + ["Jane", "Mike"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var requestedIds = fixture.Matcher.Requests.Single().Candidates.Select(candidate => candidate.IdentityId).ToList(); + Assert.Equal([displayNameMatch.Id, aliasMatch.Id, unrelated.Id], requestedIds); + } + + [Fact] + public async Task SpeakerMatchingFiltersInactiveNonAttendeeIdentitiesAndLimitsCandidates() + { + await using var fixture = await SpeakerIdentityFixture.CreateAsync(options => + { + options.MaxMatchCandidates = 2; + options.MatchIdentityActiveAge = TimeSpan.FromDays(365); + }); + var activeHigh = await fixture.AddIdentityAsync([], [1], "Active High", transcriptionCount: 50); + await fixture.AddIdentityAsync([], [2], "Stale High", transcriptionCount: 100, updatedAt: DateTimeOffset.UtcNow.AddYears(-2)); + var activeLow = await fixture.AddIdentityAsync([], [3], "Active Low", transcriptionCount: 5); + var staleAttendee = await fixture.AddIdentityAsync([], [4], "Former Employee", transcriptionCount: 1, updatedAt: DateTimeOffset.UtcNow.AddYears(-2), aliases: ["Former"]); + var service = fixture.CreateService(); + + await service.IdentifyKnownSpeakersAsync( + fixture.CreateRequest( + ["Former"], + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "hello")]), + CancellationToken.None); + + var requestedIds = fixture.Matcher.Requests.Single().Candidates.Select(candidate => candidate.IdentityId).ToList(); + Assert.Equal([staleAttendee.Id, activeHigh.Id], requestedIds); + Assert.DoesNotContain(activeLow.Id, requestedIds); + } + + [Fact] + public async Task SchemaUpgradeAddsLastModifiedColumnInitializedFromCreatedAt() + { + var dbPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", $"{Guid.NewGuid():N}.db"); + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); + var createdAt = DateTimeOffset.Parse("2026-05-01T10:00:00+00:00"); + await using var context = new SpeakerIdentityDbContext(new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbPath};Pooling=False") + .Options); + await context.Database.ExecuteSqlRawAsync( + """ + CREATE TABLE "SpeakerIdentities" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerIdentities" PRIMARY KEY AUTOINCREMENT, + "CanonicalName" TEXT NULL, + "TranscriptionCount" INTEGER NOT NULL, + "CreatedAt" TEXT NOT NULL + ); + """); + await context.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "SpeakerIdentities" ("CanonicalName", "TranscriptionCount", "CreatedAt") + VALUES ('Legacy Speaker', 3, {createdAt}); + """); + + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None); + + context.ChangeTracker.Clear(); + var saved = await context.SpeakerIdentities.SingleAsync(); + Assert.Equal(createdAt, saved.UpdatedAt); + await context.DisposeAsync(); + File.Delete(dbPath); + } + + private sealed class SpeakerIdentityFixture : IAsyncDisposable + { + private readonly string dbPath; + private readonly SpeakerIdentificationOptions options; + + private SpeakerIdentityFixture( + string dbPath, + SpeakerIdentityDbContext context, + SpeakerIdentificationOptions options) + { + this.dbPath = dbPath; + Context = context; + this.options = options; + } + + public SpeakerIdentityDbContext Context { get; } + + public FakeSpeakerIdentityMatcher Matcher { get; } = new(); + + public FakeSpeakerSnippetExtractor SnippetExtractor { get; } = new(); + + public static async Task CreateAsync( + Action? configureOptions = null) + { + var dbPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", $"{Guid.NewGuid():N}.db"); + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); + var options = new SpeakerIdentificationOptions + { + DatabasePath = dbPath, + MaxSnippetsPerSpeaker = 3, + MatchBatchSize = 6 + }; + configureOptions?.Invoke(options); + var context = new SpeakerIdentityDbContext(new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbPath};Pooling=False") + .Options); + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None); + return new SpeakerIdentityFixture(dbPath, context, options); + } + + public SpeakerIdentityService CreateService() + { + return new SpeakerIdentityService( + new TestSpeakerIdentityDbContextFactory(dbPath), + SnippetExtractor, + Matcher, + Options.Create(new MeetingAssistantOptions { SpeakerIdentification = options }), + NullLogger.Instance); + } + + public SpeakerIdentificationRequest CreateRequest( + IReadOnlyList attendees, + IReadOnlyList segments, + IReadOnlyList? samples = null) + { + return new SpeakerIdentificationRequest( + "test.wav", + MeetingNoteTemplate.Create( + "Test Meeting", + DateTimeOffset.Parse("2026-05-20T12:00:00+02:00"), + attendees: attendees, + transcriptPath: "transcript.md", + assistantContextPath: "context.md", + summaryPath: "summary.md"), + segments, + samples); + } + + public async Task AddIdentityAsync( + IReadOnlyList candidates, + byte[] snippet, + string? canonicalName = null, + int transcriptionCount = 0, + IReadOnlyList? aliases = null, + DateTimeOffset? updatedAt = null) + { + var timestamp = updatedAt ?? DateTimeOffset.UtcNow; + var identity = new SpeakerIdentity + { + CanonicalName = canonicalName, + TranscriptionCount = transcriptionCount, + CreatedAt = timestamp, + UpdatedAt = timestamp, + Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [], + CandidateNames = candidates.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList(), + Snippets = + [ + new SpeakerSnippet + { + WavBytes = snippet, + CreatedAt = timestamp.AddMinutes(-10) + } + ] + }; + Context.SpeakerIdentities.Add(identity); + await Context.SaveChangesAsync(); + return identity; + } + + public Task LoadIdentityAsync(int id) + { + Context.ChangeTracker.Clear(); + return Context.SpeakerIdentities + .Include(identity => identity.CandidateNames) + .Include(identity => identity.Aliases) + .Include(identity => identity.Snippets) + .SingleAsync(identity => identity.Id == id); + } + + public async ValueTask DisposeAsync() + { + await Context.DisposeAsync(); + if (File.Exists(dbPath)) + { + File.Delete(dbPath); + } + } + } + + private sealed class TestSpeakerIdentityDbContextFactory : IDbContextFactory + { + private readonly string dbPath; + + public TestSpeakerIdentityDbContextFactory(string dbPath) + { + this.dbPath = dbPath; + } + + public SpeakerIdentityDbContext CreateDbContext() + { + return new SpeakerIdentityDbContext(new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbPath};Pooling=False") + .Options); + } + } + + private sealed class FakeSpeakerSnippetExtractor : ISpeakerSnippetExtractor + { + public int ExtractCalls { get; private set; } + + public Task ExtractSnippetAsync( + string audioPath, + IReadOnlyList speakerSegments, + CancellationToken cancellationToken) + { + ExtractCalls++; + return Task.FromResult([7, 8, 9]); + } + } + + private sealed class FakeSpeakerIdentityMatcher : ISpeakerIdentityMatcher + { + private bool hasMatched; + + public int? MatchIdentityId { get; set; } + + public byte[]? LastUnknownSnippet { get; private set; } + + public List Requests { get; } = []; + + public Task MatchAsync( + SpeakerIdentityMatchRequest request, + CancellationToken cancellationToken) + { + LastUnknownSnippet = request.UnknownSnippet; + Requests.Add(request); + if (hasMatched || MatchIdentityId is null) + { + return Task.FromResult(null); + } + + hasMatched = true; + return Task.FromResult(new SpeakerIdentityMatch(MatchIdentityId.Value)); + } + } +} diff --git a/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs b/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs index 8f583d2..536b00c 100644 --- a/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs +++ b/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs @@ -68,6 +68,13 @@ public sealed class SpeechRecognitionPipelineHostedServiceTests } public Task InitializeAsync(CancellationToken cancellationToken) + { + return InitializeAsync(SpeechRecognitionPipelineOptions.Default, cancellationToken); + } + + public Task InitializeAsync( + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) { InitializeCount++; initialized.TrySetResult(); diff --git a/MeetingAssistant.Tests/VaultPathTests.cs b/MeetingAssistant.Tests/VaultPathTests.cs new file mode 100644 index 0000000..91e694e --- /dev/null +++ b/MeetingAssistant.Tests/VaultPathTests.cs @@ -0,0 +1,34 @@ +using MeetingAssistant; + +namespace MeetingAssistant.Tests; + +public sealed class VaultPathTests +{ + [Fact] + public void ResolveUsesVaultBaseFolderForRelativePaths() + { + var vault = new VaultOptions + { + BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta" + }; + + var resolved = VaultPath.Resolve(vault, @"Meetings\Transcripts"); + + Assert.Equal( + @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta\Meetings\Transcripts", + resolved); + } + + [Fact] + public void ResolveKeepsAbsolutePathsIndependentFromVaultBaseFolder() + { + var vault = new VaultOptions + { + BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta" + }; + + var resolved = VaultPath.Resolve(vault, @"C:\Other\Path"); + + Assert.Equal(@"C:\Other\Path", resolved); + } +} diff --git a/MeetingAssistant/MeetingAssistant.csproj b/MeetingAssistant/MeetingAssistant.csproj index 82abf5d..20d769a 100644 --- a/MeetingAssistant/MeetingAssistant.csproj +++ b/MeetingAssistant/MeetingAssistant.csproj @@ -13,6 +13,7 @@ + diff --git a/MeetingAssistant/MeetingAssistantOptions.cs b/MeetingAssistant/MeetingAssistantOptions.cs index b43c5db..347464e 100644 --- a/MeetingAssistant/MeetingAssistantOptions.cs +++ b/MeetingAssistant/MeetingAssistantOptions.cs @@ -14,6 +14,8 @@ public sealed class MeetingAssistantOptions public FunAsrOptions FunAsr { get; set; } = new(); + public SpeakerIdentificationOptions SpeakerIdentification { get; set; } = new(); + public AgentOptions Agent { get; set; } = new(); public ApiOptions Api { get; set; } = new(); @@ -26,22 +28,24 @@ public sealed class HotkeyOptions public sealed class VaultOptions { - public string TranscriptsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Transcripts"; + public string BaseFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Exxeta"; - public string MeetingNotesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Notes"; + public string TranscriptsFolder { get; set; } = @"Meetings\Transcripts"; - public string AssistantContextFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Assistant Context"; + public string MeetingNotesFolder { get; set; } = @"Meetings\Notes"; - public string SummariesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Summaries"; + public string AssistantContextFolder { get; set; } = @"Meetings\Assistant Context"; - public string ProjectsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Projects"; + public string SummariesFolder { get; set; } = @"Meetings\Summaries"; - public string DictationWordsPath { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\dictation-words.md"; + public string ProjectsFolder { get; set; } = @"Projects"; + + public string DictationWordsPath { get; set; } = @"Meetings\dictation-words.md"; } public sealed class RecordingOptions { - public string TranscriptionProvider { get; set; } = "funasr"; + public string TranscriptionProvider { get; set; } = "azure-speech"; public int SampleRate { get; set; } = 16000; @@ -49,6 +53,12 @@ public sealed class RecordingOptions public TimeSpan StopProcessingTimeout { get; set; } = TimeSpan.FromMinutes(10); + public TimeSpan MetadataLookupTimeout { get; set; } = TimeSpan.FromSeconds(10); + + public TimeSpan BackgroundMetadataLookupTimeout { get; set; } = TimeSpan.FromMinutes(1); + + public TimeSpan OpenMeetingNoteDelay { get; set; } = TimeSpan.FromSeconds(1); + public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings"; } @@ -92,11 +102,15 @@ public sealed class PyannoteDiarizationOptions public sealed class AzureSpeechOptions { - public string Endpoint { get; set; } = "https://germanywestcentral.api.cognitive.microsoft.com/"; + public string Endpoint { get; set; } = ""; public string Region { get; set; } = "germanywestcentral"; - public string Language { get; set; } = "en-US"; + public string Language { get; set; } = "auto"; + + public string[] AutoDetectLanguages { get; set; } = ["de-DE", "en-US"]; + + public string LanguageIdMode { get; set; } = "Continuous"; public string? Key { get; set; } @@ -105,6 +119,8 @@ public sealed class AzureSpeechOptions public TimeSpan RecognitionStopTimeout { get; set; } = TimeSpan.FromSeconds(10); public bool DiarizeIntermediateResults { get; set; } = true; + + public double PhraseListWeight { get; set; } = 1.5; } public enum PyannoteAnnotationSource @@ -192,6 +208,33 @@ public sealed class FunAsrDiarizationOptions public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30); } +public sealed class SpeakerIdentificationOptions +{ + public bool Enabled { get; set; } = true; + + public string DatabasePath { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\SpeakerIdentity\speaker-identities.db"; + + public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMinutes(2); + + public TimeSpan Interval { get; set; } = TimeSpan.FromMinutes(2); + + public int MatchBatchSize { get; set; } = 6; + + public int MaxMatchCandidates { get; set; } = 100; + + public TimeSpan MatchIdentityActiveAge { get; set; } = TimeSpan.FromDays(365); + + public int MaxSnippetsPerSpeaker { get; set; } = 3; + + public double SilenceBetweenSnippetsSeconds { get; set; } = 1; + + public TimeSpan LiveSampleBufferDuration { get; set; } = TimeSpan.FromMinutes(10); + + public TimeSpan MergeRecentIdentityAge { get; set; } = TimeSpan.FromDays(14); + + public AzureSpeechOptions AzureSpeech { get; set; } = new(); +} + public sealed class AgentOptions { public string Endpoint { get; set; } = "https://litellm.schweigert.cloud"; @@ -219,6 +262,8 @@ public sealed class AgentOptions public double CompactionRemainingRatio { get; set; } = 0.10; public string ResponsesCompactPath { get; set; } = "responses/compact"; + + public string? InitialPrompt { get; set; } } public sealed class ApiOptions diff --git a/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs b/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs index a34dc3f..302b119 100644 --- a/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs +++ b/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs @@ -8,6 +8,12 @@ public interface IMeetingArtifactStore DateTimeOffset? scheduledEnd, CancellationToken cancellationToken); + Task UpdateAssistantContextMetadataAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken); + Task UpdateAssistantContextStateAsync( MeetingSessionArtifacts artifacts, AssistantContextState state, diff --git a/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs b/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs index a7890f3..b4bc51f 100644 --- a/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs +++ b/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs @@ -7,12 +7,29 @@ public interface IMeetingMetadataProvider CancellationToken cancellationToken); } +public interface IMeetingMetadataDiagnosticProvider +{ + Task DiagnoseCurrentMeetingAsync( + DateTimeOffset startedAt, + TimeSpan timeout, + CancellationToken cancellationToken); +} + public sealed record MeetingMetadata( string Title, IReadOnlyList Attendees, string Agenda, DateTimeOffset? ScheduledEnd = null); +public sealed record MeetingMetadataDiagnosticResult( + string Provider, + DateTimeOffset StartedAt, + bool Succeeded, + bool TimedOut, + long ElapsedMilliseconds, + MeetingMetadata? Metadata, + string? Error); + public sealed class NoopMeetingMetadataProvider : IMeetingMetadataProvider { public Task GetCurrentMeetingAsync( diff --git a/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs b/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs index f4a5c15..4086de8 100644 --- a/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs +++ b/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs @@ -58,6 +58,28 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore ToYamlState(state)); } + public async Task UpdateAssistantContextMetadataAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + var existing = File.Exists(artifacts.AssistantContextPath) + ? MarkdownDocumentParser.SplitOptional(await File.ReadAllTextAsync(artifacts.AssistantContextPath, cancellationToken)) + : new MarkdownDocument(false, "", "## Live Context" + Environment.NewLine); + var existingFrontmatter = ReadFrontmatter(existing.Frontmatter); + var state = ParseState(existingFrontmatter.State); + + await File.WriteAllTextAsync( + artifacts.AssistantContextPath, + Render(artifacts, state, agenda, scheduledEnd, existing.Body), + cancellationToken); + logger.LogInformation( + "Updated assistant context note {AssistantContextPath} calendar metadata", + artifacts.AssistantContextPath); + } + private static string Render( MeetingSessionArtifacts artifacts, AssistantContextState state, @@ -100,6 +122,9 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore private sealed class AssistantContextFrontmatterYaml { + [YamlMember(Alias = "state")] + public string? State { get; set; } + [YamlMember(Alias = "agenda")] public string? Agenda { get; set; } @@ -107,6 +132,19 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore public string? ScheduledEnd { get; set; } } + private static AssistantContextState ParseState(string? value) + { + return value?.Trim().ToLowerInvariant() switch + { + "transcribing" => AssistantContextState.Transcribing, + "speaker recognition" => AssistantContextState.SpeakerRecognition, + "summarizing" => AssistantContextState.Summarizing, + "finished" => AssistantContextState.Finished, + "error" => AssistantContextState.Error, + _ => AssistantContextState.Transcribing + }; + } + private static string ToYamlState(AssistantContextState state) { return state switch diff --git a/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs b/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs index 5f3e57f..e9d862a 100644 --- a/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs +++ b/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs @@ -23,7 +23,7 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore public async Task SaveAsync(MeetingNote note, CancellationToken cancellationToken) { - var folder = VaultPath.Resolve(options.Vault.MeetingNotesFolder); + var folder = VaultPath.Resolve(options.Vault, options.Vault.MeetingNotesFolder); Directory.CreateDirectory(folder); var path = string.IsNullOrWhiteSpace(note.Path) diff --git a/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs b/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs index f5ebccb..89c98ef 100644 --- a/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs +++ b/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs @@ -1,18 +1,28 @@ using System.Diagnostics; +using Microsoft.Extensions.Options; namespace MeetingAssistant.MeetingNotes; public sealed class ObsidianMeetingNoteOpener : IMeetingNoteOpener { + private readonly MeetingAssistantOptions options; private readonly ILogger logger; - public ObsidianMeetingNoteOpener(ILogger logger) + public ObsidianMeetingNoteOpener( + IOptions options, + ILogger logger) { + this.options = options.Value; this.logger = logger; } - public Task OpenAsync(string notePath, CancellationToken cancellationToken) + public async Task OpenAsync(string notePath, CancellationToken cancellationToken) { + if (options.Recording.OpenMeetingNoteDelay > TimeSpan.Zero) + { + await Task.Delay(options.Recording.OpenMeetingNoteDelay, cancellationToken); + } + var uri = $"obsidian://open?path={Uri.EscapeDataString(Path.GetFullPath(notePath))}"; Process.Start(new ProcessStartInfo { @@ -20,7 +30,5 @@ public sealed class ObsidianMeetingNoteOpener : IMeetingNoteOpener UseShellExecute = true }); logger.LogInformation("Opened meeting note in Obsidian via {ObsidianUri}", uri); - - return Task.CompletedTask; } } diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs index 6230681..251d9c2 100644 --- a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs +++ b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs @@ -1,8 +1,10 @@ using System.Runtime.InteropServices; +using System.Diagnostics; +using System.Collections; namespace MeetingAssistant.MeetingNotes; -public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProvider +public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProvider, IMeetingMetadataDiagnosticProvider { private const int OutlookCalendarFolder = 9; private readonly ILogger logger; @@ -19,7 +21,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv cancellationToken.ThrowIfCancellationRequested(); try { - return Task.FromResult(GetCurrentMeeting(startedAt)); + return RunStaAsync(() => GetCurrentMeeting(startedAt), cancellationToken); } catch (Exception exception) when (exception is not OperationCanceledException) { @@ -28,13 +30,66 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv } } + public async Task DiagnoseCurrentMeetingAsync( + DateTimeOffset startedAt, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + try + { + var lookup = RunStaAsync(() => GetCurrentMeeting(startedAt), cancellationToken); + var metadata = timeout > TimeSpan.Zero + ? await lookup.WaitAsync(timeout, cancellationToken) + : await lookup.WaitAsync(cancellationToken); + stopwatch.Stop(); + return new MeetingMetadataDiagnosticResult( + nameof(OutlookClassicMeetingMetadataProvider), + startedAt, + metadata is not null, + false, + stopwatch.ElapsedMilliseconds, + metadata, + metadata is null ? "No matching Teams appointment was found." : null); + } + catch (TimeoutException exception) + { + stopwatch.Stop(); + return new MeetingMetadataDiagnosticResult( + nameof(OutlookClassicMeetingMetadataProvider), + startedAt, + false, + true, + stopwatch.ElapsedMilliseconds, + null, + exception.Message); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + stopwatch.Stop(); + return new MeetingMetadataDiagnosticResult( + nameof(OutlookClassicMeetingMetadataProvider), + startedAt, + false, + false, + stopwatch.ElapsedMilliseconds, + null, + exception.Message); + } + } + private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt) { + EnsureOutlookRunning(); + object? application = null; object? session = null; object? calendar = null; object? items = null; - object? restrictedItems = null; try { @@ -53,31 +108,50 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv session = GetProperty(application, "Session"); calendar = Invoke(session!, "GetDefaultFolder", OutlookCalendarFolder); items = GetProperty(calendar!, "Items"); - SetProperty(items!, "IncludeRecurrences", true); Invoke(items!, "Sort", "[Start]"); + SetProperty(items!, "IncludeRecurrences", true); var startedLocal = startedAt.LocalDateTime; - var windowStart = startedLocal.AddMinutes(-1); + var windowStart = startedLocal.AddHours(-4); var windowEnd = startedLocal.AddMinutes(5); - var filter = - $"[Start] <= '{windowEnd:g}' AND [End] >= '{windowStart:g}'"; - restrictedItems = Invoke(items!, "Restrict", filter); var candidates = new List(); - var count = Convert.ToInt32(GetProperty(restrictedItems!, "Count")); - for (var index = 1; index <= count; index++) + foreach (var item in EnumerateItems(items!)) { - var appointment = Invoke(restrictedItems!, "Item", index); - if (appointment is not null && IsTeamsAppointment(appointment)) + if (item is null) { - candidates.Add(new OutlookAppointmentCandidate( - appointment, - Convert.ToDateTime(GetProperty(appointment, "Start")), - Convert.ToDateTime(GetProperty(appointment, "End")))); + continue; } - else + + var keepAppointment = false; + try { - ReleaseComObject(appointment); + if (!IsAppointment(item)) + { + continue; + } + + var start = Convert.ToDateTime(GetProperty(item, "Start")); + if (start > windowEnd) + { + break; + } + + var end = Convert.ToDateTime(GetProperty(item, "End")); + if (end < windowStart || !IsTeamsAppointment(item)) + { + continue; + } + + candidates.Add(new OutlookAppointmentCandidate(item, start, end)); + keepAppointment = true; + } + finally + { + if (!keepAppointment) + { + ReleaseComObject(item); + } } } @@ -118,7 +192,6 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv } finally { - ReleaseComObject(restrictedItems); ReleaseComObject(items); ReleaseComObject(calendar); ReleaseComObject(session); @@ -126,6 +199,83 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv } } + private static IEnumerable EnumerateItems(object items) + { + if (items is IEnumerable enumerable) + { + foreach (var item in enumerable) + { + yield return item; + } + } + } + + private static bool IsAppointment(object item) + { + try + { + return Convert.ToInt32(GetProperty(item, "Class")) == 26; + } + catch + { + return false; + } + } + + private void EnsureOutlookRunning() + { + if (Process.GetProcessesByName("OUTLOOK").Length > 0) + { + return; + } + + try + { + Process.Start(new ProcessStartInfo + { + FileName = "outlook.exe", + UseShellExecute = true + }); + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + catch (Exception exception) + { + logger.LogDebug(exception, "Could not start Outlook Classic before metadata lookup"); + } + } + + private static Task RunStaAsync( + Func action, + CancellationToken cancellationToken) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var thread = new Thread(() => + { + try + { + completion.TrySetResult(action()); + } + catch (Exception exception) + { + completion.TrySetException(exception); + } + }) + { + IsBackground = true, + Name = "Meeting Assistant Outlook COM Lookup" + }; + thread.SetApartmentState(ApartmentState.STA); + + var registration = cancellationToken.Register(() => completion.TrySetCanceled(cancellationToken)); + _ = completion.Task.ContinueWith( + _ => registration.Dispose(), + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + thread.Start(); + return completion.Task; + } + internal static string ExtractAgenda(string body) { if (string.IsNullOrWhiteSpace(body)) @@ -215,10 +365,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv ReleaseComObject(recipients); } - return attendees - .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToList(); + return NormalizeAttendees(attendees); } private static DateTimeOffset ToLocalOffset(DateTime value) @@ -226,6 +373,58 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local)); } + internal static IReadOnlyList NormalizeAttendees(IEnumerable attendees) + { + return attendees + .Select(NormalizeAttendee) + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase) + .Select(group => group + .OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal)) + .ThenBy(attendee => attendee.Length) + .First()) + .ToList(); + } + + private static string NormalizeAttendee(string attendee) + { + var normalized = attendee.Trim(); + var mailtoPrefix = "mailto:"; + normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase); + while (normalized.Contains(" ", StringComparison.Ordinal)) + { + normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); + } + + var emailStart = normalized.IndexOf('<', StringComparison.Ordinal); + var emailEnd = normalized.LastIndexOf('>'); + if (emailStart > 0 && emailEnd > emailStart) + { + var name = normalized[..emailStart].Trim(); + var email = normalized[(emailStart + 1)..emailEnd].Trim(); + if (name.Equals(email, StringComparison.OrdinalIgnoreCase)) + { + return email; + } + + return $"{name} <{email}>"; + } + + return normalized; + } + + private static string GetAttendeeDeduplicationKey(string attendee) + { + var emailStart = attendee.IndexOf('<', StringComparison.Ordinal); + var emailEnd = attendee.LastIndexOf('>'); + if (emailStart > 0 && emailEnd > emailStart) + { + return attendee[(emailStart + 1)..emailEnd].Trim(); + } + + return attendee.Trim(); + } + private static string FormatRecipient(object recipient) { var name = Convert.ToString(GetProperty(recipient, "Name"))?.Trim() ?? ""; diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index 23d0de7..9de2c51 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -2,8 +2,10 @@ using MeetingAssistant; using MeetingAssistant.Hotkeys; using MeetingAssistant.MeetingNotes; using MeetingAssistant.Recording; +using MeetingAssistant.Speakers; using MeetingAssistant.Summary; using MeetingAssistant.Transcription; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; var builder = WebApplication.CreateBuilder(args); @@ -25,8 +27,22 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddDbContextFactory((services, dbOptions) => +{ + var appOptions = services.GetRequiredService>().Value; + var databasePath = VaultPath.Resolve(appOptions.SpeakerIdentification.DatabasePath); + Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!); + dbOptions.UseSqlite($"Data Source={databasePath}"); +}); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -41,6 +57,7 @@ builder.Services.AddTransient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHostedService(); #if WINDOWS builder.Services.AddHostedService(); @@ -55,6 +72,48 @@ app.MapGet("/health", () => Results.Ok(new })); app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus)); +app.MapGet("/diagnostics/outlook/current-meeting", async ( + IMeetingMetadataProvider metadataProvider, + IOptions options, + DateTimeOffset? startedAt, + double? timeoutSeconds, + CancellationToken cancellationToken) => +{ + var lookupStartedAt = startedAt ?? DateTimeOffset.Now; + var timeout = timeoutSeconds is > 0 + ? TimeSpan.FromSeconds(timeoutSeconds.Value) + : options.Value.Recording.MetadataLookupTimeout; + if (metadataProvider is IMeetingMetadataDiagnosticProvider diagnostics) + { + return Results.Ok(await diagnostics.DiagnoseCurrentMeetingAsync( + lookupStartedAt, + timeout, + cancellationToken)); + } + + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + var metadata = await metadataProvider.GetCurrentMeetingAsync(lookupStartedAt, cancellationToken); + stopwatch.Stop(); + return Results.Ok(new MeetingMetadataDiagnosticResult( + metadataProvider.GetType().Name, + lookupStartedAt, + metadata is not null, + false, + stopwatch.ElapsedMilliseconds, + metadata, + metadata is null ? "No meeting metadata provider result was available." : null)); +}); +app.MapPost("/diagnostics/speaker-identities/merge", async ( + SpeakerIdentityMergeDiagnosticRequest request, + ISpeakerIdentityMergeService mergeService, + IOptions options, + CancellationToken cancellationToken) => +{ + var recentAge = request.RecentDays is > 0 + ? TimeSpan.FromDays(request.RecentDays.Value) + : options.Value.SpeakerIdentification.MergeRecentIdentityAge; + return Results.Ok(await mergeService.MergeRecentIdentitiesAsync(recentAge, cancellationToken)); +}); app.MapPost("/recording/toggle", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => Results.Ok(await coordinator.ToggleAsync(cancellationToken))); app.MapPost("/recording/start", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => @@ -189,3 +248,5 @@ public partial class Program; public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null); public sealed record SummaryRetryRequest(string SummaryPath); + +public sealed record SpeakerIdentityMergeDiagnosticRequest(double? RecentDays = null); diff --git a/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs index 2945448..baf9478 100644 --- a/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs +++ b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs @@ -46,6 +46,10 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource await foreach (var sourceChunk in channel.Reader.ReadAllAsync(cancellationToken)) { + logger.LogDebug( + "Captured {ByteCount} byte(s) from {SourceKind}", + sourceChunk.Chunk.Pcm.Length, + sourceChunk.Kind); if (sourceChunk.Kind == AudioSourceKind.Microphone) { pendingMicrophone = sourceChunk.Chunk; @@ -106,9 +110,21 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource ChannelWriter writer, CancellationToken cancellationToken) { - await foreach (var chunk in source.CaptureAsync(cancellationToken)) + try { - await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken); + await foreach (var chunk in source.CaptureAsync(cancellationToken)) + { + await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + writer.TryComplete(new InvalidOperationException($"{kind} audio capture failed.", exception)); + throw; } } diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs index b78b682..d9091aa 100644 --- a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -1,5 +1,7 @@ using System.Threading.Channels; +using System.Collections.Concurrent; using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Speakers; using MeetingAssistant.Summary; using MeetingAssistant.Transcription; using Microsoft.Extensions.Options; @@ -17,6 +19,8 @@ public sealed class MeetingRecordingCoordinator private readonly IMeetingMetadataProvider meetingMetadataProvider; private readonly IRecordedAudioStore recordedAudioStore; private readonly IMeetingSummaryPipeline summaryPipeline; + private readonly ISpeakerIdentificationService? speakerIdentificationService; + private readonly IDictationWordStore dictationWordStore; private readonly MeetingAssistantOptions options; private readonly ILogger logger; private readonly SemaphoreSlim gate = new(1, 1); @@ -36,7 +40,9 @@ public sealed class MeetingRecordingCoordinator IMeetingSummaryPipeline summaryPipeline, IOptions options, ILogger logger, - IMeetingMetadataProvider? meetingMetadataProvider = null) + IMeetingMetadataProvider? meetingMetadataProvider = null, + ISpeakerIdentificationService? speakerIdentificationService = null, + IDictationWordStore? dictationWordStore = null) { this.audioSource = audioSource; this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory; @@ -47,6 +53,8 @@ public sealed class MeetingRecordingCoordinator this.meetingMetadataProvider = meetingMetadataProvider ?? new NoopMeetingMetadataProvider(); this.recordedAudioStore = recordedAudioStore; this.summaryPipeline = summaryPipeline; + this.dictationWordStore = dictationWordStore ?? new EmptyDictationWordStore(); + this.speakerIdentificationService = speakerIdentificationService; this.options = options.Value; this.logger = logger; } @@ -84,18 +92,11 @@ public sealed class MeetingRecordingCoordinator var startedAt = DateTimeOffset.Now; var assistantContextPath = GetAssistantContextPath(startedAt); var summaryPath = GetSummaryPath(startedAt); - var meetingMetadata = await meetingMetadataProvider.GetCurrentMeetingAsync(startedAt, cancellationToken); - var title = string.IsNullOrWhiteSpace(meetingMetadata?.Title) - ? $"Meeting {startedAt:yyyy-MM-dd HH:mm}" - : meetingMetadata.Title; - var attendees = meetingMetadata?.Attendees ?? []; - var agenda = meetingMetadata?.Agenda ?? ""; - var scheduledEnd = meetingMetadata?.ScheduledEnd; currentMeetingNote = await meetingNoteStore.SaveAsync( MeetingNoteTemplate.Create( - title: title, + title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}", startTime: startedAt, - attendees: attendees, + attendees: [], transcriptPath: currentSession.TranscriptPath, assistantContextPath: assistantContextPath, summaryPath: summaryPath), @@ -105,23 +106,42 @@ public sealed class MeetingRecordingCoordinator currentSession.TranscriptPath, assistantContextPath, summaryPath); - await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, agenda, scheduledEnd, cancellationToken); + await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, "", null, cancellationToken); + _ = Task.Run( + () => ApplyMeetingMetadataWhenAvailableAsync(startedAt, currentArtifacts, currentMeetingNote.Path), + CancellationToken.None); await transcriptStore.UpdateMetadataAsync( currentSession, currentArtifacts, currentMeetingNote, cancellationToken); - await meetingNoteOpener.OpenAsync(currentMeetingNote.Path, cancellationToken); + await OpenMeetingNoteAsync(currentMeetingNote.Path, cancellationToken); var captureCancellation = new CancellationTokenSource(); var transcriptionCancellation = new CancellationTokenSource(); var pipeline = speechRecognitionPipelineFactory.Create(); - var run = new RecordingRun(captureCancellation, transcriptionCancellation, recordedAudio, pipeline); + var pipelineOptions = await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken); + var run = new RecordingRun( + captureCancellation, + transcriptionCancellation, + recordedAudio, + pipeline, + pipelineOptions, + options.SpeakerIdentification.LiveSampleBufferDuration, + options.SpeakerIdentification.MaxSnippetsPerSpeaker); run.Task = Task.Run(() => RecordAsync(currentSession, run), CancellationToken.None); currentRun = run; logger.LogInformation("Meeting recording started"); return CurrentStatus; } + catch + { + currentRun = null; + currentSession = null; + currentMeetingNote = null; + currentArtifacts = null; + throw; + } finally { gate.Release(); @@ -171,12 +191,15 @@ public sealed class MeetingRecordingCoordinator TranscriptSession session, RecordingRun run) { - await run.Pipeline.InitializeAsync(run.TranscriptionCancellation); + await run.Pipeline.InitializeAsync(run.PipelineOptions, run.TranscriptionCancellation); var liveTranscriptTask = Task.Run( - () => StoreLiveTranscriptAsync(session, run.Pipeline, run.TranscriptionCancellation), + () => StoreLiveTranscriptAsync(session, run, run.TranscriptionCancellation), CancellationToken.None); var captureTask = Task.Run( - () => CaptureAudioAsync(run.Pipeline, run.RecordedAudio, run.CaptureCancellation), + () => CaptureAudioAsync(run), + CancellationToken.None); + var liveIdentificationTask = Task.Run( + () => IdentifyLiveSpeakersAsync(session, run, run.TranscriptionCancellation), CancellationToken.None); try @@ -184,6 +207,9 @@ public sealed class MeetingRecordingCoordinator await captureTask; await run.Pipeline.CompleteAsync(run.TranscriptionCancellation); await liveTranscriptTask; + run.CancelLiveIdentification(); + await liveIdentificationTask; + await run.RecordedAudio.DisposeAsync(); var artifacts = currentArtifacts; if (artifacts is not null && HasConfiguredFinalizer()) { @@ -193,7 +219,7 @@ public sealed class MeetingRecordingCoordinator run.TranscriptionCancellation); } - await RewriteTranscriptWithFinishedSegmentsAsync(session, run.Pipeline, run.RecordedAudio.AudioPath, run.TranscriptionCancellation); + await RewriteTranscriptWithFinishedSegmentsAsync(session, run.Pipeline, run, run.TranscriptionCancellation); var completedMeetingNote = await CompleteMeetingNoteAsync(run.TranscriptionCancellation); if (artifacts is not null && completedMeetingNote is not null) { @@ -222,54 +248,268 @@ public sealed class MeetingRecordingCoordinator } finally { + run.CancelLiveIdentification(); await run.RecordedAudio.DeleteAsync(CancellationToken.None); await run.Pipeline.DisposeAsync(); await CompleteRunAsync(run); } } - private async Task CaptureAudioAsync( - ISpeechRecognitionPipeline pipeline, - IRecordedAudioSink recordedAudio, - CancellationToken cancellationToken) + private async Task CaptureAudioAsync(RecordingRun run) { + var chunkCount = 0; + long byteCount = 0; try { - await foreach (var chunk in audioSource.CaptureAsync(cancellationToken)) + await foreach (var chunk in audioSource.CaptureAsync(run.CaptureCancellation)) { - await recordedAudio.AppendAsync(chunk, cancellationToken); - await pipeline.WriteAsync(chunk, cancellationToken); + chunkCount++; + byteCount += chunk.Pcm.Length; + if (chunkCount == 1) + { + logger.LogInformation( + "Captured first mixed audio chunk: {ByteCount} bytes at {SampleRate} Hz/{Channels} channel(s)", + chunk.Pcm.Length, + chunk.SampleRate, + chunk.Channels); + } + + run.AppendAudio(chunk); + await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation); + await run.Pipeline.WriteAsync(chunk, run.CaptureCancellation); } - await recordedAudio.CompleteAsync(cancellationToken); - await pipeline.CompleteAsync(cancellationToken); + logger.LogInformation( + "Audio capture completed after {ChunkCount} chunk(s), {ByteCount} byte(s)", + chunkCount, + byteCount); + await run.RecordedAudio.CompleteAsync(run.CaptureCancellation); + await run.Pipeline.CompleteAsync(run.CaptureCancellation); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) when (run.CaptureCancellation.IsCancellationRequested) { - await recordedAudio.CompleteAsync(CancellationToken.None); - await pipeline.CompleteAsync(CancellationToken.None); + logger.LogInformation( + "Audio capture stopping after {ChunkCount} chunk(s), {ByteCount} byte(s)", + chunkCount, + byteCount); + await run.RecordedAudio.CompleteAsync(CancellationToken.None); + await run.Pipeline.CompleteAsync(CancellationToken.None); } } private async Task StoreLiveTranscriptAsync( TranscriptSession session, - ISpeechRecognitionPipeline pipeline, + RecordingRun run, CancellationToken cancellationToken) { - await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken)) + await foreach (var segment in run.Pipeline.ReadLiveTranscriptAsync(cancellationToken)) { - await transcriptStore.AppendAsync(session, segment, cancellationToken); + run.TryAddSpeakerSample(segment); + var relabeledSegment = run.Relabel(segment); + run.AddLiveSegment(relabeledSegment); + await transcriptStore.AppendAsync(session, relabeledSegment, cancellationToken); + } + } + + private async Task IdentifyLiveSpeakersAsync( + TranscriptSession session, + RecordingRun run, + CancellationToken cancellationToken) + { + if (speakerIdentificationService is null || !options.SpeakerIdentification.Enabled) + { + return; + } + + var initialDelay = options.SpeakerIdentification.InitialDelay; + var interval = options.SpeakerIdentification.Interval; + try + { + if (initialDelay > TimeSpan.Zero) + { + await Task.Delay(initialDelay, run.LiveIdentificationCancellation); + } + + while (!run.LiveIdentificationCancellation.IsCancellationRequested) + { + await IdentifyLiveSpeakersOnceAsync(session, run, run.LiveIdentificationCancellation); + await Task.Delay(interval > TimeSpan.Zero ? interval : TimeSpan.FromMinutes(1), run.LiveIdentificationCancellation); + } + } + catch (OperationCanceledException) when (run.LiveIdentificationCancellation.IsCancellationRequested) + { + } + } + + private async Task IdentifyLiveSpeakersOnceAsync( + TranscriptSession session, + RecordingRun run, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) + { + return; + } + + var segments = run.GetLiveSegmentsSnapshot(); + var samples = run.GetSpeakerSamplesSnapshot(); + if (segments.Count == 0 || samples.Count == 0) + { + return; + } + + try + { + var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote); + if (checkpoint is null) + { + return; + } + + var result = await speakerIdentificationService!.IdentifyKnownSpeakersAsync( + new SpeakerIdentificationRequest("", meetingNote, segments, samples), + cancellationToken); + run.MarkLiveIdentificationAttempted(checkpoint); + await AddIdentifiedSpeakersToMeetingAttendeesAsync(result.AttendeeMatches, cancellationToken); + if (run.AddSpeakerMappings(result.SpeakerMappings)) + { + await transcriptStore.ReplaceAsync( + session, + run.GetRelabeledLiveSegmentsSnapshot(), + cancellationToken); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogDebug(exception, "Live speaker identity matching skipped for this interval"); + } + } + + private async Task ApplyMeetingMetadataWhenAvailableAsync( + DateTimeOffset startedAt, + MeetingSessionArtifacts artifacts, + string meetingNotePath) + { + var metadata = await GetMeetingMetadataAsync( + startedAt, + options.Recording.BackgroundMetadataLookupTimeout, + CancellationToken.None); + if (metadata is null) + { + return; + } + + await gate.WaitAsync(CancellationToken.None); + try + { + var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, CancellationToken.None); + ApplyMeetingMetadata(meetingNote, metadata); + meetingNote = await meetingNoteStore.SaveAsync(meetingNote, CancellationToken.None); + if (currentMeetingNote?.Path == meetingNote.Path) + { + currentMeetingNote = meetingNote; + } + + await meetingArtifactStore.UpdateAssistantContextMetadataAsync( + artifacts, + metadata.Agenda, + metadata.ScheduledEnd, + CancellationToken.None); + logger.LogInformation( + "Applied Outlook meeting metadata to {MeetingNotePath}", + meetingNotePath); + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not apply Outlook meeting metadata to {MeetingNotePath}", + meetingNotePath); + } + finally + { + gate.Release(); + } + } + + private static void ApplyMeetingMetadata(MeetingNote meetingNote, MeetingMetadata metadata) + { + if (!string.IsNullOrWhiteSpace(metadata.Title)) + { + meetingNote.Frontmatter.Title = metadata.Title; + } + + if (metadata.Attendees.Count > 0) + { + meetingNote.Frontmatter.Attendees = metadata.Attendees.ToList(); + } + } + + private async Task GetMeetingMetadataAsync( + DateTimeOffset startedAt, + TimeSpan timeout, + CancellationToken cancellationToken) + { + try + { + var lookup = meetingMetadataProvider.GetCurrentMeetingAsync(startedAt, cancellationToken); + return timeout > TimeSpan.Zero + ? await lookup.WaitAsync(timeout, cancellationToken) + : await lookup.WaitAsync(cancellationToken); + } + catch (TimeoutException exception) + { + logger.LogWarning( + exception, + "Meeting metadata lookup timed out after {Timeout}; using a default meeting note", + timeout); + return null; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Meeting metadata lookup failed; using a default meeting note"); + return null; + } + } + + private async Task OpenMeetingNoteAsync( + string meetingNotePath, + CancellationToken cancellationToken) + { + try + { + await meetingNoteOpener.OpenAsync(meetingNotePath, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not open meeting note {MeetingNotePath}; recording will continue", + meetingNotePath); } } private async Task RewriteTranscriptWithFinishedSegmentsAsync( TranscriptSession session, ISpeechRecognitionPipeline pipeline, - string audioPath, + RecordingRun run, CancellationToken cancellationToken) { var finishedSegments = await pipeline.ReadFinishedTranscriptAsync( - audioPath, + run.RecordedAudio.AudioPath, await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken), cancellationToken); if (finishedSegments.Count == 0) @@ -277,9 +517,98 @@ public sealed class MeetingRecordingCoordinator return; } + finishedSegments = finishedSegments + .Select(run.Relabel) + .ToList(); + finishedSegments = await IdentifySpeakersAsync( + run.RecordedAudio.AudioPath, + finishedSegments, + run.GetSpeakerSamplesSnapshot(), + cancellationToken); await transcriptStore.ReplaceAsync(session, finishedSegments, cancellationToken); } + private async Task> IdentifySpeakersAsync( + string audioPath, + IReadOnlyList finishedSegments, + IReadOnlyList samples, + CancellationToken cancellationToken) + { + if (speakerIdentificationService is null || string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) + { + return finishedSegments; + } + + try + { + var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + var result = await speakerIdentificationService.ProcessFinishedTranscriptAsync( + new SpeakerIdentificationRequest(audioPath, meetingNote, finishedSegments, samples), + cancellationToken); + await AddIdentifiedSpeakersToMeetingAttendeesAsync(result.AttendeeMatches, cancellationToken); + return result.Segments; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Speaker identity learning failed; keeping ASR speaker labels"); + return finishedSegments; + } + } + + private async Task AddIdentifiedSpeakersToMeetingAttendeesAsync( + IReadOnlyList? matches, + CancellationToken cancellationToken) + { + if (matches is null || matches.Count == 0 || string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) + { + return; + } + + var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + var existingNames = meetingNote.Frontmatter.Attendees + .Select(NormalizeAttendeeName) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var changed = false; + + foreach (var match in matches) + { + if (string.IsNullOrWhiteSpace(match.DisplayName)) + { + continue; + } + + var acceptedNames = match.AcceptedNames + .Append(match.DisplayName) + .Select(NormalizeAttendeeName) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .ToList(); + if (acceptedNames.Any(existingNames.Contains)) + { + continue; + } + + var displayName = match.DisplayName.Trim(); + meetingNote.Frontmatter.Attendees.Add(displayName); + existingNames.Add(NormalizeAttendeeName(displayName)); + changed = true; + } + + if (!changed) + { + return; + } + + currentMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, cancellationToken); + logger.LogInformation( + "Added identified speaker(s) to meeting note attendees for {MeetingNotePath}", + currentMeetingNote.Path); + } + private async Task CompleteMeetingNoteAsync(CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) @@ -293,6 +622,13 @@ public sealed class MeetingRecordingCoordinator return currentMeetingNote; } + private static string NormalizeAttendeeName(string attendee) + { + var trimmed = attendee.Trim(); + var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal); + return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed; + } + private async Task RunSummaryAsync( MeetingSessionArtifacts artifacts, CancellationToken cancellationToken) @@ -336,31 +672,36 @@ public sealed class MeetingRecordingCoordinator private async Task BuildSpeechRecognitionPipelineOptionsAsync( CancellationToken cancellationToken) { + var dictationWords = await dictationWordStore.ReadWordsAsync(cancellationToken); if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) { - return SpeechRecognitionPipelineOptions.Default; + return new SpeechRecognitionPipelineOptions(DictationWords: dictationWords); } var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee)); return attendeeCount > 1 - ? new SpeechRecognitionPipelineOptions(attendeeCount) - : SpeechRecognitionPipelineOptions.Default; + ? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords) + : new SpeechRecognitionPipelineOptions(DictationWords: dictationWords); } private string GetAssistantContextPath(DateTimeOffset startedAt) { - return GetMeetingArtifactPath(options.Vault.AssistantContextFolder, startedAt, "assistant-context"); + return GetMeetingArtifactPath(options.Vault, options.Vault.AssistantContextFolder, startedAt, "assistant-context"); } private string GetSummaryPath(DateTimeOffset startedAt) { - return GetMeetingArtifactPath(options.Vault.SummariesFolder, startedAt, "summary"); + return GetMeetingArtifactPath(options.Vault, options.Vault.SummariesFolder, startedAt, "summary"); } - private static string GetMeetingArtifactPath(string configuredFolder, DateTimeOffset startedAt, string artifactName) + private static string GetMeetingArtifactPath( + VaultOptions vault, + string configuredFolder, + DateTimeOffset startedAt, + string artifactName) { - var folder = VaultPath.Resolve(configuredFolder); + var folder = VaultPath.Resolve(vault, configuredFolder); return Path.Combine(folder, $"{startedAt:yyyyMMdd-HHmmss}-{artifactName}.md"); } @@ -389,31 +730,49 @@ public sealed class MeetingRecordingCoordinator private sealed class RecordingRun : IDisposable { private int completed; + private readonly object liveSegmentsGate = new(); + private readonly object liveIdentificationGate = new(); + private readonly List liveSegments = []; + private readonly ConcurrentDictionary speakerMappings = new(StringComparer.OrdinalIgnoreCase); + private readonly SpeakerAudioSampleCollector speakerSampleCollector; + private LiveIdentificationCheckpoint? lastLiveIdentificationCheckpoint; public RecordingRun( CancellationTokenSource captureCancellation, CancellationTokenSource transcriptionCancellation, IRecordedAudioSink recordedAudio, - ISpeechRecognitionPipeline pipeline) + ISpeechRecognitionPipeline pipeline, + SpeechRecognitionPipelineOptions pipelineOptions, + TimeSpan liveSampleBufferDuration, + int maxSpeakerSamples) { CaptureCancellationSource = captureCancellation; TranscriptionCancellationSource = transcriptionCancellation; + LiveIdentificationCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(transcriptionCancellation.Token); RecordedAudio = recordedAudio; Pipeline = pipeline; + PipelineOptions = pipelineOptions; + speakerSampleCollector = new SpeakerAudioSampleCollector(liveSampleBufferDuration, maxSpeakerSamples); } public CancellationTokenSource CaptureCancellationSource { get; } public CancellationTokenSource TranscriptionCancellationSource { get; } + public CancellationTokenSource LiveIdentificationCancellationSource { get; } + public IRecordedAudioSink RecordedAudio { get; } public ISpeechRecognitionPipeline Pipeline { get; } + public SpeechRecognitionPipelineOptions PipelineOptions { get; } + public CancellationToken CaptureCancellation => CaptureCancellationSource.Token; public CancellationToken TranscriptionCancellation => TranscriptionCancellationSource.Token; + public CancellationToken LiveIdentificationCancellation => LiveIdentificationCancellationSource.Token; + public Task Task { get; set; } = Task.CompletedTask; public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested; @@ -428,6 +787,124 @@ public sealed class MeetingRecordingCoordinator TranscriptionCancellationSource.Cancel(); } + public void CancelLiveIdentification() + { + LiveIdentificationCancellationSource.Cancel(); + } + + public void AddLiveSegment(TranscriptionSegment segment) + { + lock (liveSegmentsGate) + { + liveSegments.Add(segment); + } + } + + public void AppendAudio(AudioChunk chunk) + { + speakerSampleCollector.AppendAudio(chunk); + } + + public void TryAddSpeakerSample(TranscriptionSegment segment) + { + speakerSampleCollector.TryAdd(segment); + } + + public IReadOnlyList GetLiveSegmentsSnapshot() + { + lock (liveSegmentsGate) + { + return liveSegments.ToList(); + } + } + + public IReadOnlyList GetSpeakerSamplesSnapshot() + { + return speakerSampleCollector.Snapshot(); + } + + public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote) + { + var speakers = GetUnmappedSampleSpeakers(); + if (speakers.Count == 0) + { + return null; + } + + var checkpoint = new LiveIdentificationCheckpoint( + speakers, + BuildAttendeeSignature(meetingNote)); + lock (liveIdentificationGate) + { + return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true + ? null + : checkpoint; + } + } + + public void MarkLiveIdentificationAttempted(LiveIdentificationCheckpoint checkpoint) + { + lock (liveIdentificationGate) + { + lastLiveIdentificationCheckpoint = checkpoint; + } + } + + public bool AddSpeakerMappings(IReadOnlyDictionary mappings) + { + var added = false; + foreach (var (diarizedSpeaker, canonicalSpeaker) in mappings) + { + if (speakerMappings.TryGetValue(diarizedSpeaker, out var existingSpeaker) && + string.Equals(existingSpeaker, canonicalSpeaker, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + speakerMappings[diarizedSpeaker] = canonicalSpeaker; + added = true; + } + + return added; + } + + public TranscriptionSegment Relabel(TranscriptionSegment segment) + { + return speakerMappings.TryGetValue(segment.Speaker, out var speaker) + ? segment with { Speaker = speaker } + : segment; + } + + public IReadOnlyList GetRelabeledLiveSegmentsSnapshot() + { + lock (liveSegmentsGate) + { + return liveSegments.Select(Relabel).ToList(); + } + } + + private IReadOnlyList GetUnmappedSampleSpeakers() + { + return GetSpeakerSamplesSnapshot() + .Select(sample => sample.Speaker) + .Where(speaker => !string.IsNullOrWhiteSpace(speaker)) + .Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase)) + .Where(speaker => !speakerMappings.ContainsKey(speaker)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string BuildAttendeeSignature(MeetingNote meetingNote) + { + return string.Join( + "\n", + meetingNote.Frontmatter.Attendees + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .Select(attendee => attendee.Trim()) + .Order(StringComparer.OrdinalIgnoreCase)); + } + public bool TryComplete() { return Interlocked.Exchange(ref completed, 1) == 0; @@ -437,6 +914,32 @@ public sealed class MeetingRecordingCoordinator { CaptureCancellationSource.Dispose(); TranscriptionCancellationSource.Dispose(); + LiveIdentificationCancellationSource.Dispose(); + } + + public sealed record LiveIdentificationCheckpoint( + IReadOnlyList Speakers, + string AttendeeSignature) + { + public bool Matches(LiveIdentificationCheckpoint other) + { + return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) && + Speakers.Count == other.Speakers.Count && + Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase); + } + } + } + + private sealed class EmptyDictationWordStore : IDictationWordStore + { + public Task> ReadWordsAsync(CancellationToken cancellationToken) + { + return Task.FromResult>([]); + } + + public Task> AddWordAsync(string word, CancellationToken cancellationToken) + { + return ReadWordsAsync(cancellationToken); } } } diff --git a/MeetingAssistant/Recording/NaudioCaptureSource.cs b/MeetingAssistant/Recording/NaudioCaptureSource.cs index d0eb214..4ba70a3 100644 --- a/MeetingAssistant/Recording/NaudioCaptureSource.cs +++ b/MeetingAssistant/Recording/NaudioCaptureSource.cs @@ -8,10 +8,14 @@ namespace MeetingAssistant.Recording; public sealed class MicrophoneAudioSource : IMeetingAudioSource { private readonly MeetingAssistantOptions options; + private readonly ILogger logger; - public MicrophoneAudioSource(IOptions options) + public MicrophoneAudioSource( + IOptions options, + ILogger logger) { this.options = options.Value; + this.logger = logger; } public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) @@ -22,19 +26,37 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource private IAsyncEnumerable CaptureAsync(IWaveIn capture, CancellationToken cancellationToken) { capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels); - return CaptureWith(capture, cancellationToken); + return CaptureWith(capture, "microphone", logger, cancellationToken); } internal static async IAsyncEnumerable CaptureWith( IWaveIn capture, + string sourceName, + ILogger logger, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { var channel = Channel.CreateUnbounded(); + var chunks = 0; capture.DataAvailable += (_, args) => { + if (args.BytesRecorded <= 0) + { + return; + } + var buffer = new byte[args.BytesRecorded]; Buffer.BlockCopy(args.Buffer, 0, buffer, 0, args.BytesRecorded); + if (Interlocked.Increment(ref chunks) == 1) + { + logger.LogInformation( + "Audio capture source {SourceName} produced first chunk: {ByteCount} bytes at {SampleRate} Hz/{Channels} channel(s)", + sourceName, + args.BytesRecorded, + capture.WaveFormat.SampleRate, + capture.WaveFormat.Channels); + } + channel.Writer.TryWrite(new AudioChunk(buffer, capture.WaveFormat.SampleRate, capture.WaveFormat.Channels)); }; capture.RecordingStopped += (_, args) => @@ -51,6 +73,11 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource try { using var registration = cancellationToken.Register(capture.StopRecording); + logger.LogInformation( + "Starting audio capture source {SourceName} at {SampleRate} Hz/{Channels} channel(s)", + sourceName, + capture.WaveFormat.SampleRate, + capture.WaveFormat.Channels); capture.StartRecording(); await foreach (var chunk in channel.Reader.ReadAllAsync(cancellationToken)) @@ -68,10 +95,14 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource public sealed class SystemAudioSource : IMeetingAudioSource { private readonly MeetingAssistantOptions options; + private readonly ILogger logger; - public SystemAudioSource(IOptions options) + public SystemAudioSource( + IOptions options, + ILogger logger) { this.options = options.Value; + this.logger = logger; } public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) @@ -81,6 +112,6 @@ public sealed class SystemAudioSource : IMeetingAudioSource WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels) }; - return MicrophoneAudioSource.CaptureWith(capture, cancellationToken); + return MicrophoneAudioSource.CaptureWith(capture, "system", logger, cancellationToken); } } diff --git a/MeetingAssistant/Recording/RollingAudioBuffer.cs b/MeetingAssistant/Recording/RollingAudioBuffer.cs new file mode 100644 index 0000000..91d953e --- /dev/null +++ b/MeetingAssistant/Recording/RollingAudioBuffer.cs @@ -0,0 +1,109 @@ +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +internal sealed class RollingAudioBuffer +{ + private readonly TimeSpan retention; + private readonly object gate = new(); + private readonly Queue chunks = new(); + private TimeSpan position; + + public RollingAudioBuffer(TimeSpan retention) + { + this.retention = retention > TimeSpan.Zero ? retention : TimeSpan.FromMinutes(10); + } + + public void Append(AudioChunk chunk) + { + if (chunk.Pcm.Length == 0 || chunk.SampleRate <= 0 || chunk.Channels <= 0) + { + return; + } + + lock (gate) + { + var start = position; + var end = start + chunk.Duration; + chunks.Enqueue(new TimedAudioChunk(start, end, chunk)); + position = end; + Prune(end - retention); + } + } + + public byte[] TryExtractWav(TimeSpan start, TimeSpan end) + { + if (end <= start) + { + return []; + } + + List matchingChunks; + lock (gate) + { + matchingChunks = chunks + .Where(chunk => chunk.End > start && chunk.Start < end) + .ToList(); + } + + if (matchingChunks.Count == 0) + { + return []; + } + + var format = new WaveFormat( + matchingChunks[0].Chunk.SampleRate, + 16, + matchingChunks[0].Chunk.Channels); + using var output = new MemoryStream(); + using (var writer = new WaveFileWriter(output, format)) + { + foreach (var timedChunk in matchingChunks) + { + if (timedChunk.Chunk.SampleRate != format.SampleRate || timedChunk.Chunk.Channels != format.Channels) + { + continue; + } + + var blockAlign = format.BlockAlign; + var copyStart = Max(start, timedChunk.Start); + var copyEnd = Min(end, timedChunk.End); + var firstFrame = (int)Math.Floor((copyStart - timedChunk.Start).TotalSeconds * format.SampleRate); + var afterLastFrame = (int)Math.Ceiling((copyEnd - timedChunk.Start).TotalSeconds * format.SampleRate); + var byteOffset = Clamp(firstFrame * blockAlign, 0, timedChunk.Chunk.Pcm.Length); + var byteCount = Clamp(afterLastFrame * blockAlign, byteOffset, timedChunk.Chunk.Pcm.Length) - byteOffset; + if (byteCount > 0) + { + writer.Write(timedChunk.Chunk.Pcm, byteOffset, byteCount); + } + } + } + + return output.ToArray(); + } + + private void Prune(TimeSpan cutoff) + { + while (chunks.Count > 0 && chunks.Peek().End < cutoff) + { + chunks.Dequeue(); + } + } + + private static TimeSpan Max(TimeSpan left, TimeSpan right) + { + return left >= right ? left : right; + } + + private static TimeSpan Min(TimeSpan left, TimeSpan right) + { + return left <= right ? left : right; + } + + private static int Clamp(int value, int min, int max) + { + return Math.Min(Math.Max(value, min), max); + } + + private sealed record TimedAudioChunk(TimeSpan Start, TimeSpan End, AudioChunk Chunk); +} diff --git a/MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs b/MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs new file mode 100644 index 0000000..dd6538c --- /dev/null +++ b/MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs @@ -0,0 +1,105 @@ +using MeetingAssistant.Speakers; +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Recording; + +internal sealed class SpeakerAudioSampleCollector +{ + private readonly object gate = new(); + private readonly RollingAudioBuffer audioBuffer; + private readonly Dictionary> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase); + private readonly int maxSamplesPerSpeaker; + + public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker) + { + audioBuffer = new RollingAudioBuffer(bufferDuration); + this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker); + } + + public void AppendAudio(AudioChunk chunk) + { + audioBuffer.Append(chunk); + } + + public void TryAdd(TranscriptionSegment segment) + { + if (!IsDiarizedSpeaker(segment.Speaker)) + { + return; + } + + var score = Score(segment); + if (score <= 0) + { + return; + } + + var wavBytes = audioBuffer.TryExtractWav(segment.Start, segment.End); + if (wavBytes.Length == 0) + { + return; + } + + var sample = new SpeakerAudioSample(segment.Speaker, segment, wavBytes, score); + lock (gate) + { + if (!samplesBySpeaker.TryGetValue(segment.Speaker, out var samples)) + { + samples = []; + samplesBySpeaker[segment.Speaker] = samples; + } + + samples.Add(sample); + var bestSamples = samples + .OrderByDescending(candidate => candidate.Score) + .Take(maxSamplesPerSpeaker) + .ToList(); + samples.Clear(); + samples.AddRange(bestSamples); + } + } + + public IReadOnlyList Snapshot() + { + lock (gate) + { + return samplesBySpeaker.Values + .SelectMany(samples => samples) + .OrderBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase) + .ThenByDescending(sample => sample.Score) + .ToList(); + } + } + + private static bool IsDiarizedSpeaker(string speaker) + { + return !string.IsNullOrWhiteSpace(speaker) && + !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase); + } + + private static double Score(TranscriptionSegment segment) + { + var durationSeconds = (segment.End - segment.Start).TotalSeconds; + if (durationSeconds < 2 || durationSeconds > 30) + { + return 0; + } + + var words = segment.Text + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Length; + if (words < 3 || words > 80) + { + return 0; + } + + var durationScore = 1 - Math.Min(Math.Abs(durationSeconds - 8) / 8, 1); + var wordScore = 1 - Math.Min(Math.Abs(words - 18) / 18.0, 1); + var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') || + segment.Text.TrimEnd().EndsWith('?') || + segment.Text.TrimEnd().EndsWith('!') + ? 5 + : 0; + return durationScore * 70 + wordScore * 30 + sentenceBonus; + } +} diff --git a/MeetingAssistant/Recording/VaultTranscriptStore.cs b/MeetingAssistant/Recording/VaultTranscriptStore.cs index fe76bf4..df5a5b0 100644 --- a/MeetingAssistant/Recording/VaultTranscriptStore.cs +++ b/MeetingAssistant/Recording/VaultTranscriptStore.cs @@ -17,7 +17,7 @@ public sealed class VaultTranscriptStore : ITranscriptStore public async Task CreateSessionAsync(CancellationToken cancellationToken) { - var folder = VaultPath.Resolve(options.Vault.TranscriptsFolder); + var folder = VaultPath.Resolve(options.Vault, options.Vault.TranscriptsFolder); Directory.CreateDirectory(folder); var fileName = $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-transcript.md"; diff --git a/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityDiarizationClient.cs b/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityDiarizationClient.cs new file mode 100644 index 0000000..85a1232 --- /dev/null +++ b/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityDiarizationClient.cs @@ -0,0 +1,127 @@ +using System.Threading.Channels; +using MeetingAssistant.Transcription; +using Microsoft.CognitiveServices.Speech; +using Microsoft.CognitiveServices.Speech.Audio; +using Microsoft.CognitiveServices.Speech.Transcription; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Speakers; + +public sealed class AzureSpeechSpeakerIdentityDiarizationClient : ISpeakerIdentityDiarizationClient +{ + private readonly AzureSpeechOptions options; + private readonly ILogger logger; + + public AzureSpeechSpeakerIdentityDiarizationClient( + IOptions options, + ILogger logger) + { + this.options = options.Value.SpeakerIdentification.AzureSpeech; + this.logger = logger; + } + + public async Task> DiarizeAsync( + string wavPath, + CancellationToken cancellationToken) + { + var key = AzureSpeechStreamingTranscriptionProvider.ResolveKey(options); + if (string.IsNullOrWhiteSpace(key)) + { + logger.LogWarning( + "Speaker identity Azure Speech matching is enabled but no key is configured directly or via {KeyEnv}", + options.KeyEnv); + return []; + } + + var speechConfig = string.IsNullOrWhiteSpace(options.Region) + ? SpeechConfig.FromEndpoint(new Uri(options.Endpoint), key) + : SpeechConfig.FromSubscription(key, options.Region); + speechConfig.SpeechRecognitionLanguage = ResolveRecognitionLanguage(options); + speechConfig.OutputFormat = OutputFormat.Detailed; + speechConfig.SetProperty( + PropertyId.SpeechServiceResponse_DiarizeIntermediateResults, + options.DiarizeIntermediateResults ? "true" : "false"); + + using var audioConfig = AudioConfig.FromWavFileInput(wavPath); + using var recognizer = new ConversationTranscriber(speechConfig, audioConfig); + var channel = Channel.CreateUnbounded(); + + recognizer.Transcribed += (_, args) => + { + if (args.Result.Reason != ResultReason.RecognizedSpeech) + { + return; + } + + var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); + channel.Writer.TryWrite(new TranscriptionSegment( + start, + start + args.Result.Duration, + NormalizeSpeakerId(args.Result.SpeakerId), + args.Result.Text.Trim())); + }; + recognizer.Canceled += (_, args) => + { + if (args.Reason == CancellationReason.Error) + { + channel.Writer.TryComplete(new InvalidOperationException( + $"Azure Speech identity matching canceled: {args.ErrorCode} {args.ErrorDetails}")); + return; + } + + channel.Writer.TryComplete(); + }; + recognizer.SessionStopped += (_, _) => channel.Writer.TryComplete(); + + await recognizer.StartTranscribingAsync(); + var segments = new List(); + try + { + await foreach (var segment in channel.Reader.ReadAllAsync(cancellationToken)) + { + segments.Add(segment); + } + } + finally + { + try + { + var stopTask = recognizer.StopTranscribingAsync(); + if (options.RecognitionStopTimeout > TimeSpan.Zero) + { + await stopTask.WaitAsync(options.RecognitionStopTimeout, CancellationToken.None); + } + else + { + await stopTask.WaitAsync(CancellationToken.None); + } + } + catch (TimeoutException) + { + logger.LogWarning("Timed out while stopping Azure Speech identity diarization recognizer"); + } + } + + return segments; + } + + internal static string ResolveRecognitionLanguage(AzureSpeechOptions options) + { + if (!options.Language.Equals("auto", StringComparison.OrdinalIgnoreCase)) + { + return options.Language; + } + + return options.AutoDetectLanguages + .Select(language => language.Trim()) + .FirstOrDefault(language => !string.IsNullOrWhiteSpace(language)) + ?? "de-DE"; + } + + private static string NormalizeSpeakerId(string? speakerId) + { + return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase) + ? "Unknown" + : speakerId; + } +} diff --git a/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityMatcher.cs b/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityMatcher.cs new file mode 100644 index 0000000..fd282fb --- /dev/null +++ b/MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityMatcher.cs @@ -0,0 +1,229 @@ +using MeetingAssistant.Transcription; +using NAudio.Wave; + +namespace MeetingAssistant.Speakers; + +public interface ISpeakerIdentityDiarizationClient +{ + Task> DiarizeAsync( + string wavPath, + CancellationToken cancellationToken); +} + +public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher +{ + private readonly ISpeakerIdentityDiarizationClient diarizationClient; + private readonly SpeakerIdentificationOptions options; + private readonly ILogger logger; + + public AzureSpeechSpeakerIdentityMatcher( + ISpeakerIdentityDiarizationClient diarizationClient, + Microsoft.Extensions.Options.IOptions options, + ILogger logger) + { + this.diarizationClient = diarizationClient; + this.options = options.Value.SpeakerIdentification; + this.logger = logger; + } + + public async Task MatchAsync( + SpeakerIdentityMatchRequest request, + CancellationToken cancellationToken) + { + if (!options.Enabled || request.Candidates.Count == 0) + { + return null; + } + + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} against {CandidateCount} candidate(s)", + request.DiarizedSpeaker, + request.Candidates.Count); + var tempPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-speaker-match", $"{Guid.NewGuid():N}.wav"); + try + { + Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!); + var layout = WriteCompositeWav(tempPath, request); + if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0) + { + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} skipped because no readable snippets were available", + request.DiarizedSpeaker); + return null; + } + + var segments = await diarizationClient.DiarizeAsync(tempPath, cancellationToken); + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} received {SegmentCount} diarized segment(s)", + request.DiarizedSpeaker, + segments.Count); + var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments); + if (string.IsNullOrWhiteSpace(unknownSpeaker)) + { + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} found no speaker for the unknown snippet", + request.DiarizedSpeaker); + return null; + } + + foreach (var known in layout.KnownSegments.GroupBy(segment => segment.IdentityId)) + { + var matchingSnippetCount = known.Count(segment => + string.Equals( + FindBestSpeaker(segment.Segment, segments), + unknownSpeaker, + StringComparison.Ordinal)); + var requiredMatches = known.Count() > 1 ? 2 : 1; + if (matchingSnippetCount >= requiredMatches) + { + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} matched identity {IdentityId}", + request.DiarizedSpeaker, + known.Key); + return new SpeakerIdentityMatch(known.Key); + } + } + + logger.LogInformation( + "Speaker identity matching {DiarizedSpeaker} found no matching identity", + request.DiarizedSpeaker); + return null; + } + finally + { + TryDelete(tempPath); + } + } + + private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request) + { + using var firstReader = OpenFirstReadableWave(request); + if (firstReader is null) + { + return new CompositeLayout(null, []); + } + + var silence = TimeSpan.FromSeconds(Math.Max(0, options.SilenceBetweenSnippetsSeconds)); + using var writer = new WaveFileWriter(path, firstReader.WaveFormat); + var current = TimeSpan.Zero; + var knownSegments = new List(); + + foreach (var candidate in request.Candidates) + { + foreach (var snippet in candidate.Snippets.Where(storedSnippet => storedSnippet.Length > 0)) + { + var segment = AppendSnippet(writer, firstReader.WaveFormat, snippet, current); + if (segment is null) + { + continue; + } + + knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value)); + current = segment.Value.End + silence; + WriteSilence(writer, firstReader.WaveFormat, silence); + } + } + + var unknownSegment = AppendSnippet(writer, firstReader.WaveFormat, request.UnknownSnippet, current); + return new CompositeLayout(unknownSegment, knownSegments); + } + + private static WaveFileReader? OpenFirstReadableWave(SpeakerIdentityMatchRequest request) + { + foreach (var bytes in request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet)) + { + if (bytes.Length == 0) + { + continue; + } + + try + { + return new WaveFileReader(new MemoryStream(bytes)); + } + catch (FormatException) + { + } + } + + return null; + } + + private static TimeSegment? AppendSnippet( + WaveFileWriter writer, + WaveFormat targetFormat, + byte[] snippet, + TimeSpan start) + { + if (snippet.Length == 0) + { + return null; + } + + using var reader = new WaveFileReader(new MemoryStream(snippet)); + if (!WaveFormatsMatch(reader.WaveFormat, targetFormat)) + { + throw new InvalidDataException("Speaker identity snippets must use the same WAV format."); + } + + reader.CopyTo(writer); + return new TimeSegment(start, start + reader.TotalTime); + } + + private static void WriteSilence(WaveFileWriter writer, WaveFormat format, TimeSpan duration) + { + if (duration <= TimeSpan.Zero) + { + return; + } + + var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)]; + writer.Write(bytes, 0, bytes.Length); + } + + private static string? FindBestSpeaker(TimeSegment segment, IReadOnlyList segments) + { + var bestOverlap = 0d; + string? bestSpeaker = null; + foreach (var transcriptSegment in segments) + { + var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds) + - Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds); + if (overlap > bestOverlap) + { + bestOverlap = overlap; + bestSpeaker = transcriptSegment.Speaker; + } + } + + return bestSpeaker; + } + + private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right) + { + return left.Encoding == right.Encoding + && left.SampleRate == right.SampleRate + && left.Channels == right.Channels + && left.BitsPerSample == right.BitsPerSample; + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch (IOException) + { + } + } + + private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList KnownSegments); + + private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment); + + private readonly record struct TimeSegment(TimeSpan Start, TimeSpan End); +} diff --git a/MeetingAssistant/Speakers/NoopSpeakerIdentityMatcher.cs b/MeetingAssistant/Speakers/NoopSpeakerIdentityMatcher.cs new file mode 100644 index 0000000..4eaa49f --- /dev/null +++ b/MeetingAssistant/Speakers/NoopSpeakerIdentityMatcher.cs @@ -0,0 +1,11 @@ +namespace MeetingAssistant.Speakers; + +public sealed class NoopSpeakerIdentityMatcher : ISpeakerIdentityMatcher +{ + public Task MatchAsync( + SpeakerIdentityMatchRequest request, + CancellationToken cancellationToken) + { + return Task.FromResult(null); + } +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentificationModels.cs b/MeetingAssistant/Speakers/SpeakerIdentificationModels.cs new file mode 100644 index 0000000..887b2a2 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentificationModels.cs @@ -0,0 +1,64 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Speakers; + +public sealed record SpeakerIdentificationRequest( + string AudioPath, + MeetingNote MeetingNote, + IReadOnlyList Segments, + IReadOnlyList? Samples = null); + +public sealed record SpeakerAudioSample( + string Speaker, + TranscriptionSegment Segment, + byte[] WavBytes, + double Score); + +public sealed record SpeakerIdentificationResult( + IReadOnlyList Segments, + IReadOnlyDictionary SpeakerMappings, + IReadOnlyList? AttendeeMatches = null); + +public sealed record SpeakerIdentityAttendeeMatch( + string DisplayName, + IReadOnlyList AcceptedNames); + +public sealed record SpeakerIdentityMatchRequest( + string DiarizedSpeaker, + byte[] UnknownSnippet, + IReadOnlyList Candidates); + +public sealed record SpeakerIdentityMatchCandidate( + int IdentityId, + string? CanonicalName, + int TranscriptionCount, + IReadOnlyList Snippets); + +public sealed record SpeakerIdentityMatch(int IdentityId); + +public interface ISpeakerIdentityMatcher +{ + Task MatchAsync( + SpeakerIdentityMatchRequest request, + CancellationToken cancellationToken); +} + +public interface ISpeakerSnippetExtractor +{ + Task ExtractSnippetAsync( + string audioPath, + IReadOnlyList speakerSegments, + CancellationToken cancellationToken); +} + +public interface ISpeakerIdentificationService +{ + Task IdentifyKnownSpeakersAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken); + + Task ProcessFinishedTranscriptAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentity.cs b/MeetingAssistant/Speakers/SpeakerIdentity.cs new file mode 100644 index 0000000..27a0e47 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentity.cs @@ -0,0 +1,116 @@ +using Microsoft.EntityFrameworkCore; + +namespace MeetingAssistant.Speakers; + +public sealed class SpeakerIdentity +{ + public int Id { get; set; } + + public string? CanonicalName { get; set; } + + public int TranscriptionCount { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public List CandidateNames { get; set; } = []; + + public List Aliases { get; set; } = []; + + public List Snippets { get; set; } = []; + + public string? GetDisplayName() + { + if (!string.IsNullOrWhiteSpace(CanonicalName)) + { + return CanonicalName; + } + + return Aliases + .Select(alias => alias.Name) + .Where(alias => !string.IsNullOrWhiteSpace(alias)) + .Order(StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + } +} + +public sealed class SpeakerCandidateName +{ + public int Id { get; set; } + + public int SpeakerIdentityId { get; set; } + + public SpeakerIdentity? SpeakerIdentity { get; set; } + + public string Name { get; set; } = ""; +} + +public sealed class SpeakerSnippet +{ + public int Id { get; set; } + + public int SpeakerIdentityId { get; set; } + + public SpeakerIdentity? SpeakerIdentity { get; set; } + + public byte[] WavBytes { get; set; } = []; + + public DateTimeOffset CreatedAt { get; set; } +} + +public sealed class SpeakerAlias +{ + public int Id { get; set; } + + public int SpeakerIdentityId { get; set; } + + public SpeakerIdentity? SpeakerIdentity { get; set; } + + public string Name { get; set; } = ""; +} + +public sealed class SpeakerIdentityDbContext : DbContext +{ + public SpeakerIdentityDbContext(DbContextOptions options) + : base(options) + { + } + + public DbSet SpeakerIdentities => Set(); + + public DbSet SpeakerCandidateNames => Set(); + + public DbSet SpeakerAliases => Set(); + + public DbSet SpeakerSnippets => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasMany(identity => identity.CandidateNames) + .WithOne(candidate => candidate.SpeakerIdentity) + .HasForeignKey(candidate => candidate.SpeakerIdentityId) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasMany(identity => identity.Aliases) + .WithOne(alias => alias.SpeakerIdentity) + .HasForeignKey(alias => alias.SpeakerIdentityId) + .OnDelete(DeleteBehavior.Cascade); + + entity.HasMany(identity => identity.Snippets) + .WithOne(snippet => snippet.SpeakerIdentity) + .HasForeignKey(snippet => snippet.SpeakerIdentityId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity() + .HasIndex(candidate => new { candidate.SpeakerIdentityId, candidate.Name }) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name }) + .IsUnique(); + } +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentityDatabaseInitializer.cs b/MeetingAssistant/Speakers/SpeakerIdentityDatabaseInitializer.cs new file mode 100644 index 0000000..ab6fb59 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentityDatabaseInitializer.cs @@ -0,0 +1,33 @@ +using Microsoft.EntityFrameworkCore; + +namespace MeetingAssistant.Speakers; + +public sealed class SpeakerIdentityDatabaseInitializer : IHostedService +{ + private static readonly SemaphoreSlim InitializeLock = new(1, 1); + private readonly IDbContextFactory dbContextFactory; + + public SpeakerIdentityDatabaseInitializer(IDbContextFactory dbContextFactory) + { + this.dbContextFactory = dbContextFactory; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + await InitializeLock.WaitAsync(cancellationToken); + try + { + await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); + } + finally + { + InitializeLock.Release(); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentityMergeService.cs b/MeetingAssistant/Speakers/SpeakerIdentityMergeService.cs new file mode 100644 index 0000000..9d821d9 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentityMergeService.cs @@ -0,0 +1,208 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Speakers; + +public interface ISpeakerIdentityMergeService +{ + Task MergeRecentIdentitiesAsync( + TimeSpan? recentIdentityAge, + CancellationToken cancellationToken); +} + +public sealed record SpeakerIdentityMergeResult( + int RecentIdentityCount, + int CandidateIdentityCount, + int MatchAttempts, + int MergedPairs); + +public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService +{ + private readonly IDbContextFactory dbContextFactory; + private readonly ISpeakerIdentityMatcher matcher; + private readonly SpeakerIdentificationOptions options; + private readonly ILogger logger; + + public SpeakerIdentityMergeService( + IDbContextFactory dbContextFactory, + ISpeakerIdentityMatcher matcher, + IOptions options, + ILogger logger) + { + this.dbContextFactory = dbContextFactory; + this.matcher = matcher; + this.options = options.Value.SpeakerIdentification; + this.logger = logger; + } + + public async Task MergeRecentIdentitiesAsync( + TimeSpan? recentIdentityAge, + CancellationToken cancellationToken) + { + await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); + + var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge); + var identities = await context.SpeakerIdentities + .Include(identity => identity.Aliases) + .Include(identity => identity.CandidateNames) + .Include(identity => identity.Snippets) + .OrderByDescending(identity => identity.TranscriptionCount) + .ThenBy(identity => identity.Id) + .ToListAsync(cancellationToken); + + var recentIds = identities + .Where(identity => identity.CreatedAt >= cutoff) + .Select(identity => identity.Id) + .ToHashSet(); + var mergedPairs = 0; + var attempts = 0; + + foreach (var sourceId in recentIds.ToList()) + { + var source = identities.SingleOrDefault(identity => identity.Id == sourceId); + if (source is null || source.Snippets.Count == 0) + { + continue; + } + + var targets = identities + .Where(identity => identity.Id != source.Id && identity.Snippets.Count > 0) + .ToList(); + foreach (var batch in targets.Chunk(Math.Max(1, options.MatchBatchSize))) + { + var firstSnippet = SelectSnippet(source, excludedSnippet: null); + if (firstSnippet is null) + { + break; + } + + attempts++; + var firstMatch = await matcher.MatchAsync( + CreateRequest(source, firstSnippet, batch), + cancellationToken); + if (firstMatch is null || firstMatch.IdentityId == source.Id) + { + continue; + } + + var target = batch.SingleOrDefault(identity => identity.Id == firstMatch.IdentityId); + if (target is null) + { + continue; + } + + var secondSnippet = SelectSnippet(source, excludedSnippet: firstSnippet); + if (secondSnippet is null) + { + continue; + } + + attempts++; + var secondMatch = await matcher.MatchAsync( + CreateRequest(source, secondSnippet, batch), + cancellationToken); + if (secondMatch?.IdentityId != target.Id) + { + continue; + } + + MergeIdentities(target, source); + context.SpeakerIdentities.Remove(source); + identities.Remove(source); + mergedPairs++; + break; + } + } + + await context.SaveChangesAsync(cancellationToken); + logger.LogInformation( + "Speaker identity merge diagnostics completed: {MergedPairs} merge(s), {Attempts} match attempt(s)", + mergedPairs, + attempts); + return new SpeakerIdentityMergeResult( + recentIds.Count, + identities.Count, + attempts, + mergedPairs); + } + + private static SpeakerIdentityMatchRequest CreateRequest( + SpeakerIdentity source, + SpeakerSnippet unknownSnippet, + IReadOnlyList candidates) + { + return new SpeakerIdentityMatchRequest( + source.GetDisplayName() ?? $"identity-{source.Id}", + unknownSnippet.WavBytes, + candidates.Select(identity => new SpeakerIdentityMatchCandidate( + identity.Id, + identity.GetDisplayName(), + identity.TranscriptionCount, + identity.Snippets + .OrderBy(snippet => snippet.CreatedAt) + .Select(snippet => snippet.WavBytes) + .ToList())) + .ToList()); + } + + private static SpeakerSnippet? SelectSnippet(SpeakerIdentity identity, SpeakerSnippet? excludedSnippet) + { + return identity.Snippets + .OrderBy(snippet => snippet.CreatedAt) + .FirstOrDefault(snippet => excludedSnippet is null || snippet.Id != excludedSnippet.Id); + } + + private void MergeIdentities(SpeakerIdentity target, SpeakerIdentity source) + { + AddAlias(target, source.CanonicalName); + foreach (var alias in source.Aliases) + { + AddAlias(target, alias.Name); + } + + foreach (var candidate in source.CandidateNames) + { + AddAlias(target, candidate.Name); + } + + target.TranscriptionCount += source.TranscriptionCount; + target.UpdatedAt = DateTimeOffset.UtcNow; + var retainedSnippets = target.Snippets + .Concat(source.Snippets) + .OrderBy(snippet => StableSnippetKey(snippet.WavBytes)) + .Take(Math.Max(1, options.MaxSnippetsPerSpeaker)) + .Select(snippet => new SpeakerSnippet + { + WavBytes = snippet.WavBytes, + CreatedAt = snippet.CreatedAt + }) + .ToList(); + target.Snippets.Clear(); + target.Snippets.AddRange(retainedSnippets); + } + + private static void AddAlias(SpeakerIdentity identity, string? alias) + { + if (string.IsNullOrWhiteSpace(alias) || + string.Equals(identity.CanonicalName, alias, StringComparison.OrdinalIgnoreCase) || + identity.Aliases.Any(existing => string.Equals(existing.Name, alias, StringComparison.OrdinalIgnoreCase))) + { + return; + } + + identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() }); + } + + private static int StableSnippetKey(byte[] snippet) + { + var hash = new HashCode(); + foreach (var value in snippet) + { + hash.Add(value); + } + + return hash.ToHashCode(); + } + +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentitySchema.cs b/MeetingAssistant/Speakers/SpeakerIdentitySchema.cs new file mode 100644 index 0000000..de89a13 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentitySchema.cs @@ -0,0 +1,103 @@ +using Microsoft.EntityFrameworkCore; + +namespace MeetingAssistant.Speakers; + +internal static class SpeakerIdentitySchema +{ + public static async Task EnsureCreatedOrUpdatedAsync( + SpeakerIdentityDbContext context, + CancellationToken cancellationToken) + { + await context.Database.EnsureCreatedAsync(cancellationToken); + await EnsureSpeakerIdentityTimestampColumnsAsync(context, cancellationToken); + await context.Database.ExecuteSqlRawAsync( + """ + CREATE TABLE IF NOT EXISTS "SpeakerAliases" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerAliases" PRIMARY KEY AUTOINCREMENT, + "SpeakerIdentityId" INTEGER NOT NULL, + "Name" TEXT NOT NULL, + CONSTRAINT "FK_SpeakerAliases_SpeakerIdentities_SpeakerIdentityId" + FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE + ); + """, + cancellationToken); + await context.Database.ExecuteSqlRawAsync( + """ + CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerAliases_SpeakerIdentityId_Name" + ON "SpeakerAliases" ("SpeakerIdentityId", "Name"); + """, + cancellationToken); + } + + private static async Task EnsureSpeakerIdentityTimestampColumnsAsync( + SpeakerIdentityDbContext context, + CancellationToken cancellationToken) + { + const string defaultTimestamp = "1970-01-01T00:00:00.0000000+00:00"; + var hasCreatedAt = await HasColumnAsync(context, "SpeakerIdentities", "CreatedAt", cancellationToken); + if (!hasCreatedAt) + { + await context.Database.ExecuteSqlRawAsync( + $""" + ALTER TABLE "SpeakerIdentities" + ADD COLUMN "CreatedAt" TEXT NOT NULL DEFAULT '{defaultTimestamp}'; + """, + cancellationToken); + } + + var hasUpdatedAt = await HasColumnAsync(context, "SpeakerIdentities", "UpdatedAt", cancellationToken); + if (!hasUpdatedAt) + { + await context.Database.ExecuteSqlRawAsync( + $""" + ALTER TABLE "SpeakerIdentities" + ADD COLUMN "UpdatedAt" TEXT NOT NULL DEFAULT '{defaultTimestamp}'; + """, + cancellationToken); + await context.Database.ExecuteSqlRawAsync( + $""" + UPDATE "SpeakerIdentities" + SET "UpdatedAt" = "CreatedAt" + WHERE "UpdatedAt" = '{defaultTimestamp}'; + """, + cancellationToken); + } + } + + private static async Task HasColumnAsync( + SpeakerIdentityDbContext context, + string tableName, + string columnName, + CancellationToken cancellationToken) + { + var connection = context.Database.GetDbConnection(); + var shouldClose = connection.State == System.Data.ConnectionState.Closed; + if (shouldClose) + { + await connection.OpenAsync(cancellationToken); + } + + try + { + await using var command = connection.CreateCommand(); + command.CommandText = $"""PRAGMA table_info("{tableName}")"""; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + finally + { + if (shouldClose) + { + await connection.CloseAsync(); + } + } + } +} diff --git a/MeetingAssistant/Speakers/SpeakerIdentityService.cs b/MeetingAssistant/Speakers/SpeakerIdentityService.cs new file mode 100644 index 0000000..52d7f50 --- /dev/null +++ b/MeetingAssistant/Speakers/SpeakerIdentityService.cs @@ -0,0 +1,379 @@ +using MeetingAssistant.Transcription; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Speakers; + +public sealed class SpeakerIdentityService : ISpeakerIdentificationService +{ + private readonly IDbContextFactory dbContextFactory; + private readonly ISpeakerSnippetExtractor snippetExtractor; + private readonly ISpeakerIdentityMatcher matcher; + private readonly SpeakerIdentificationOptions options; + private readonly ILogger logger; + + public SpeakerIdentityService( + IDbContextFactory dbContextFactory, + ISpeakerSnippetExtractor snippetExtractor, + ISpeakerIdentityMatcher matcher, + IOptions options, + ILogger logger) + { + this.dbContextFactory = dbContextFactory; + this.snippetExtractor = snippetExtractor; + this.matcher = matcher; + this.options = options.Value.SpeakerIdentification; + this.logger = logger; + } + + public async Task ProcessFinishedTranscriptAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken); + } + + public async Task IdentifyKnownSpeakersAsync( + SpeakerIdentificationRequest request, + CancellationToken cancellationToken) + { + return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken); + } + + private async Task ProcessTranscriptAsync( + SpeakerIdentificationRequest request, + SpeakerIdentityProcessingMode mode, + CancellationToken cancellationToken) + { + if (!options.Enabled || request.Segments.Count == 0) + { + return new SpeakerIdentificationResult(request.Segments, new Dictionary()); + } + + await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); + await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); + + var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees); + var speakerMappings = new Dictionary(StringComparer.OrdinalIgnoreCase); + var attendeeMatches = new List(); + var matchedAcceptedNames = new HashSet(StringComparer.OrdinalIgnoreCase); + var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>(); + var samplesBySpeaker = request.Samples? + .Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0) + .GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase) + .ToDictionary( + group => group.Key, + group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(), + StringComparer.OrdinalIgnoreCase) + ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var group in request.Segments + .Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker)) + .GroupBy(segment => segment.Speaker) + .OrderBy(group => group.Min(segment => segment.Start))) + { + var speaker = group.Key; + if (speakerMappings.ContainsKey(speaker)) + { + continue; + } + + var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet) + ? suppliedSnippet + : await snippetExtractor.ExtractSnippetAsync( + request.AudioPath, + group.ToList(), + cancellationToken); + if (snippet.Length == 0) + { + unmatchedSpeakers.Add((speaker, snippet)); + continue; + } + + var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken); + if (match is null) + { + unmatchedSpeakers.Add((speaker, snippet)); + continue; + } + + var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken); + if (identity is null) + { + unmatchedSpeakers.Add((speaker, snippet)); + continue; + } + + if (mode == SpeakerIdentityProcessingMode.Final) + { + UpdateMatchedIdentity(identity, attendees, snippet); + foreach (var acceptedName in GetAcceptedNames(identity)) + { + matchedAcceptedNames.Add(acceptedName); + } + } + + var speakerName = identity.GetDisplayName(); + if (!string.IsNullOrWhiteSpace(speakerName)) + { + speakerMappings[speaker] = speakerName; + attendeeMatches.Add(new SpeakerIdentityAttendeeMatch( + speakerName, + GetAcceptedNames(identity).ToList())); + } + } + + if (mode == SpeakerIdentityProcessingMode.Final) + { + await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken); + } + + await context.SaveChangesAsync(cancellationToken); + + var relabeledSegments = request.Segments + .Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName) + ? segment with { Speaker = speakerName } + : segment) + .ToList(); + return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches); + } + + private async Task FindMatchAsync( + SpeakerIdentityDbContext context, + IReadOnlyList attendees, + string speaker, + byte[] snippet, + CancellationToken cancellationToken) + { + var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge; + var maxCandidates = Math.Max(1, options.MaxMatchCandidates); + var identities = await context.SpeakerIdentities + .Include(identity => identity.Snippets) + .Include(identity => identity.Aliases) + .OrderByDescending(identity => identity.TranscriptionCount) + .ThenBy(identity => identity.Id) + .ToListAsync(cancellationToken); + identities = identities + .Select(identity => new + { + Identity = identity, + IsAttendee = MatchesAttendees(identity, attendees), + IsActive = identity.UpdatedAt >= activeCutoff + }) + .Where(candidate => candidate.IsAttendee || candidate.IsActive) + .OrderByDescending(candidate => candidate.IsAttendee) + .ThenByDescending(candidate => candidate.Identity.TranscriptionCount) + .ThenBy(candidate => candidate.Identity.Id) + .Take(maxCandidates) + .Select(candidate => candidate.Identity) + .ToList(); + + foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize))) + { + var request = new SpeakerIdentityMatchRequest( + speaker, + snippet, + batch.Select(identity => new SpeakerIdentityMatchCandidate( + identity.Id, + identity.CanonicalName, + identity.TranscriptionCount, + identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList())) + .ToList()); + var match = await matcher.MatchAsync(request, cancellationToken); + if (match is not null) + { + return match; + } + } + + return null; + } + + private static bool MatchesAttendees( + SpeakerIdentity identity, + IReadOnlyList attendees) + { + var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase); + return GetAcceptedNames(identity).Any(attendeeSet.Contains); + } + + private static Task LoadIdentityAsync( + SpeakerIdentityDbContext context, + int identityId, + CancellationToken cancellationToken) + { + return context.SpeakerIdentities + .Include(identity => identity.CandidateNames) + .Include(identity => identity.Aliases) + .Include(identity => identity.Snippets) + .SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken); + } + + private void UpdateMatchedIdentity( + SpeakerIdentity identity, + IReadOnlyList attendees, + byte[] snippet) + { + identity.TranscriptionCount++; + identity.UpdatedAt = DateTimeOffset.UtcNow; + + if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0) + { + var currentCandidates = identity.CandidateNames + .Select(candidate => candidate.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null; + var aliasToCandidate = identity.Aliases + .Where(alias => !string.IsNullOrWhiteSpace(alias.Name)) + .SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate })) + .Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) || + pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) || + pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase)) + .ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase); + var intersection = attendees + .Select(attendee => currentCandidates.Contains(attendee) + ? attendee + : aliasToCandidate.GetValueOrDefault(attendee) ?? + (identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase)) + ? fallbackAliasCandidate + : null)) + .Where(candidate => !string.IsNullOrWhiteSpace(candidate)) + .Select(candidate => candidate!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (intersection.Count == 0) + { + ResetCandidates(identity, attendees); + ReplaceOldestSnippet(identity, snippet); + return; + } + + ResetCandidates(identity, intersection); + if (intersection.Count == 1) + { + identity.CanonicalName = intersection[0]; + } + } + + AddSnippetIfNeeded(identity, snippet); + } + + private async Task LearnUnmatchedSpeakersAsync( + SpeakerIdentityDbContext context, + IReadOnlyList attendees, + IEnumerable matchedCanonicalNames, + IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers, + CancellationToken cancellationToken) + { + var remainingCandidates = attendees + .Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + if (remainingCandidates.Count == 0) + { + return; + } + + foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0)) + { + var now = DateTimeOffset.UtcNow; + context.SpeakerIdentities.Add(new SpeakerIdentity + { + CreatedAt = now, + UpdatedAt = now, + CandidateNames = remainingCandidates + .Select(candidate => new SpeakerCandidateName { Name = candidate }) + .ToList(), + Snippets = + [ + new SpeakerSnippet + { + WavBytes = snippet, + CreatedAt = now + } + ] + }); + } + + await Task.CompletedTask; + } + + private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet) + { + if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker) + { + return; + } + + identity.Snippets.Add(new SpeakerSnippet + { + WavBytes = snippet, + CreatedAt = DateTimeOffset.UtcNow + }); + } + + private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet) + { + var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault(); + if (oldest is not null) + { + identity.Snippets.Remove(oldest); + } + + if (snippet.Length > 0) + { + identity.Snippets.Add(new SpeakerSnippet + { + WavBytes = snippet, + CreatedAt = DateTimeOffset.UtcNow + }); + } + } + + private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList candidates) + { + identity.CandidateNames.Clear(); + identity.CandidateNames.AddRange(candidates + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .Select(candidate => new SpeakerCandidateName { Name = candidate })); + } + + private static IReadOnlySet GetAcceptedNames(SpeakerIdentity identity) + { + return new[] + { + identity.CanonicalName + } + .Concat(identity.Aliases.Select(alias => alias.Name)) + .Concat(identity.CandidateNames.Select(candidate => candidate.Name)) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .Select(name => name!.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + private static IReadOnlyList NormalizeAttendees(IEnumerable attendees) + { + return attendees + .Select(NormalizeAttendee) + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string NormalizeAttendee(string attendee) + { + var trimmed = attendee.Trim(); + var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal); + return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed; + } + + private enum SpeakerIdentityProcessingMode + { + LiveReadOnly, + Final + } +} diff --git a/MeetingAssistant/Speakers/WavSpeakerSnippetExtractor.cs b/MeetingAssistant/Speakers/WavSpeakerSnippetExtractor.cs new file mode 100644 index 0000000..27275da --- /dev/null +++ b/MeetingAssistant/Speakers/WavSpeakerSnippetExtractor.cs @@ -0,0 +1,49 @@ +using MeetingAssistant.Transcription; +using NAudio.Wave; + +namespace MeetingAssistant.Speakers; + +public sealed class WavSpeakerSnippetExtractor : ISpeakerSnippetExtractor +{ + public Task ExtractSnippetAsync( + string audioPath, + IReadOnlyList speakerSegments, + CancellationToken cancellationToken) + { + if (speakerSegments.Count == 0 || !File.Exists(audioPath)) + { + return Task.FromResult([]); + } + + var start = speakerSegments.Min(segment => segment.Start); + var end = speakerSegments.Max(segment => segment.End); + if (end <= start) + { + return Task.FromResult([]); + } + + using var reader = new WaveFileReader(audioPath); + using var output = new MemoryStream(); + using (var writer = new WaveFileWriter(output, reader.WaveFormat)) + { + reader.CurrentTime = start; + var bytesRemaining = reader.WaveFormat.AverageBytesPerSecond * (end - start).TotalSeconds; + var buffer = new byte[reader.WaveFormat.AverageBytesPerSecond]; + while (bytesRemaining > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var bytesToRead = (int)Math.Min(buffer.Length, bytesRemaining); + var bytesRead = reader.Read(buffer, 0, bytesToRead); + if (bytesRead == 0) + { + break; + } + + writer.Write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + + return Task.FromResult(output.ToArray()); + } +} diff --git a/MeetingAssistant/Summary/IMeetingSummaryInstructionBuilder.cs b/MeetingAssistant/Summary/IMeetingSummaryInstructionBuilder.cs new file mode 100644 index 0000000..9c82378 --- /dev/null +++ b/MeetingAssistant/Summary/IMeetingSummaryInstructionBuilder.cs @@ -0,0 +1,10 @@ +using MeetingAssistant.MeetingNotes; + +namespace MeetingAssistant.Summary; + +public interface IMeetingSummaryInstructionBuilder +{ + Task BuildAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs b/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs index 9f69189..42117ef 100644 --- a/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs +++ b/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs @@ -33,7 +33,7 @@ public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactReso return null; } - var meetingNotesFolder = VaultPath.Resolve(options.Vault.MeetingNotesFolder); + var meetingNotesFolder = VaultPath.Resolve(options.Vault, options.Vault.MeetingNotesFolder); if (!Directory.Exists(meetingNotesFolder)) { return null; @@ -65,7 +65,7 @@ public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactReso return null; } - var summariesFolder = VaultPath.Resolve(options.Vault.SummariesFolder); + var summariesFolder = VaultPath.Resolve(options.Vault, options.Vault.SummariesFolder); var requested = Environment.ExpandEnvironmentVariables(summaryPath); var fullPath = Path.IsPathRooted(requested) ? Path.GetFullPath(requested) diff --git a/MeetingAssistant/Summary/MeetingSummaryInstructionBuilder.cs b/MeetingAssistant/Summary/MeetingSummaryInstructionBuilder.cs new file mode 100644 index 0000000..047bebe --- /dev/null +++ b/MeetingAssistant/Summary/MeetingSummaryInstructionBuilder.cs @@ -0,0 +1,135 @@ +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Options; +using YamlDotNet.Serialization; + +namespace MeetingAssistant.Summary; + +public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructionBuilder +{ + public const string DefaultInitialPrompt = """ + You are the Meeting Assistant summary agent. + + Use the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files. + All read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines. + Then write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary. + Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time. + Use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note. + Use add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time. + After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context. + Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files. + The summary note should contain concise sections for summary, decisions, open questions, and next steps. + Keep the output grounded in the source material and explicitly say when a section has no known items. + """; + + private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + + private readonly MeetingAssistantOptions options; + + public MeetingSummaryInstructionBuilder(IOptions options) + { + this.options = options.Value; + } + + public async Task BuildAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt) + ? DefaultInitialPrompt + : options.Agent.InitialPrompt.Trim(); + var projectInstructions = await BuildProjectInstructionsAsync(artifacts, cancellationToken); + return string.IsNullOrWhiteSpace(projectInstructions) + ? instructions + : instructions.TrimEnd() + "\n\n" + projectInstructions; + } + + private async Task BuildProjectInstructionsAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, cancellationToken); + if (projects.Count == 0) + { + return ""; + } + + var blocks = projects.Select(project => + $"# {project.Name}\n\n{project.Instructions.Trim()}"); + return "---\nprojects:\n\n" + string.Join("\n\n", blocks); + } + + private async Task> GetBoundProjectsWithInstructionsAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + var projectNames = await ReadMeetingProjectNamesAsync(artifacts.MeetingNotePath, cancellationToken); + if (projectNames.Count == 0) + { + return []; + } + + var projectsRoot = VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder); + if (!Directory.Exists(projectsRoot)) + { + return []; + } + + var projects = new List(); + foreach (var projectDirectory in Directory.EnumerateDirectories(projectsRoot).Order(StringComparer.OrdinalIgnoreCase)) + { + var projectName = Path.GetFileName(projectDirectory); + if (!projectNames.Contains(projectName)) + { + continue; + } + + var agentsPath = Path.Combine(projectDirectory, "AGENTS.md"); + if (!File.Exists(agentsPath)) + { + continue; + } + + var content = await File.ReadAllTextAsync(agentsPath, cancellationToken); + if (!string.IsNullOrWhiteSpace(content)) + { + projects.Add(new ProjectInstructions(projectName, content)); + } + } + + return projects; + } + + private static async Task> ReadMeetingProjectNamesAsync( + string meetingNotePath, + CancellationToken cancellationToken) + { + if (!File.Exists(meetingNotePath)) + { + return []; + } + + var content = await File.ReadAllTextAsync(meetingNotePath, cancellationToken); + var document = MarkdownDocumentParser.SplitOptional(content); + if (!document.HasFrontmatter) + { + return []; + } + + var frontmatter = YamlDeserializer.Deserialize(document.Frontmatter) + ?? new ProjectFrontmatter(); + return (frontmatter.Projects ?? []) + .Where(project => !string.IsNullOrWhiteSpace(project)) + .Select(project => project.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + private sealed record ProjectInstructions(string Name, string Instructions); + + private sealed class ProjectFrontmatter + { + [YamlMember(Alias = "projects")] + public List? Projects { get; set; } + } +} diff --git a/MeetingAssistant/Summary/MeetingSummaryTools.cs b/MeetingAssistant/Summary/MeetingSummaryTools.cs index 4b8eb53..5a9bf3d 100644 --- a/MeetingAssistant/Summary/MeetingSummaryTools.cs +++ b/MeetingAssistant/Summary/MeetingSummaryTools.cs @@ -1,4 +1,5 @@ using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Transcription; using System.Diagnostics; using System.Text.Json; using YamlDotNet.Serialization; @@ -13,16 +14,21 @@ public sealed class MeetingSummaryTools private readonly MeetingSessionArtifacts artifacts; private readonly MeetingAssistantOptions options; + private readonly IDictationWordStore? dictationWordStore; public MeetingSummaryTools(MeetingSessionArtifacts artifacts) - : this(artifacts, new MeetingAssistantOptions()) + : this(artifacts, new MeetingAssistantOptions(), null) { } - public MeetingSummaryTools(MeetingSessionArtifacts artifacts, MeetingAssistantOptions options) + public MeetingSummaryTools( + MeetingSessionArtifacts artifacts, + MeetingAssistantOptions options, + IDictationWordStore? dictationWordStore = null) { this.artifacts = artifacts; this.options = options; + this.dictationWordStore = dictationWordStore; } public Task ReadTranscript(int? @from = null, int? to = null) @@ -55,7 +61,23 @@ public sealed class MeetingSummaryTools public Task ReadGlossary(int? @from = null, int? to = null) { - return ReadIfExistsAsync(VaultPath.Resolve(options.Vault.DictationWordsPath), @from, to); + return ReadIfExistsAsync(VaultPath.Resolve(options.Vault, options.Vault.DictationWordsPath), @from, to); + } + + public async Task AddDictationWord(string word) + { + if (dictationWordStore is null) + { + return "Dictation word store is not available."; + } + + if (string.IsNullOrWhiteSpace(word)) + { + return "Refused: word must not be empty."; + } + + var words = await dictationWordStore.AddWordAsync(word, CancellationToken.None); + return $"Added '{word.Trim()}'. Dictionary now contains {words.Count} word(s)."; } public async Task WriteSummary(string markdown, string? title = null) @@ -436,7 +458,7 @@ public sealed class MeetingSummaryTools private string GetProjectsRoot() { - return VaultPath.Resolve(options.Vault.ProjectsFolder); + return VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder); } private static async Task RunRipgrepAsync( diff --git a/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs b/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs index 41074f7..23653a5 100644 --- a/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs +++ b/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs @@ -1,4 +1,5 @@ using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Transcription; using Microsoft.Agents.AI; using Microsoft.Agents.AI.Compaction; using Microsoft.Extensions.AI; @@ -9,38 +10,30 @@ namespace MeetingAssistant.Summary; #pragma warning disable MAAI001 public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline { - private const string Instructions = """ - You are the Meeting Assistant summary agent. - - Use the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files. - All read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines. - Then write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary. - Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time. - Use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note. - After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context. - Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files. - The summary note should contain concise sections for summary, decisions, open questions, and next steps. - Keep the output grounded in the source material and explicitly say when a section has no known items. - """; - private readonly MeetingAssistantOptions options; private readonly ILoggerFactory loggerFactory; private readonly ILogger logger; private readonly IServiceProvider services; private readonly IMeetingSummaryFailureWriter failureWriter; + private readonly IMeetingSummaryInstructionBuilder instructionBuilder; + private readonly IDictationWordStore dictationWordStore; public OpenAiMeetingSummaryAgentPipeline( IOptions options, ILoggerFactory loggerFactory, ILogger logger, IServiceProvider services, - IMeetingSummaryFailureWriter failureWriter) + IMeetingSummaryFailureWriter failureWriter, + IMeetingSummaryInstructionBuilder instructionBuilder, + IDictationWordStore dictationWordStore) { this.options = options.Value; this.loggerFactory = loggerFactory; this.logger = logger; this.services = services; this.failureWriter = failureWriter; + this.instructionBuilder = instructionBuilder; + this.dictationWordStore = dictationWordStore; } public async Task RunAsync( @@ -49,7 +42,8 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline { var agentOptions = options.Agent; var key = ResolveApiKey(agentOptions); - var tools = CreateTools(new MeetingSummaryTools(artifacts, options)); + var tools = CreateTools(new MeetingSummaryTools(artifacts, options, dictationWordStore)); + var instructions = await instructionBuilder.BuildAsync(artifacts, cancellationToken); using var compactionSummaryClient = new LiteLlmResponsesChatClient( new Uri(agentOptions.Endpoint), key, @@ -76,7 +70,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline .UseFunctionInvocation() .Build() .AsAIAgent( - Instructions, + instructions, "meeting_summary", "Writes the markdown summary note for a captured meeting.", tools, @@ -135,6 +129,10 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline tools.ReadGlossary, "read_glossary", "Read the glossary or vocabulary context for this meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range. May be empty."), + AIFunctionFactory.Create( + tools.AddDictationWord, + "add_dictation_word", + "Add one domain term, name, acronym, or unusual word to the transcription dictionary with deduplication."), AIFunctionFactory.Create( tools.WriteSummary, "write_summary", diff --git a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs index 6934881..5bc66d2 100644 --- a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs @@ -22,6 +22,7 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions pipelineOptions, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { await using var enumerator = audio.GetAsyncEnumerator(cancellationToken); @@ -38,7 +39,8 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc (byte)firstChunk.Channels)); using var audioConfig = AudioConfig.FromStreamInput(pushStream); var speechConfig = CreateSpeechConfig(); - using var recognizer = new ConversationTranscriber(speechConfig, audioConfig); + using var recognizer = CreateTranscriber(speechConfig, audioConfig); + AddPhraseList(recognizer, pipelineOptions.DictationWords); var segments = Channel.CreateUnbounded(); recognizer.Transcribed += (_, args) => @@ -94,7 +96,51 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc await pumpTask.WaitAsync(cancellationToken); } - private SpeechConfig CreateSpeechConfig() + internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig) + { + var languages = GetAutoDetectLanguages(options.AzureSpeech); + if (languages.Count == 0) + { + return new ConversationTranscriber(speechConfig, audioConfig); + } + + logger.LogInformation( + "Azure Speech auto language detection enabled for {Languages}", + string.Join(", ", languages)); + return new ConversationTranscriber( + speechConfig, + AutoDetectSourceLanguageConfig.FromLanguages(languages.ToArray()), + audioConfig); + } + + internal void AddPhraseList(ConversationTranscriber recognizer, IReadOnlyList? words) + { + var phrases = NormalizeDictationWords(words); + if (phrases.Count == 0) + { + return; + } + + var phraseList = PhraseListGrammar.FromRecognizer(recognizer); + foreach (var phrase in phrases) + { + phraseList.AddPhrase(phrase); + } + + phraseList.SetWeight(options.AzureSpeech.PhraseListWeight); + logger.LogInformation("Configured Azure Speech phrase list with {PhraseCount} phrase(s)", phrases.Count); + } + + internal static IReadOnlyList NormalizeDictationWords(IReadOnlyList? words) + { + return (words ?? []) + .Select(word => word.Trim()) + .Where(word => !string.IsNullOrWhiteSpace(word)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + internal SpeechConfig CreateSpeechConfig() { var azure = options.AzureSpeech; var key = ResolveKey(azure); @@ -104,10 +150,19 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc $"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}."); } - var speechConfig = string.IsNullOrWhiteSpace(azure.Region) - ? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key) - : SpeechConfig.FromSubscription(key, azure.Region); - speechConfig.SpeechRecognitionLanguage = azure.Language; + var speechConfig = SpeechConfig.FromEndpoint(GetEndpoint(azure), key); + var autoDetectLanguages = GetAutoDetectLanguages(azure); + if (autoDetectLanguages.Count == 0) + { + speechConfig.SpeechRecognitionLanguage = azure.Language; + } + else + { + speechConfig.SetProperty( + PropertyId.SpeechServiceConnection_LanguageIdMode, + NormalizeLanguageIdMode(azure.LanguageIdMode)); + } + speechConfig.OutputFormat = OutputFormat.Detailed; speechConfig.SetProperty( PropertyId.SpeechServiceResponse_DiarizeIntermediateResults, @@ -115,6 +170,42 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc return speechConfig; } + internal static IReadOnlyList GetAutoDetectLanguages(AzureSpeechOptions options) + { + if (!options.Language.Equals("auto", StringComparison.OrdinalIgnoreCase)) + { + return []; + } + + return options.AutoDetectLanguages + .Select(language => language.Trim()) + .Where(language => !string.IsNullOrWhiteSpace(language)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + internal static Uri GetEndpoint(AzureSpeechOptions options) + { + if (!string.IsNullOrWhiteSpace(options.Endpoint)) + { + return new Uri(options.Endpoint); + } + + if (string.IsNullOrWhiteSpace(options.Region)) + { + throw new InvalidOperationException("Azure Speech region or endpoint must be configured."); + } + + return new Uri($"wss://{options.Region}.stt.speech.microsoft.com/speech/universal/v2"); + } + + internal static string NormalizeLanguageIdMode(string? value) + { + return value?.Trim().Equals("AtStart", StringComparison.OrdinalIgnoreCase) == true + ? "AtStart" + : "Continuous"; + } + private static async Task PumpAudioAsync( AudioChunk firstChunk, IAsyncEnumerator audio, @@ -176,15 +267,21 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc : speakerId; } - private static string ResolveKey(AzureSpeechOptions options) + internal static string ResolveKey(AzureSpeechOptions options) { if (!string.IsNullOrWhiteSpace(options.Key)) { return options.Key; } - return string.IsNullOrWhiteSpace(options.KeyEnv) - ? "" - : Environment.GetEnvironmentVariable(options.KeyEnv) ?? ""; + if (string.IsNullOrWhiteSpace(options.KeyEnv)) + { + return ""; + } + + return Environment.GetEnvironmentVariable(options.KeyEnv) ?? + Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.User) ?? + Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.Machine) ?? + ""; } } diff --git a/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs index 1c172b5..cc56609 100644 --- a/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs @@ -26,10 +26,11 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { var segments = Channel.CreateUnbounded(); - var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None); + var session = Task.Run(() => RunSessionAsync(audio, options, segments.Writer, cancellationToken), CancellationToken.None); await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) { @@ -41,6 +42,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti private async Task RunSessionAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions pipelineOptions, ChannelWriter segments, CancellationToken cancellationToken) { @@ -61,7 +63,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken); - await SendStartAsync(connection, firstChunk, cancellationToken); + await SendStartAsync(connection, firstChunk, pipelineOptions.DictationWords, cancellationToken); await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken); while (await enumerator.MoveNextAsync()) @@ -139,6 +141,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti private async Task SendStartAsync( IFunAsrWebSocketConnection connection, AudioChunk firstChunk, + IReadOnlyList? dictationWords, CancellationToken cancellationToken) { var message = new Dictionary @@ -152,24 +155,17 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti ["wav_name"] = options.FunAsr.WavName, ["wav_format"] = "pcm", ["is_speaking"] = true, - ["hotwords"] = await LoadHotwordsAsync(cancellationToken), + ["hotwords"] = BuildHotwords(dictationWords), ["itn"] = options.FunAsr.Itn }; await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken); } - private async Task LoadHotwordsAsync(CancellationToken cancellationToken) + internal static string BuildHotwords(IReadOnlyList? dictationWords) { - var path = VaultPath.Resolve(options.Vault.DictationWordsPath); - if (!File.Exists(path)) - { - return ""; - } - - var words = await File.ReadAllLinesAsync(path, cancellationToken); - var hotwords = words - .Select(line => line.Trim().TrimStart('-', '*').Trim()) + var hotwords = (dictationWords ?? []) + .Select(word => word.Trim()) .Where(line => !string.IsNullOrWhiteSpace(line)) .Distinct(StringComparer.OrdinalIgnoreCase) .ToDictionary(word => word, _ => 20); diff --git a/MeetingAssistant/Transcription/IDictationWordStore.cs b/MeetingAssistant/Transcription/IDictationWordStore.cs new file mode 100644 index 0000000..cc2903c --- /dev/null +++ b/MeetingAssistant/Transcription/IDictationWordStore.cs @@ -0,0 +1,8 @@ +namespace MeetingAssistant.Transcription; + +public interface IDictationWordStore +{ + Task> ReadWordsAsync(CancellationToken cancellationToken); + + Task> AddWordAsync(string word, CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs index 7a716d1..fadc74d 100644 --- a/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs +++ b/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs @@ -6,6 +6,10 @@ public interface ISpeechRecognitionPipeline : IAsyncDisposable { Task InitializeAsync(CancellationToken cancellationToken); + Task InitializeAsync( + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken); + Task WaitUntilReadyAsync(CancellationToken cancellationToken); ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken); @@ -25,7 +29,9 @@ public interface ISpeechRecognitionPipelineFactory ISpeechRecognitionPipeline Create(); } -public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null) +public sealed record SpeechRecognitionPipelineOptions( + int? NumSpeakers = null, + IReadOnlyList? DictationWords = null) { public static SpeechRecognitionPipelineOptions Default { get; } = new(); } diff --git a/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs index aac211e..7cc2685 100644 --- a/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs @@ -6,5 +6,6 @@ public interface IStreamingTranscriptionProvider { IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions options, CancellationToken cancellationToken); } diff --git a/MeetingAssistant/Transcription/MarkdownDictationWordStore.cs b/MeetingAssistant/Transcription/MarkdownDictationWordStore.cs new file mode 100644 index 0000000..06537ac --- /dev/null +++ b/MeetingAssistant/Transcription/MarkdownDictationWordStore.cs @@ -0,0 +1,68 @@ +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class MarkdownDictationWordStore : IDictationWordStore +{ + private readonly MeetingAssistantOptions options; + + public MarkdownDictationWordStore(IOptions options) + { + this.options = options.Value; + } + + public async Task> ReadWordsAsync(CancellationToken cancellationToken) + { + var path = GetPath(); + if (!File.Exists(path)) + { + return []; + } + + var lines = await File.ReadAllLinesAsync(path, cancellationToken); + return Normalize(lines); + } + + public async Task> AddWordAsync(string word, CancellationToken cancellationToken) + { + var path = GetPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var existing = File.Exists(path) + ? await File.ReadAllLinesAsync(path, cancellationToken) + : []; + var words = Normalize(existing.Append(word)); + + await File.WriteAllTextAsync( + path, + string.Join(Environment.NewLine, words.Select(value => $"- {value}")) + Environment.NewLine, + cancellationToken); + return words; + } + + private string GetPath() + { + return VaultPath.Resolve(options.Vault, options.Vault.DictationWordsPath); + } + + internal static IReadOnlyList Normalize(IEnumerable lines) + { + return lines + .Select(ParseLine) + .Where(word => !string.IsNullOrWhiteSpace(word)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string ParseLine(string line) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("- ", StringComparison.Ordinal) || + trimmed.StartsWith("* ", StringComparison.Ordinal)) + { + trimmed = trimmed[2..].Trim(); + } + + return trimmed.Trim('`', '"', '\'').Trim(); + } +} diff --git a/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs index da8e12f..8f02916 100644 --- a/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs +++ b/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs @@ -9,6 +9,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip private readonly Channel audio = Channel.CreateUnbounded(); private readonly Channel liveTranscript = Channel.CreateUnbounded(); private readonly List liveSegments = []; + private SpeechRecognitionPipelineOptions options = SpeechRecognitionPipelineOptions.Default; private Task? transcriptionTask; private bool initialized; @@ -18,6 +19,13 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip } public virtual Task InitializeAsync(CancellationToken cancellationToken) + { + return InitializeAsync(SpeechRecognitionPipelineOptions.Default, cancellationToken); + } + + public virtual Task InitializeAsync( + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) { if (initialized) { @@ -25,6 +33,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip } initialized = true; + this.options = options; transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None); return Task.CompletedTask; } @@ -90,6 +99,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip { await foreach (var segment in transcriptionProvider.TranscribeAsync( audio.Reader.ReadAllAsync(cancellationToken), + options, cancellationToken)) { liveSegments.Add(segment); diff --git a/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs index be0e46f..81533eb 100644 --- a/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs @@ -20,6 +20,7 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans public async IAsyncEnumerable TranscribeAsync( IAsyncEnumerable audio, + SpeechRecognitionPipelineOptions pipelineOptions, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath)); @@ -29,9 +30,7 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans } using var factory = WhisperFactory.FromPath(modelPath); - using var processor = factory.CreateBuilder() - .WithLanguage(options.WhisperLocal.Language) - .Build(); + using var processor = CreateProcessor(factory, pipelineOptions.DictationWords); var window = new List(); var windowDuration = TimeSpan.Zero; @@ -119,6 +118,37 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans } } + private WhisperProcessor CreateProcessor( + WhisperFactory factory, + IReadOnlyList? dictationWords) + { + var builder = factory.CreateBuilder() + .WithLanguage(options.WhisperLocal.Language); + var prompt = BuildPrompt(dictationWords); + if (!string.IsNullOrWhiteSpace(prompt)) + { + builder.WithPrompt(prompt) + .WithCarryInitialPrompt(true); + logger.LogInformation( + "Configured Whisper prompt with {WordCount} dictation word(s)", + dictationWords?.Count ?? 0); + } + + return builder.Build(); + } + + internal static string BuildPrompt(IReadOnlyList? dictationWords) + { + var words = (dictationWords ?? []) + .Select(word => word.Trim()) + .Where(word => !string.IsNullOrWhiteSpace(word)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + return words.Count == 0 + ? "" + : "Prefer these domain terms and acronyms: " + string.Join(", ", words) + "."; + } + private static bool IsControlMarker(string text) { return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]"; diff --git a/MeetingAssistant/VaultPath.cs b/MeetingAssistant/VaultPath.cs index 818f06f..f5386da 100644 --- a/MeetingAssistant/VaultPath.cs +++ b/MeetingAssistant/VaultPath.cs @@ -14,4 +14,15 @@ public static class VaultPath return Path.GetFullPath(expanded); } + + public static string Resolve(VaultOptions vault, string path) + { + var expanded = Environment.ExpandEnvironmentVariables(path); + if (Path.IsPathRooted(expanded) || expanded.StartsWith("~", StringComparison.Ordinal)) + { + return Resolve(expanded); + } + + return Path.GetFullPath(Path.Combine(Resolve(vault.BaseFolder), expanded)); + } } diff --git a/MeetingAssistant/appsettings.json b/MeetingAssistant/appsettings.json index ebad478..3e0984c 100644 --- a/MeetingAssistant/appsettings.json +++ b/MeetingAssistant/appsettings.json @@ -4,24 +4,32 @@ "Toggle": "Ctrl+Alt+M" }, "Vault": { - "TranscriptsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Transcripts", - "MeetingNotesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Notes", - "AssistantContextFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Assistant Context", - "SummariesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Summaries", - "ProjectsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Projects", - "DictationWordsPath": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\dictation-words.md" + "BaseFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Exxeta", + "TranscriptsFolder": "Meetings\\Transcripts", + "MeetingNotesFolder": "Meetings\\Notes", + "AssistantContextFolder": "Meetings\\Assistant Context", + "SummariesFolder": "Meetings\\Summaries", + "ProjectsFolder": "Projects", + "DictationWordsPath": "C:\\Users\\masc3\\OpenCloud\\Persönlich\\Vault\\Exxeta\\Meetings\\Dictionary.md" }, "Recording": { - "TranscriptionProvider": "funasr", + "TranscriptionProvider": "azure-speech", "SampleRate": 16000, "Channels": 1, "StopProcessingTimeout": "00:10:00", + "MetadataLookupTimeout": "00:00:10", + "BackgroundMetadataLookupTimeout": "00:01:00", + "OpenMeetingNoteDelay": "00:00:01", "TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings" }, "FunAsr": { "Endpoint": "ws://127.0.0.1:10095", "Mode": "2pass", - "ChunkSize": [5, 10, 5], + "ChunkSize": [ + 5, + 10, + 5 + ], "ChunkInterval": 10, "EncoderChunkLookBack": 4, "DecoderChunkLookBack": 1, @@ -72,12 +80,30 @@ } }, "AzureSpeech": { - "Endpoint": "https://germanywestcentral.api.cognitive.microsoft.com/", + "Endpoint": "wss://germanywestcentral.stt.speech.microsoft.com/speech/universal/v2", "Region": "germanywestcentral", - "Language": "en-US", + "Language": "de-DE", + "AutoDetectLanguages": [ + "de-DE" + ], + "LanguageIdMode": "Continuous", "KeyEnv": "AZURE_SPEECH_KEY", "RecognitionStopTimeout": "00:00:10", - "DiarizeIntermediateResults": true + "DiarizeIntermediateResults": true, + "PhraseListWeight": 1.5 + }, + "SpeakerIdentification": { + "Enabled": true, + "DatabasePath": "%LOCALAPPDATA%\\MeetingAssistant\\SpeakerIdentity\\speaker-identities.db", + "InitialDelay": "00:02:00", + "Interval": "00:02:00", + "MatchBatchSize": 6, + "MaxMatchCandidates": 100, + "MatchIdentityActiveAge": "365.00:00:00", + "MaxSnippetsPerSpeaker": 3, + "SilenceBetweenSnippetsSeconds": 1, + "LiveSampleBufferDuration": "00:10:00", + "MergeRecentIdentityAge": "14.00:00:00" }, "Agent": { "Endpoint": "https://litellm.schweigert.cloud", @@ -90,8 +116,9 @@ "ContextWindowTokens": 128000, "MaxOutputTokens": 8192, "EnableCompaction": true, - "CompactionRemainingRatio": 0.10, - "ResponsesCompactPath": "responses/compact" + "CompactionRemainingRatio": 0.1, + "ResponsesCompactPath": "responses/compact", + "InitialPrompt": "You are the Meeting Assistant summary agent.\n\nUse the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files.\nAll read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines.\nThen write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary.\nUse read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.\nUse read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note.\nUse add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time.\nAfter writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.\nUse list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.\nThe summary note should contain concise sections for summary, decisions, open questions, and next steps.\nKeep the output grounded in the source material and explicitly say when a section has no known items." }, "Api": { "PublicBaseUrl": "http://localhost:5090" diff --git a/README.md b/README.md index 579e2f7..760ad21 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,26 @@ Run the server: dotnet run --project MeetingAssistant ``` +## Local Autostart and Restart + +For the local Windows workstation, use the snippets restart helper instead of running the app directly when you want a durable background instance: + +```powershell +powershell -ExecutionPolicy Bypass -File C:\Manuel\snippets\restart-meeting-assistant.ps1 +``` + +The restart helper publishes `MeetingAssistant\MeetingAssistant.csproj` as `net10.0-windows` into a fresh temp folder under `C:\Manuel\meeting-assistant\tmp\meeting-assistant-runtime`. Only after that publish succeeds does it stop any active Meeting Assistant instance on port `5090` and start the newly published executable detached from the script. + +The detached starter writes logs and the last process id to: + +```text +%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.out.log +%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.err.log +%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.pid +``` + +This script is also the intended autostart target. Add a Windows Startup shortcut to `C:\Manuel\snippets\restart-meeting-assistant.ps1` when the app should rebuild and restart automatically after login. + The initial endpoints are: ```text diff --git a/openspec/changes/add-speaker-identity-learning/proposal.md b/openspec/changes/add-speaker-identity-learning/proposal.md new file mode 100644 index 0000000..686e1db --- /dev/null +++ b/openspec/changes/add-speaker-identity-learning/proposal.md @@ -0,0 +1,17 @@ +# Add Speaker Identity Learning + +## Why +Meeting Assistant currently receives diarized speaker labels from ASR backends, but those labels are backend-local names such as `Guest01` or `Speaker 0`. Users need transcripts to converge toward real attendee names without manually maintaining a roster. + +## What Changes +- Add a local SQLite-backed speaker identity store under the user's application data folder. +- Store a small bounded set of WAV snippets per learned speaker identity. +- Match unknown diarized speakers against known identities during and after transcription using a dedicated Azure Speech diarization verifier component separate from the configured ASR pipeline. +- Confirm matches before applying them, then rewrite transcript speaker labels and use the known name for later segments. +- Learn new identities automatically from unmatched diarized speakers using meeting attendees as candidate names. +- Eliminate candidate names across future meetings until a canonical speaker name is known. + +## Impact +- Adds EF Core SQLite as a runtime dependency. +- Adds speaker identity services and a dedicated Azure Speech identity matching component to the transcription flow. +- Adds persisted local voice snippets; these are explicit identity artifacts, not temporary full recordings. diff --git a/openspec/changes/add-speaker-identity-learning/specs/meeting-transcription/spec.md b/openspec/changes/add-speaker-identity-learning/specs/meeting-transcription/spec.md new file mode 100644 index 0000000..0436bb7 --- /dev/null +++ b/openspec/changes/add-speaker-identity-learning/specs/meeting-transcription/spec.md @@ -0,0 +1,158 @@ +## ADDED Requirements + +### Requirement: Meeting Assistant learns speaker identities locally +Meeting Assistant SHALL maintain a local SQLite speaker identity database in the user's application data folder. + +The speaker identity database SHALL store speaker identities, optional canonical names, aliases, candidate names, a transcription participation counter, and a bounded set of WAV snippets per identity. + +Each speaker identity SHALL store a last-modified timestamp used by active-age filtering, and Meeting Assistant SHALL update it whenever the identity is created or modified by identification, candidate updates, snippet changes, or merge operations. + +The configured maximum snippet count per identity SHALL prevent unbounded growth. + +#### Scenario: Unknown speaker is learned from meeting attendees +- **WHEN** a finished transcript contains an unmatched diarized speaker and the meeting note has attendees +- **THEN** Meeting Assistant stores a new unnamed speaker identity with candidate names from the attendees that were not already matched in that meeting + +#### Scenario: Speaker snippets are bounded +- **WHEN** Meeting Assistant adds a snippet for an identity that already has the configured maximum number of snippets +- **THEN** Meeting Assistant does not store more snippets for that identity + +#### Scenario: Identity modification updates active-age timestamp +- **WHEN** Meeting Assistant creates, identifies, updates candidates for, stores snippets for, or merges a speaker identity +- **THEN** Meeting Assistant updates that identity's last-modified timestamp + +### Requirement: Speaker identities can be merged diagnostically +Meeting Assistant SHALL expose a diagnostic endpoint that merges duplicate speaker identities. + +The merge process SHALL compare recently-created identities, using a configurable recent age that defaults to two weeks, against all other identities in matcher batches bounded by the configured batch size. + +The merge process SHALL require a match and a second validation match using a different source sample before merging two identities. + +When identities are merged, Meeting Assistant SHALL retain one identity, move useful names from the merged identity into aliases, combine transcription counts, and retain a bounded set of snippets from both identities. + +#### Scenario: Recently-created duplicate identity is merged +- **GIVEN** a recently-created identity and an older identity have matching speaker snippets +- **WHEN** the diagnostic merge endpoint is triggered +- **THEN** Meeting Assistant validates the match twice with different source snippets +- **AND** merges the recent identity into the older identity +- **AND** stores the recent identity name as an alias on the retained identity + +#### Scenario: Old identities are not used as merge sources +- **GIVEN** two identities older than the configured recent age +- **WHEN** the diagnostic merge endpoint is triggered +- **THEN** Meeting Assistant does not compare them as source identities + +### Requirement: Speaker candidates are eliminated across meetings +Meeting Assistant SHALL update candidate names for a matched unnamed identity using the intersection of its existing candidate names and the current meeting attendees. + +Meeting Assistant SHALL treat identity aliases as acceptable names during candidate elimination and when excluding already-matched attendees from new unmatched speaker candidates. + +When the candidate intersection leaves exactly one candidate, Meeting Assistant SHALL promote that candidate to the canonical speaker name. + +When the candidate intersection is empty, Meeting Assistant SHALL replace the oldest stored snippet for that identity and reset candidates from the current meeting attendees, because the previous snippet may have been dirty. + +#### Scenario: Candidate elimination promotes canonical name +- **GIVEN** an unnamed identity has candidate names `John` and `Mike` +- **WHEN** that identity matches a speaker in a later meeting with attendees `Jane`, `John`, and `Chris` +- **THEN** Meeting Assistant removes `Mike` from the identity candidates and promotes `John` as the canonical name + +#### Scenario: Alias participates in candidate elimination +- **GIVEN** an unnamed identity has candidate name `Michael` and alias `Mike` +- **WHEN** that identity matches a speaker in a later meeting with attendee `Mike` +- **THEN** Meeting Assistant treats `Mike` as matching `Michael` +- **AND** promotes `Michael` as the canonical name + +#### Scenario: Empty candidate intersection resets candidates +- **GIVEN** an unnamed identity has candidate names `John` and `Mike` +- **WHEN** that identity matches a speaker in a later meeting with attendees `Jane` and `Chris` +- **THEN** Meeting Assistant resets the identity candidates to `Jane` and `Chris` +- **AND** replaces the oldest stored snippet for that identity with the current speaker snippet + +### Requirement: Speaker identity matches relabel transcripts +Meeting Assistant SHALL attempt to match unknown diarized speaker snippets against known speaker identities ordered by transcription participation count. + +Speaker identity matching SHALL use a dedicated Azure Speech diarization verifier component rather than the configured speech recognition pipeline. + +The matcher SHALL test at most the configured batch size of known people per diarization session and continue with later batches until a match is found or no candidates remain. + +The matcher SHALL prioritize identities whose canonical name or aliases match current meeting attendees. + +After attendee-matched identities, the matcher SHALL order identities by transcription participation count, filter out non-attendee identities whose last update is older than the configured active age, and cap the candidate set at the configured maximum match candidate count. + +When a match is confirmed and the identity has a canonical name, Meeting Assistant SHALL rewrite finished transcript segments for that diarized speaker with the canonical name. + +When a match is confirmed and the matched speaker is not already listed in meeting note attendees by display name or alias, Meeting Assistant SHALL add the speaker display name to the attendee list. + +#### Scenario: Finished transcript is relabeled after a confirmed match +- **GIVEN** the speaker identity database contains canonical speaker `Chris` +- **WHEN** a finished transcript has diarized speaker `Guest03` and the matching backend confirms it is `Chris` +- **THEN** Meeting Assistant rewrites `Guest03` segments in the transcript as `Chris` + +#### Scenario: Confirmed match adds missing attendee +- **GIVEN** the speaker identity database contains canonical speaker `Chris` +- **AND** the meeting note attendees do not contain `Chris` or one of that identity's aliases +- **WHEN** live or final speaker matching confirms a diarized speaker is `Chris` +- **THEN** Meeting Assistant adds `Chris` to the meeting note attendees + +#### Scenario: Confirmed match does not duplicate attendee aliases +- **GIVEN** the speaker identity database contains canonical speaker `Christopher` with alias `Chris` +- **AND** the meeting note attendees already contain `Chris ` +- **WHEN** live or final speaker matching confirms a diarized speaker is `Christopher` +- **THEN** Meeting Assistant does not add another attendee for `Christopher` + +#### Scenario: Attendee identities are tried first +- **GIVEN** the meeting attendees include names matching known identity display names or aliases +- **WHEN** Meeting Assistant tries to identify a diarized speaker +- **THEN** attendee-matched identities are tried before non-attendee identities + +#### Scenario: Stale non-attendee identities are skipped +- **GIVEN** a known identity has not matched within the configured active age +- **AND** that identity does not match the current meeting attendees +- **WHEN** Meeting Assistant tries to identify a diarized speaker +- **THEN** that stale identity is not included in the match candidates + +### Requirement: Speaker matching runs during active transcription +Meeting Assistant SHALL start speaker identity matching only after the configured initial transcription duration has elapsed. + +For backends that emit live diarized transcript segments, Meeting Assistant SHALL keep a bounded in-memory sliding audio buffer with chunk timestamps and extract candidate WAV snippets from that buffer when live diarized segments arrive. + +Meeting Assistant SHALL keep only the configured best candidate snippets per diarized speaker in memory. Better snippets SHALL be preferred when the segment looks like a continuous medium-length sentence. + +Meeting Assistant SHALL periodically match unresolved diarized speaker samples while transcription is active and attempt to match them against the local identity database. + +Meeting Assistant SHALL run live matching incrementally at the configured interval only when at least one new unmapped diarized speaker sample appears or the meeting note attendee frontmatter changes while unmapped speaker samples still exist. + +When a speaker is matched during transcription, Meeting Assistant SHALL rewrite already-written live transcript segments for that diarized speaker and write future transcript segments using the canonical name. + +Live speaker matching SHALL be read-only with respect to the speaker identity database. Candidate elimination, canonical promotion, transcription counters, stored snippet updates, and new unmatched identity creation SHALL happen only after transcription is finished, using the latest meeting note frontmatter. + +For backends that only provide diarization after finalization, Meeting Assistant SHALL defer speaker identity matching until finished diarization is available, extract candidate snippets from the completed temporary recording, complete identity matching, and only then allow summary generation to start. + +#### Scenario: Matching waits for useful speech duration +- **WHEN** transcription has been active for less than the configured speaker identification initial delay +- **THEN** Meeting Assistant does not run speaker identity matching yet + +#### Scenario: Live matching uses in-memory speaker samples +- **WHEN** a live diarized transcript segment identifies an unresolved speaker +- **THEN** Meeting Assistant extracts a WAV snippet for that segment from the in-memory sliding audio buffer +- **AND** uses retained speaker samples for live identity matching without reading the temporary recording file + +#### Scenario: Live match rewrites current and future transcript writes +- **WHEN** periodic matching confirms that diarized speaker `Guest03` is canonical speaker `Chris` +- **THEN** already-written live transcript segments for `Guest03` are rewritten as `Chris` +- **AND** later live transcript segments for `Guest03` are written as `Chris` + +#### Scenario: New live speaker triggers another identification round +- **GIVEN** live matching already checked the current unresolved speaker samples +- **WHEN** a new unmapped diarized speaker sample appears +- **THEN** Meeting Assistant runs another live matching round at the next configured interval + +#### Scenario: Attendee changes trigger another identification round +- **GIVEN** live matching already checked unresolved speaker samples +- **WHEN** the meeting note attendee frontmatter changes +- **THEN** Meeting Assistant runs another live matching round at the next configured interval using the latest attendees + +#### Scenario: Live matching does not make final identity decisions +- **WHEN** live matching finds a possible speaker identity during transcription +- **THEN** Meeting Assistant does not change candidate names, promote canonical names, increment counters, store snippets, or create unmatched identities +- **AND** the final speaker identity pass uses the latest meeting note attendees after transcription finishes diff --git a/openspec/changes/add-speaker-identity-learning/tasks.md b/openspec/changes/add-speaker-identity-learning/tasks.md new file mode 100644 index 0000000..2a9d540 --- /dev/null +++ b/openspec/changes/add-speaker-identity-learning/tasks.md @@ -0,0 +1,34 @@ +## 1. Specification +- [x] 1.1 Define requirements for local speaker identity storage and candidate elimination. +- [x] 1.2 Define requirements for live periodic matching and transcript relabeling. + +## 2. Implementation +- [x] 2.1 Add SQLite speaker identity database in app data. +- [x] 2.2 Add speaker snippet extraction and matching orchestration. +- [x] 2.3 Learn unmatched speaker identities from meeting attendees at transcript completion. +- [x] 2.4 Eliminate candidate names on future matches and promote a canonical name. +- [x] 2.5 Relabel finished transcript segments when canonical identity matches are found. +- [x] 2.6 Add live periodic matching after an initial delay and apply matches to future transcript writes. +- [x] 2.7 Keep timestamped in-memory audio samples for live diarized speaker matching and reserve recording-file extraction for final-only diarization. +- [x] 2.8 Add speaker aliases and a diagnostic duplicate-identity merge process. +- [x] 2.9 Use aliases during candidate elimination and unmatched-candidate filtering. +- [x] 2.10 Keep live speaker matching read-only so final identity decisions use the latest meeting note frontmatter. +- [x] 2.11 Rewrite already-written live transcript segments when a live speaker match is found. +- [x] 2.12 Prioritize attendee-matched speaker identities, filter stale non-attendees, and cap match candidates. +- [x] 2.13 Run live speaker identification incrementally when new unmapped speakers appear or attendees change. +- [x] 2.14 Maintain speaker identity last-modified timestamps for active-age filtering. +- [x] 2.15 Add identified speaker display names to meeting note attendees without duplicating display names or aliases. + +## 3. Validation +- [x] 3.1 Add tests for candidate creation, elimination, canonical promotion, and reset on empty candidate intersection. +- [x] 3.2 Add tests for recording coordinator transcript relabel integration. +- [x] 3.3 Add tests for in-memory speaker samples and rolling audio buffer extraction. +- [x] 3.4 Add tests for duplicate speaker identity merging and the diagnostic endpoint. +- [x] 3.5 Add tests for alias-aware candidate elimination. +- [x] 3.6 Add tests that live matching does not mutate speaker identities. +- [x] 3.7 Add tests that live speaker matches relabel current and future transcript segments. +- [x] 3.8 Add tests for speaker identity candidate ordering, active-age filtering, and candidate caps. +- [x] 3.9 Add tests for incremental live speaker identification triggers. +- [x] 3.10 Add tests for speaker identity last-modified timestamp creation, updates, and schema upgrade. +- [x] 3.11 Add tests for adding matched speaker identities to meeting attendees and alias duplicate suppression. +- [x] 3.12 Run `dotnet test`. diff --git a/openspec/changes/configure-summary-agent-instructions/proposal.md b/openspec/changes/configure-summary-agent-instructions/proposal.md new file mode 100644 index 0000000..d9743c9 --- /dev/null +++ b/openspec/changes/configure-summary-agent-instructions/proposal.md @@ -0,0 +1,14 @@ +# Configure Summary Agent Instructions + +## Why +The summary agent prompt is currently hard-coded, so changing agent behavior requires code changes. Project-specific agent instructions also live in project folders and should be supplied to the summarizer when a meeting is bound to those projects. + +## What Changes +- Add a configurable summary agent initial prompt in appsettings. +- Keep the current prompt in code as the default when settings do not provide one. +- Append project instruction context for meeting-bound projects. +- For each bound project with an `AGENTS.md` file in the project folder, append the project name and file content under a `projects:` section. + +## Impact +- Changes summary agent prompt construction. +- Reads configured project folders and current meeting note frontmatter before starting the summary agent. diff --git a/openspec/changes/configure-summary-agent-instructions/specs/meeting-summary/spec.md b/openspec/changes/configure-summary-agent-instructions/specs/meeting-summary/spec.md new file mode 100644 index 0000000..08cd6a0 --- /dev/null +++ b/openspec/changes/configure-summary-agent-instructions/specs/meeting-summary/spec.md @@ -0,0 +1,33 @@ +## ADDED Requirements + +### Requirement: Summary agent instructions are configurable +Meeting Assistant SHALL allow the summary agent initial prompt to be configured through application settings. + +When no configured initial prompt is supplied, Meeting Assistant SHALL use the built-in default prompt. + +#### Scenario: Configured initial prompt is used +- **WHEN** `MeetingAssistant:Agent:InitialPrompt` is configured with non-empty content +- **THEN** the summary agent uses that configured prompt as its base instructions + +#### Scenario: Empty initial prompt falls back to built-in default +- **WHEN** `MeetingAssistant:Agent:InitialPrompt` is missing or blank +- **THEN** the summary agent uses the built-in default instructions + +### Requirement: Summary agent receives bound project instructions +Meeting Assistant SHALL append project instructions to the summary agent instructions for projects bound to the meeting note frontmatter. + +For each bound project folder under the configured projects folder, when an `AGENTS.md` file exists directly in that project folder, Meeting Assistant SHALL append the project name and the `AGENTS.md` content under a project instructions section. + +When no bound projects have `AGENTS.md`, Meeting Assistant SHALL not append the project instructions section. + +#### Scenario: Project AGENTS files are appended +- **GIVEN** the meeting note frontmatter lists projects `Alpha` and `Beta` +- **AND** both configured project folders contain `AGENTS.md` +- **WHEN** the summary agent is created +- **THEN** its instructions include `---`, `projects:`, `# Alpha`, Alpha's `AGENTS.md`, `# Beta`, and Beta's `AGENTS.md` + +#### Scenario: Projects without AGENTS files are skipped +- **GIVEN** the meeting note frontmatter lists project `Alpha` +- **AND** the configured project folder does not contain `AGENTS.md` +- **WHEN** the summary agent is created +- **THEN** no empty project instruction entry is appended for `Alpha` diff --git a/openspec/changes/configure-summary-agent-instructions/tasks.md b/openspec/changes/configure-summary-agent-instructions/tasks.md new file mode 100644 index 0000000..87caa83 --- /dev/null +++ b/openspec/changes/configure-summary-agent-instructions/tasks.md @@ -0,0 +1,13 @@ +## 1. Specification +- [x] 1.1 Define configurable initial prompt and project instruction append behavior. + +## 2. Implementation +- [x] 2.1 Add `Agent.InitialPrompt` setting with the current prompt in appsettings. +- [x] 2.2 Keep the current prompt as the code default when no setting is configured. +- [x] 2.3 Build summary agent instructions by appending `AGENTS.md` content for bound projects. +- [x] 2.4 Use the built instructions when creating the summary agent. + +## 3. Validation +- [x] 3.1 Test configurable prompt fallback and override behavior. +- [x] 3.2 Test project `AGENTS.md` instruction appendix formatting. +- [x] 3.3 Run targeted and full tests.