diff --git a/.gitignore b/.gitignore index 0808c4a..42f39ea 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ # dotenv files .env +.env.* +!.env.example # User-specific files *.rsuser @@ -35,6 +37,33 @@ bld/ [Ll]og/ [Ll]ogs/ +# Local Whisper models +MeetingAssistant/models/ +models/ +*.ggml +*.bin + +# Meeting Assistant local runtime data +MeetingAssistant/appsettings.Local.json +MeetingAssistant/appsettings.*.local.json +MeetingAssistant/**/appsettings.Local.json +MeetingAssistant/**/appsettings.*.local.json +.meeting-assistant/ +recordings/ +Recordings/ +transcripts/ +Transcripts/ +summaries/ +Summaries/ +assistant-context/ +Assistant Context/ +FunASR/ +Pyannote/ +*.onnx +*.pt +*.safetensors +*.ckpt + # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot diff --git a/AGENTS.md b/AGENTS.md index 6ffb15c..f6f0dc2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,16 @@ When in doubt, make the reasoning explicit before editing specs or code. A spec/change is not considered done merely because code is merged, tests pass, or `openspec validate` succeeds. A change is ready to be accepted or archived only after the agent has verified the intended behavior through the most direct operational surface available. +Before archiving any OpenSpec change, perform a refactoring pass over the code and specs touched by that change. The pass must inspect both the current diff and the surrounding implementation context, because a small diff may reveal repeated patterns or structural problems that only become obvious when compared with nearby code. + +Do the refactoring pass in these distinct phases, in order: + +1. Check for DRYness. Look for duplication introduced by the change and for existing nearby duplication that the change now makes worth consolidating. A change may be small on its own, but if it is the fifth copy of the same idea, it is a refactoring target. +2. Check for SOLID violations. Look for responsibilities that are mixed together, abstractions that are hard to replace or test, interface shapes that force unrelated dependencies, and code paths that require modifying stable code for each new variant. +3. Check whether the implementation can be made simpler under KISS. Remove accidental abstractions, reduce branching, clarify names, and prefer the smallest structure that still supports the tested behavior and current spec. + +Preserve behavior during this pass and run the relevant tests again afterward. + For Meeting Assistant, prefer verification through: - local application endpoints diff --git a/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs b/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs new file mode 100644 index 0000000..f70c945 --- /dev/null +++ b/MeetingAssistant.Tests/AsrDiagnosticEndpointTests.cs @@ -0,0 +1,174 @@ +using System.Net; +using System.Net.Http.Json; +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeetingAssistant.Tests; + +public sealed class AsrDiagnosticEndpointTests +{ + [Fact] + public async Task EndpointTranscribesLocalWavThroughConfiguredProvider() + { + await using var factory = CreateFactory(); + using var client = factory.CreateClient(); + var wavPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "sample-16khz-mono.wav"); + + using var response = await client.PostAsJsonAsync("/asr/transcribe-file", new { path = wavPath }); + var body = await response.Content.ReadFromJsonAsync(); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(wavPath, body?.Path); + var segment = Assert.Single(body?.Segments ?? []); + Assert.Equal("Unknown", segment.Speaker); + Assert.StartsWith("endpoint-bytes:", segment.Text, StringComparison.Ordinal); + } + + [Fact] + public async Task EndpointReportsProviderFailuresAsBadGateway() + { + await using var factory = CreateFactory(); + using var client = factory.CreateClient(); + var wavPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "sample-16khz-mono.wav"); + + using var response = await client.PostAsJsonAsync("/asr/transcribe-file", new { path = wavPath }); + + Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode); + } + + [Fact] + public async Task EndpointRunsFinalDiarizationForLocalWav() + { + SpeechRecognitionPipelineOptions? capturedOptions = null; + await using var factory = CreateFactory( + (audioPath, liveSegments, options, cancellationToken) => + { + capturedOptions = options; + return Task.FromResult>( + liveSegments + .Select(segment => segment with { Speaker = "Speaker 0", Text = $"diarized:{segment.Text}" }) + .ToList()); + }); + using var client = factory.CreateClient(); + var wavPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "sample-16khz-mono.wav"); + + using var response = await client.PostAsJsonAsync("/asr/diarize-file", new { path = wavPath, numSpeakers = 5 }); + var body = await response.Content.ReadFromJsonAsync(); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(5, capturedOptions?.NumSpeakers); + Assert.Equal(wavPath, body?.Path); + var segment = Assert.Single(body?.Segments ?? []); + Assert.Equal("Speaker 0", segment.Speaker); + Assert.StartsWith("diarized:endpoint-bytes:", segment.Text, StringComparison.Ordinal); + } + + private static WebApplicationFactory CreateFactory() + where TProvider : class, IStreamingTranscriptionProvider + { + return CreateFactory((_, _, _, _) => Task.FromResult>([])); + } + + private static WebApplicationFactory CreateFactory( + Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize) + where TProvider : class, IStreamingTranscriptionProvider + { + return new WebApplicationFactory() + .WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new Dictionary + { + ["MeetingAssistant:FunAsr:Backend:Enabled"] = "false" + }); + }); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.AddSingleton( + new TestSpeechRecognitionPipelineFactory(finalize)); + }); + }); + } + + private sealed class EndpointFakeTranscriptionProvider : IStreamingTranscriptionProvider + { + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var bytes = 0; + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + bytes += chunk.Pcm.Length; + } + + yield return new TranscriptionSegment(TimeSpan.Zero, TimeSpan.Zero, "Unknown", $"endpoint-bytes:{bytes}"); + } + } + + private sealed class FailingTranscriptionProvider : IStreamingTranscriptionProvider + { + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + throw new InvalidOperationException("backend unavailable"); +#pragma warning disable CS0162 + yield break; +#pragma warning restore CS0162 + } + } + + private sealed class TestSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory + where TProvider : class, IStreamingTranscriptionProvider + { + private readonly Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize; + + public TestSpeechRecognitionPipelineFactory( + Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize) + { + this.finalize = finalize; + } + + public ISpeechRecognitionPipeline Create() + { + return new TestSpeechRecognitionPipeline( + Activator.CreateInstance(), + finalize); + } + } + + private sealed class TestSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline + { + private readonly Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize; + + public TestSpeechRecognitionPipeline( + IStreamingTranscriptionProvider provider, + Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize) + : base(provider) + { + this.finalize = finalize; + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return finalize(audioPath, liveSegments, options, cancellationToken); + } + } + + private sealed record AsrDiagnosticResponse(string Path, IReadOnlyList Segments); + + private sealed record AsrDiagnosticSegment(string Speaker, string Text); +} diff --git a/MeetingAssistant.Tests/AsrDiagnosticTests.cs b/MeetingAssistant.Tests/AsrDiagnosticTests.cs new file mode 100644 index 0000000..26c69b9 --- /dev/null +++ b/MeetingAssistant.Tests/AsrDiagnosticTests.cs @@ -0,0 +1,79 @@ +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Tests; + +public sealed class AsrDiagnosticTests +{ + [Fact] + public async Task DiagnosticStreamsPcmWavThroughConfiguredProvider() + { + var provider = new CapturingTranscriptionProvider(); + var diagnostics = new AsrDiagnosticService( + new TestSpeechRecognitionPipelineFactory(provider)); + var wavPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "sample-16khz-mono.wav"); + + var result = await diagnostics.TranscribeWavAsync(wavPath, CancellationToken.None); + + Assert.Equal(wavPath, result.Path); + Assert.Equal(16000, provider.Chunks.Single().SampleRate); + Assert.Equal(1, provider.Chunks.Single().Channels); + Assert.NotEmpty(provider.Chunks.Single().Pcm); + var segment = Assert.Single(result.Segments); + Assert.Equal("Unknown", segment.Speaker); + Assert.Equal("bytes:" + provider.Chunks.Single().Pcm.Length, segment.Text); + } + + private sealed class CapturingTranscriptionProvider : IStreamingTranscriptionProvider + { + public List Chunks { get; } = []; + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + Chunks.Add(chunk); + } + + yield return new TranscriptionSegment( + TimeSpan.Zero, + TimeSpan.Zero, + "Unknown", + "bytes:" + Chunks.Sum(chunk => chunk.Pcm.Length)); + } + } + + private sealed class TestSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory + { + private readonly IStreamingTranscriptionProvider provider; + + public TestSpeechRecognitionPipelineFactory(IStreamingTranscriptionProvider provider) + { + this.provider = provider; + } + + public ISpeechRecognitionPipeline Create() + { + return new TestSpeechRecognitionPipeline(provider); + } + } + + private sealed class TestSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline + { + public TestSpeechRecognitionPipeline(IStreamingTranscriptionProvider provider) + : base(provider) + { + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult>([]); + } + } +} diff --git a/MeetingAssistant.Tests/AudioMixingTests.cs b/MeetingAssistant.Tests/AudioMixingTests.cs new file mode 100644 index 0000000..711f7b8 --- /dev/null +++ b/MeetingAssistant.Tests/AudioMixingTests.cs @@ -0,0 +1,71 @@ +using MeetingAssistant.Recording; +using Microsoft.Extensions.Logging.Abstractions; + +namespace MeetingAssistant.Tests; + +public sealed class AudioMixingTests +{ + [Fact] + public async Task CompositeAudioSourceMixesMicrophoneAndSystemAudioIntoOnePcmStream() + { + var microphone = new FixedAudioSource(Pcm16(10_000)); + var system = new FixedAudioSource(Pcm16(20_000)); + var source = new CompositeMeetingAudioSource( + microphone, + system, + NullLogger.Instance); + + var chunks = await ReadChunks(source); + + Assert.Single(chunks); + Assert.Equal(30_000, BitConverter.ToInt16(chunks[0].Pcm)); + } + + [Fact] + public async Task CompositeAudioSourceClampsMixedSamples() + { + var microphone = new FixedAudioSource(Pcm16(30_000)); + var system = new FixedAudioSource(Pcm16(10_000)); + var source = new CompositeMeetingAudioSource( + microphone, + system, + NullLogger.Instance); + + var chunks = await ReadChunks(source); + + Assert.Equal(short.MaxValue, BitConverter.ToInt16(chunks[0].Pcm)); + } + + private static async Task> ReadChunks(IMeetingAudioSource source) + { + var chunks = new List(); + await foreach (var chunk in source.CaptureAsync(CancellationToken.None)) + { + chunks.Add(chunk); + } + + return chunks; + } + + private static byte[] Pcm16(short sample) + { + return BitConverter.GetBytes(sample); + } + + private sealed class FixedAudioSource : IMeetingAudioSource + { + private readonly AudioChunk chunk; + + public FixedAudioSource(byte[] pcm) + { + chunk = new AudioChunk(pcm, 16000, 1); + } + + public async IAsyncEnumerable CaptureAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + yield return chunk; + } + } +} diff --git a/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs b/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs new file mode 100644 index 0000000..789460f --- /dev/null +++ b/MeetingAssistant.Tests/AzureSpeechRecognitionPipelineTests.cs @@ -0,0 +1,62 @@ +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; + +namespace MeetingAssistant.Tests; + +public sealed class AzureSpeechRecognitionPipelineTests +{ + [Fact] + public async Task FinishedTranscriptUsesLiveDiarizedAzureSegments() + { + await using var pipeline = new AzureSpeechRecognitionPipeline( + new StaticTranscriptionProvider( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest-1", "hello"), + new TranscriptionSegment(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), "Guest-2", "there") + ])); + + await pipeline.InitializeAsync(CancellationToken.None); + await pipeline.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None); + await pipeline.CompleteAsync(CancellationToken.None); + var liveSegments = new List(); + await foreach (var segment in pipeline.ReadLiveTranscriptAsync(CancellationToken.None)) + { + liveSegments.Add(segment); + } + + var finishedSegments = await pipeline.ReadFinishedTranscriptAsync( + "ignored.wav", + new SpeechRecognitionPipelineOptions(5), + CancellationToken.None); + + Assert.Equal(liveSegments, finishedSegments); + Assert.Collection( + finishedSegments, + first => Assert.Equal("Guest-1", first.Speaker), + second => Assert.Equal("Guest-2", second.Speaker)); + } + + private sealed class StaticTranscriptionProvider : IStreamingTranscriptionProvider + { + private readonly IReadOnlyList segments; + + public StaticTranscriptionProvider(IReadOnlyList segments) + { + this.segments = segments; + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var _ in audio.WithCancellation(cancellationToken)) + { + } + + foreach (var segment in segments) + { + yield return segment; + } + } + } +} diff --git a/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs new file mode 100644 index 0000000..261112a --- /dev/null +++ b/MeetingAssistant.Tests/AzureSpeechStreamingTranscriptionProviderTests.cs @@ -0,0 +1,39 @@ +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class AzureSpeechStreamingTranscriptionProviderTests +{ + [Fact] + public async Task TranscribeFailsClearlyWhenKeyIsMissing() + { + var provider = new AzureSpeechStreamingTranscriptionProvider( + Options.Create(new MeetingAssistantOptions + { + AzureSpeech = new AzureSpeechOptions + { + Region = "germanywestcentral", + KeyEnv = $"MEETING_ASSISTANT_MISSING_AZURE_KEY_{Guid.NewGuid():N}" + } + }), + NullLogger.Instance); + + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in provider.TranscribeAsync(ReadSingleChunk(), CancellationToken.None)) + { + } + }); + + Assert.Contains("Azure Speech key is not configured", exception.Message); + } + + private static async IAsyncEnumerable ReadSingleChunk() + { + await Task.Yield(); + yield return new AudioChunk(new byte[320], 16000, 1); + } +} diff --git a/MeetingAssistant.Tests/ConfiguredSpeechRecognitionPipelineFactoryTests.cs b/MeetingAssistant.Tests/ConfiguredSpeechRecognitionPipelineFactoryTests.cs new file mode 100644 index 0000000..aa96b57 --- /dev/null +++ b/MeetingAssistant.Tests/ConfiguredSpeechRecognitionPipelineFactoryTests.cs @@ -0,0 +1,34 @@ +using MeetingAssistant.Transcription; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class ConfiguredSpeechRecognitionPipelineFactoryTests +{ + [Fact] + public async Task FactoryCreatesAzureSpeechPipelineWhenConfigured() + { + var services = new ServiceCollection(); + services.AddSingleton(Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions + { + TranscriptionProvider = "azure-speech" + } + })); + services.AddSingleton>( + NullLogger.Instance); + services.AddTransient(); + var provider = services.BuildServiceProvider(); + var factory = new ConfiguredSpeechRecognitionPipelineFactory( + provider, + provider.GetRequiredService>()); + + await using var pipeline = factory.Create(); + + Assert.IsType(pipeline); + } +} diff --git a/MeetingAssistant.Tests/Fixtures/sample-16khz-mono.wav b/MeetingAssistant.Tests/Fixtures/sample-16khz-mono.wav new file mode 100644 index 0000000..78b2077 Binary files /dev/null and b/MeetingAssistant.Tests/Fixtures/sample-16khz-mono.wav differ diff --git a/MeetingAssistant.Tests/FunAsrBackendLifecycleTests.cs b/MeetingAssistant.Tests/FunAsrBackendLifecycleTests.cs new file mode 100644 index 0000000..eea316f --- /dev/null +++ b/MeetingAssistant.Tests/FunAsrBackendLifecycleTests.cs @@ -0,0 +1,203 @@ +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class FunAsrBackendLifecycleTests +{ + [Fact] + public async Task LifecycleInstallsStartsAndStopsManagedDockerContainer() + { + var runner = new CapturingCommandRunner(); + var readinessProbe = new CapturingReadinessProbe(); + var modelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var lifecycle = new FunAsrDockerBackendLifecycle( + runner, + readinessProbe, + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Endpoint = "ws://127.0.0.1:10095", + Backend = new FunAsrBackendOptions + { + Enabled = true, + Image = "funasr:test", + ContainerName = "meeting-assistant-funasr-test", + ModelsFolder = modelsFolder, + StartupTimeout = TimeSpan.Zero + } + } + }), + NullLogger.Instance); + + await lifecycle.EnsureStartedAsync(CancellationToken.None); + await lifecycle.EnsureStartedAsync(CancellationToken.None); + await lifecycle.DisposeAsync(); + + Assert.Equal(new Uri("ws://127.0.0.1:10095"), readinessProbe.Endpoint); + Assert.Equal(TimeSpan.Zero, readinessProbe.Timeout); + Assert.Collection( + runner.Commands, + command => + { + Assert.Equal("docker", command.FileName); + Assert.Equal(["pull", "funasr:test"], command.Arguments); + }, + command => + { + Assert.Equal(["rm", "-f", "meeting-assistant-funasr-test"], command.Arguments); + }, + command => + { + Assert.Equal("docker", command.FileName); + Assert.Contains("run", command.Arguments); + Assert.Contains("-d", command.Arguments); + Assert.Contains("--privileged=true", command.Arguments); + Assert.Contains("--name", command.Arguments); + Assert.Contains("meeting-assistant-funasr-test", command.Arguments); + Assert.Contains("-p", command.Arguments); + Assert.Contains("10095:10095", command.Arguments); + Assert.Contains($"{modelsFolder}:/workspace/models", command.Arguments); + Assert.Contains("funasr:test", command.Arguments); + }, + command => + { + Assert.Equal(["stop", "meeting-assistant-funasr-test"], command.Arguments); + }); + Assert.True(File.Exists(Path.Combine(modelsFolder, "hotwords.txt"))); + } + + [Fact] + public async Task DisabledLifecycleDoesNotRunDockerCommands() + { + var runner = new CapturingCommandRunner(); + var lifecycle = new FunAsrDockerBackendLifecycle( + runner, + new CapturingReadinessProbe(), + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Backend = new FunAsrBackendOptions { Enabled = false } + } + }), + NullLogger.Instance); + + await lifecycle.EnsureStartedAsync(CancellationToken.None); + await lifecycle.DisposeAsync(); + + Assert.Empty(runner.Commands); + } + + [Fact] + public async Task LifecycleFailsWhenDockerCommandTimesOut() + { + var runner = new HangingCommandRunner(); + var lifecycle = new FunAsrDockerBackendLifecycle( + runner, + new CapturingReadinessProbe(), + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Backend = new FunAsrBackendOptions + { + Enabled = true, + CommandTimeout = TimeSpan.FromMilliseconds(1) + } + } + }), + NullLogger.Instance); + + var exception = await Assert.ThrowsAsync( + () => lifecycle.EnsureStartedAsync(CancellationToken.None)); + + Assert.Contains("timed out", exception.Message); + } + + [Fact] + public async Task LifecycleWaitsForWebSocketReadinessBeforeMarkingStarted() + { + var runner = new CapturingCommandRunner(); + var readinessProbe = new CapturingReadinessProbe(); + var lifecycle = new FunAsrDockerBackendLifecycle( + runner, + readinessProbe, + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Endpoint = "ws://127.0.0.1:10095", + Backend = new FunAsrBackendOptions + { + Enabled = true, + Image = "funasr:test", + ContainerName = "meeting-assistant-funasr-test", + ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")), + StartupTimeout = TimeSpan.FromMinutes(3) + } + } + }), + NullLogger.Instance); + + await lifecycle.EnsureStartedAsync(CancellationToken.None); + + Assert.Equal(1, readinessProbe.CallCount); + Assert.Equal(new Uri("ws://127.0.0.1:10095"), readinessProbe.Endpoint); + Assert.Equal(TimeSpan.FromMinutes(3), readinessProbe.Timeout); + Assert.Equal(3, runner.Commands.Count); + await lifecycle.DisposeAsync(); + } + + private sealed class CapturingCommandRunner : ICommandRunner + { + public List Commands { get; } = []; + + public Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + Commands.Add(new CapturedCommand(fileName, arguments.ToArray())); + return Task.FromResult(new CommandResult(0, "", "")); + } + } + + private sealed record CapturedCommand(string FileName, IReadOnlyList Arguments); + + private sealed class HangingCommandRunner : ICommandRunner + { + public async Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new CommandResult(0, "", ""); + } + } + + private sealed class CapturingReadinessProbe : IFunAsrBackendReadinessProbe + { + public int CallCount { get; private set; } + + public Uri? Endpoint { get; private set; } + + public TimeSpan Timeout { get; private set; } + + public Task WaitUntilReadyAsync( + Uri endpoint, + TimeSpan timeout, + CancellationToken cancellationToken) + { + CallCount++; + Endpoint = endpoint; + Timeout = timeout; + return Task.CompletedTask; + } + } +} diff --git a/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs b/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs new file mode 100644 index 0000000..5569f47 --- /dev/null +++ b/MeetingAssistant.Tests/FunAsrStreamingTranscriptionProviderTests.cs @@ -0,0 +1,184 @@ +using System.Text.Json; +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class FunAsrStreamingTranscriptionProviderTests +{ + [Fact] + public async Task ProviderStreamsPcmChunksAndEmitsSpeakerAttributedSegments() + { + var connection = new FakeFunAsrWebSocketConnection( + """ + {"mode":"2pass-offline","wav_name":"meeting","text":"hello from mike","is_final":true,"spk_name":"Mike","spk_score":0.92,"timestamp":[[1000,1800]]} + """); + var provider = new FunAsrStreamingTranscriptionProvider( + new FakeFunAsrWebSocketConnectionFactory(connection), + new NoopFunAsrBackendLifecycle(), + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Endpoint = "ws://localhost:10095", + Mode = "2pass", + ChunkSize = [5, 10, 5], + ChunkInterval = 10, + FinalResultTimeout = TimeSpan.FromSeconds(1) + } + }), + NullLogger.Instance); + var chunks = new[] + { + new AudioChunk([1, 0, 2, 0], 16000, 1), + new AudioChunk([3, 0, 4, 0], 16000, 1) + }; + + var segments = await CollectAsync(provider.TranscribeAsync(ToAsyncEnumerable(chunks), CancellationToken.None)); + + Assert.Equal(new Uri("ws://localhost:10095"), connection.ConnectedEndpoint); + Assert.Equal(2, connection.BinaryMessages.Count); + Assert.Equal(chunks[0].Pcm, connection.BinaryMessages[0]); + Assert.Equal(chunks[1].Pcm, connection.BinaryMessages[1]); + Assert.Equal("Mike", Assert.Single(segments).Speaker); + Assert.Equal("hello from mike", segments[0].Text); + Assert.Equal(TimeSpan.FromSeconds(1), segments[0].Start); + Assert.Equal(TimeSpan.FromMilliseconds(1800), segments[0].End); + + using var firstMessage = JsonDocument.Parse(connection.TextMessages[0]); + Assert.Equal("2pass", firstMessage.RootElement.GetProperty("mode").GetString()); + Assert.Equal("pcm", firstMessage.RootElement.GetProperty("wav_format").GetString()); + Assert.Equal(16000, firstMessage.RootElement.GetProperty("audio_fs").GetInt32()); + Assert.True(firstMessage.RootElement.GetProperty("is_speaking").GetBoolean()); + + using var finalMessage = JsonDocument.Parse(connection.TextMessages[^1]); + Assert.False(finalMessage.RootElement.GetProperty("is_speaking").GetBoolean()); + } + + [Fact] + public async Task ProviderUsesSentenceSpeakerFieldsAndUnknownFallback() + { + var connection = new FakeFunAsrWebSocketConnection( + """ + {"mode":"2pass-offline","wav_name":"meeting","text":"ignored","is_final":true,"sentence_info":[{"text":"first sentence","spk":0,"start":100,"end":600},{"text":"second sentence","start":700,"end":1200}]} + """); + var provider = new FunAsrStreamingTranscriptionProvider( + new FakeFunAsrWebSocketConnectionFactory(connection), + new NoopFunAsrBackendLifecycle(), + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions { FinalResultTimeout = TimeSpan.FromSeconds(1) } + }), + NullLogger.Instance); + + var segments = await CollectAsync(provider.TranscribeAsync( + ToAsyncEnumerable([new AudioChunk([1, 0], 16000, 1)]), + CancellationToken.None)); + + Assert.Collection( + segments, + first => + { + Assert.Equal("Speaker 0", first.Speaker); + Assert.Equal("first sentence", first.Text); + Assert.Equal(TimeSpan.FromMilliseconds(100), first.Start); + Assert.Equal(TimeSpan.FromMilliseconds(600), first.End); + }, + second => + { + Assert.Equal("Unknown", second.Speaker); + Assert.Equal("second sentence", second.Text); + Assert.Equal(TimeSpan.FromMilliseconds(700), second.Start); + Assert.Equal(TimeSpan.FromMilliseconds(1200), second.End); + }); + } + + private static async IAsyncEnumerable ToAsyncEnumerable(IEnumerable chunks) + { + foreach (var chunk in chunks) + { + yield return chunk; + await Task.Yield(); + } + } + + private static async Task> CollectAsync(IAsyncEnumerable segments) + { + var collected = new List(); + await foreach (var segment in segments) + { + collected.Add(segment); + } + + return collected; + } + + private sealed class FakeFunAsrWebSocketConnectionFactory : IFunAsrWebSocketConnectionFactory + { + private readonly FakeFunAsrWebSocketConnection connection; + + public FakeFunAsrWebSocketConnectionFactory(FakeFunAsrWebSocketConnection connection) + { + this.connection = connection; + } + + public Task ConnectAsync(Uri endpoint, CancellationToken cancellationToken) + { + connection.ConnectedEndpoint = endpoint; + return Task.FromResult(connection); + } + } + + private sealed class NoopFunAsrBackendLifecycle : IFunAsrBackendLifecycle + { + public Task EnsureStartedAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + + private sealed class FakeFunAsrWebSocketConnection : IFunAsrWebSocketConnection + { + private readonly Queue responses; + + public FakeFunAsrWebSocketConnection(params string[] responses) + { + this.responses = new Queue(responses); + } + + public Uri? ConnectedEndpoint { get; set; } + + public List TextMessages { get; } = []; + + public List BinaryMessages { get; } = []; + + public Task SendTextAsync(string message, CancellationToken cancellationToken) + { + TextMessages.Add(message); + return Task.CompletedTask; + } + + public Task SendBinaryAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + BinaryMessages.Add(message.ToArray()); + return Task.CompletedTask; + } + + public Task ReceiveTextAsync(CancellationToken cancellationToken) + { + return Task.FromResult(responses.Count == 0 ? null : responses.Dequeue()); + } + + public Task CloseOutputAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } +} diff --git a/MeetingAssistant.Tests/FunAsrTranscriptFinalizerTests.cs b/MeetingAssistant.Tests/FunAsrTranscriptFinalizerTests.cs new file mode 100644 index 0000000..7c15815 --- /dev/null +++ b/MeetingAssistant.Tests/FunAsrTranscriptFinalizerTests.cs @@ -0,0 +1,109 @@ +using MeetingAssistant; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class FunAsrTranscriptFinalizerTests +{ + [Fact] + public async Task FinalizerRunsFunAsrAutoModelAndReturnsSentenceSpeakerSegments() + { + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + var commandRunner = new CapturingCommandRunner( + """ + setup noise + __MEETING_ASSISTANT_DIARIZATION_JSON_START__ + [{"key":"input","sentence_info":[{"text":" hello","start":410,"end":4810,"spk":0},{"text":" yes","start":22550,"end":22770,"spk":1}]}] + __MEETING_ASSISTANT_DIARIZATION_JSON_END__ + """); + var finalizer = new FunAsrTranscriptFinalizer( + commandRunner, + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Backend = new FunAsrBackendOptions + { + DockerCommand = "docker", + Image = "funasr:test", + ModelsFolder = "C:\\Models" + }, + Diarization = new FunAsrDiarizationOptions + { + Enabled = true, + CommandTimeout = TimeSpan.FromMinutes(1) + } + } + }), + NullLogger.Instance); + + var segments = await finalizer.FinalizeAsync(audioPath, [], CancellationToken.None); + + Assert.Contains("run", commandRunner.Arguments); + Assert.Contains("funasr:test", commandRunner.Arguments); + Assert.Contains($"{audioPath}:/workspace/input.wav:ro", commandRunner.Arguments); + Assert.Collection( + segments, + first => + { + Assert.Equal(TimeSpan.FromMilliseconds(410), first.Start); + Assert.Equal(TimeSpan.FromMilliseconds(4810), first.End); + Assert.Equal("Speaker 0", first.Speaker); + Assert.Equal("hello", first.Text); + }, + second => + { + Assert.Equal(TimeSpan.FromMilliseconds(22550), second.Start); + Assert.Equal(TimeSpan.FromMilliseconds(22770), second.End); + Assert.Equal("Speaker 1", second.Speaker); + Assert.Equal("yes", second.Text); + }); + } + + [Fact] + public async Task FinalizerReturnsNoSegmentsWhenDiarizationIsDisabled() + { + var commandRunner = new CapturingCommandRunner(""); + var finalizer = new FunAsrTranscriptFinalizer( + commandRunner, + Options.Create(new MeetingAssistantOptions + { + FunAsr = new FunAsrOptions + { + Diarization = new FunAsrDiarizationOptions { Enabled = false } + } + }), + NullLogger.Instance); + + var segments = await finalizer.FinalizeAsync("C:\\Recordings\\meeting.wav", [], CancellationToken.None); + + Assert.Empty(segments); + Assert.Empty(commandRunner.Arguments); + } + + private sealed class CapturingCommandRunner : ICommandRunner + { + private readonly string output; + + public CapturingCommandRunner(string output) + { + this.output = output; + } + + public IReadOnlyList Arguments { get; private set; } = []; + + public Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + Arguments = arguments; + return Task.FromResult(new CommandResult(0, output, "")); + } + } +} diff --git a/MeetingAssistant.Tests/HealthEndpointTests.cs b/MeetingAssistant.Tests/HealthEndpointTests.cs index d5e367d..615ec4e 100644 --- a/MeetingAssistant.Tests/HealthEndpointTests.cs +++ b/MeetingAssistant.Tests/HealthEndpointTests.cs @@ -1,6 +1,12 @@ using System.Net; using System.Net.Http.Json; +using MeetingAssistant.Recording; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace MeetingAssistant.Tests; @@ -10,7 +16,16 @@ public sealed class HealthEndpointTests : IClassFixture factory) { - this.factory = factory; + this.factory = factory.WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new Dictionary + { + ["MeetingAssistant:FunAsr:Backend:Enabled"] = "false" + }); + }); + }); } [Fact] @@ -26,5 +41,41 @@ public sealed class HealthEndpointTests : IClassFixture + { + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.AddSingleton(); + }); + }); + using var client = cleanupFactory.CreateClient(); + + using var response = await client.GetAsync("/health"); + var store = cleanupFactory.Services.GetRequiredService() as StartupCleanupRecordedAudioStore; + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.True(store?.StaleRecordingsDeleted); + } + private sealed record HealthResponse(string Service, string Status); + + private sealed class StartupCleanupRecordedAudioStore : IRecordedAudioStore + { + public bool StaleRecordingsDeleted { get; private set; } + + public Task CreateSessionAsync(CancellationToken cancellationToken) + { + throw new NotSupportedException(); + } + + public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken) + { + StaleRecordingsDeleted = true; + return Task.CompletedTask; + } + } } diff --git a/MeetingAssistant.Tests/HotkeyDefinitionTests.cs b/MeetingAssistant.Tests/HotkeyDefinitionTests.cs new file mode 100644 index 0000000..882a6b7 --- /dev/null +++ b/MeetingAssistant.Tests/HotkeyDefinitionTests.cs @@ -0,0 +1,15 @@ +using MeetingAssistant.Hotkeys; + +namespace MeetingAssistant.Tests; + +public sealed class HotkeyDefinitionTests +{ + [Fact] + public void HotkeyDefinitionParsesConfiguredToggleShortcut() + { + var definition = HotkeyDefinition.Parse("Ctrl+Alt+M"); + + Assert.Equal(HotkeyModifiers.Control | HotkeyModifiers.Alt, definition.Modifiers); + Assert.Equal((uint)'M', definition.VirtualKey); + } +} diff --git a/MeetingAssistant.Tests/LiteLlmResponsesChatClientTests.cs b/MeetingAssistant.Tests/LiteLlmResponsesChatClientTests.cs new file mode 100644 index 0000000..1156df6 --- /dev/null +++ b/MeetingAssistant.Tests/LiteLlmResponsesChatClientTests.cs @@ -0,0 +1,335 @@ +using MeetingAssistant.Summary; +using Microsoft.Extensions.AI; +using System.Net; + +namespace MeetingAssistant.Tests; + +public sealed class LiteLlmResponsesChatClientTests +{ + [Fact] + public void ParserIgnoresReasoningItemsWithNullStatusAndReadsText() + { + const string json = """ + { + "id": "resp_test", + "created_at": 1779147100, + "model": "gpt-5.5-2026-04-23", + "output": [ + { + "type": "reasoning", + "summary": [], + "status": null + }, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "OK" + } + ] + } + ] + } + """; + + var response = LiteLlmResponsesChatClient.ParseResponseJson(json); + + Assert.Equal("OK", response.Text); + Assert.Equal("resp_test", response.ResponseId); + Assert.Equal("gpt-5.5-2026-04-23", response.ModelId); + } + + [Fact] + public void ParserReadsFunctionCalls() + { + const string json = """ + { + "output": [ + { + "type": "function_call", + "call_id": "call_1", + "name": "write_summary", + "arguments": "{\"markdown\":\"# Summary\\nDone\"}", + "status": "completed" + } + ] + } + """; + + var response = LiteLlmResponsesChatClient.ParseResponseJson(json); + var call = Assert.IsType(Assert.Single(response.Messages[0].Contents)); + + Assert.Equal("call_1", call.CallId); + Assert.Equal("write_summary", call.Name); + Assert.NotNull(call.Arguments); + Assert.Equal("# Summary\nDone", call.Arguments["markdown"]); + } + + [Fact] + public void ParserReadsUsage() + { + const string json = """ + { + "usage": { + "input_tokens": 123, + "output_tokens": 45, + "total_tokens": 168 + }, + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "OK" + } + ] + } + ] + } + """; + + var response = LiteLlmResponsesChatClient.ParseResponseJson(json); + + Assert.NotNull(response.Usage); + Assert.Equal(123, response.Usage.InputTokenCount); + Assert.Equal(45, response.Usage.OutputTokenCount); + Assert.Equal(168, response.Usage.TotalTokenCount); + } + + [Fact] + public async Task ClientRetriesTransientServerFailure() + { + var handler = new SequencedHttpMessageHandler( + new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("Internal Server Error") + }, + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "Done." + } + ] + } + ] + } + """) + }); + using var client = CreateClient(handler, reconnectionAttempts: 1); + + var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]); + + Assert.Equal("Done.", response.Text); + Assert.Equal(2, handler.RequestCount); + } + + [Fact] + public async Task ClientDoesNotRetryNonTransientClientFailure() + { + var handler = new SequencedHttpMessageHandler( + new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent("bad request") + }); + using var client = CreateClient(handler, reconnectionAttempts: 3); + + var exception = await Assert.ThrowsAsync( + () => client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")])); + + Assert.Contains("400", exception.Message); + Assert.Equal(1, handler.RequestCount); + } + + [Fact] + public async Task ClientUsesResponsesCompactionEndpointWhenContextThresholdIsReached() + { + var handler = new RecordingHttpMessageHandler(request => + { + if (request.RequestUri?.PathAndQuery == "/v1/responses/compact") + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "input": [ + { + "role": "user", + "content": "compacted" + } + ] + } + """) + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "Done." + } + ] + } + ] + } + """) + }; + }); + using var client = CreateClient( + handler, + reconnectionAttempts: 0, + new LiteLlmResponsesCompactionOptions + { + Enabled = true, + ContextWindowTokens = 100, + MaxOutputTokens = 10, + RemainingRatio = 0.10, + CompactPath = "responses/compact" + }); + + var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, new string('x', 800))]); + + Assert.Equal("Done.", response.Text); + Assert.Equal(["/v1/responses/compact", "/v1/responses"], handler.RequestPaths); + Assert.Contains("\"content\":\"compacted\"", handler.RequestBodies[1]); + } + + [Fact] + public async Task ClientFallsBackToAgentFrameworkCompactionWhenResponsesCompactionFails() + { + var handler = new RecordingHttpMessageHandler(request => + { + if (request.RequestUri?.PathAndQuery == "/v1/responses/compact") + { + return new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent("missing") + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "Done." + } + ] + } + ] + } + """) + }; + }); + using var client = CreateClient( + handler, + reconnectionAttempts: 0, + new LiteLlmResponsesCompactionOptions + { + Enabled = true, + ContextWindowTokens = 100, + MaxOutputTokens = 10, + RemainingRatio = 0.10, + CompactPath = "responses/compact", + FallbackStrategy = OpenAiMeetingSummaryAgentPipeline.CreateFallbackCompactionStrategyForTests( + contextWindowTokens: 100, + maxOutputTokens: 10, + remainingRatio: 0.10, + summaryClient: null) + }); + + var response = await client.GetResponseAsync( + Enumerable.Range(0, 8) + .Select(index => new ChatMessage(ChatRole.User, $"turn {index} {new string('x', 200)}"))); + + Assert.Equal("Done.", response.Text); + Assert.Equal(["/v1/responses/compact", "/v1/responses"], handler.RequestPaths); + Assert.DoesNotContain("turn 0", handler.RequestBodies[1]); + } + + private static LiteLlmResponsesChatClient CreateClient( + HttpMessageHandler handler, + int reconnectionAttempts, + LiteLlmResponsesCompactionOptions? compactionOptions = null) + { + return new LiteLlmResponsesChatClient( + new HttpClient(handler) + { + BaseAddress = new Uri("https://litellm.example/v1/") + }, + "test-key", + "test-model", + enableThinking: false, + reasoningEffort: "none", + reconnectionAttempts, + TimeSpan.Zero, + compactionOptions); + } + + private sealed class SequencedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue responses; + + public SequencedHttpMessageHandler(params HttpResponseMessage[] responses) + { + this.responses = new Queue(responses); + } + + public int RequestCount { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + RequestCount++; + return Task.FromResult(responses.Dequeue()); + } + } + + private sealed class RecordingHttpMessageHandler : HttpMessageHandler + { + private readonly Func handler; + + public RecordingHttpMessageHandler(Func handler) + { + this.handler = handler; + } + + public List RequestPaths { get; } = []; + + public List RequestBodies { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + RequestPaths.Add(request.RequestUri?.PathAndQuery ?? string.Empty); + RequestBodies.Add(request.Content is null + ? string.Empty + : await request.Content.ReadAsStringAsync(cancellationToken)); + return handler(request); + } + } +} diff --git a/MeetingAssistant.Tests/MeetingArtifactStoreTests.cs b/MeetingAssistant.Tests/MeetingArtifactStoreTests.cs new file mode 100644 index 0000000..c532a18 --- /dev/null +++ b/MeetingAssistant.Tests/MeetingArtifactStoreTests.cs @@ -0,0 +1,91 @@ +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Logging.Abstractions; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingArtifactStoreTests +{ + [Fact] + public async Task StoreCreatesAssistantContextNoteLinkedToMeetingArtifacts() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var contextPath = Path.Combine(root, "Meetings", "Assistant Context", "20260519-context.md"); + var artifactStore = new MarkdownMeetingArtifactStore(NullLogger.Instance); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "20260519-meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "20260519-transcript.md"), + AssistantContextPath: contextPath, + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "20260519-summary.md")); + + await artifactStore.CreateAssistantContextAsync( + artifacts, + "Review previous decisions\nAgree next steps", + DateTimeOffset.Parse("2026-05-19T11:00:00+02:00"), + CancellationToken.None); + + var content = await File.ReadAllTextAsync(contextPath); + Assert.Contains("# Assistant Context", content); + Assert.StartsWith("---", content, StringComparison.Ordinal); + Assert.Contains("state: transcribing", content); + Assert.Contains("agenda: |-", content); + Assert.Contains(" Review previous decisions", content); + Assert.Contains(" Agree next steps", content); + Assert.Contains("scheduled_end: \"2026-05-19T11:00:00.0000000+02:00\"", content); + Assert.Contains("meeting: \"[[../Notes/20260519-meeting|Meeting Note]]\"", content); + Assert.Contains("transcript: \"[[../Transcripts/20260519-transcript|Transcript]]\"", content); + Assert.Contains("summary: \"[[../Summaries/20260519-summary|Summary]]\"", content); + Assert.Contains("[[../Notes/20260519-meeting|Meeting Note]]", content); + Assert.Contains("[[../Transcripts/20260519-transcript|Transcript]]", content); + Assert.Contains("[[../Summaries/20260519-summary|Summary]]", content); + } + + [Fact] + public async Task StoreUpdatesAssistantContextStateAndPreservesBody() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var contextPath = Path.Combine(root, "Meetings", "Assistant Context", "20260519-context.md"); + var artifactStore = new MarkdownMeetingArtifactStore(NullLogger.Instance); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "20260519-meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "20260519-transcript.md"), + AssistantContextPath: contextPath, + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "20260519-summary.md")); + + await artifactStore.CreateAssistantContextAsync( + artifacts, + "Initial agenda", + DateTimeOffset.Parse("2026-05-19T11:00:00+02:00"), + CancellationToken.None); + await File.AppendAllTextAsync(contextPath, "Observed context line."); + + await artifactStore.UpdateAssistantContextStateAsync( + artifacts, + AssistantContextState.Summarizing, + CancellationToken.None); + + var content = await File.ReadAllTextAsync(contextPath); + Assert.Contains("state: summarizing", content); + Assert.Contains("agenda: |-", content); + Assert.Contains(" Initial agenda", content); + Assert.Contains("scheduled_end: \"2026-05-19T11:00:00.0000000+02:00\"", content); + Assert.Contains("Observed context line.", content); + } + + [Fact] + public async Task StoreOmitsScheduledEndWhenNoTeamsMeetingWasDetected() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var contextPath = Path.Combine(root, "Meetings", "Assistant Context", "20260519-context.md"); + var artifactStore = new MarkdownMeetingArtifactStore(NullLogger.Instance); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "20260519-meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "20260519-transcript.md"), + AssistantContextPath: contextPath, + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "20260519-summary.md")); + + await artifactStore.CreateAssistantContextAsync(artifacts, "", null, CancellationToken.None); + + var content = await File.ReadAllTextAsync(contextPath); + Assert.DoesNotContain("scheduled_end:", content); + } +} diff --git a/MeetingAssistant.Tests/MeetingAssistant.Tests.csproj b/MeetingAssistant.Tests/MeetingAssistant.Tests.csproj index f4f279d..db567c1 100644 --- a/MeetingAssistant.Tests/MeetingAssistant.Tests.csproj +++ b/MeetingAssistant.Tests/MeetingAssistant.Tests.csproj @@ -23,4 +23,8 @@ - \ No newline at end of file + + + + + diff --git a/MeetingAssistant.Tests/MeetingNoteStoreTests.cs b/MeetingAssistant.Tests/MeetingNoteStoreTests.cs new file mode 100644 index 0000000..a5051f0 --- /dev/null +++ b/MeetingAssistant.Tests/MeetingNoteStoreTests.cs @@ -0,0 +1,127 @@ +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingNoteStoreTests +{ + [Fact] + public async Task StoreWritesTemplateFrontmatterAndReadsUserNotesBody() + { + var (vaultRoot, store) = CreateStore(); + var note = MeetingNoteTemplate.Create( + title: "Leadership Sync", + startTime: DateTimeOffset.Parse("2026-05-19T10:00:00+02:00"), + endTime: DateTimeOffset.Parse("2026-05-19T10:30:00+02:00"), + attendees: + [ + "Mike ", + "Ada" + ], + projects: + [ + "Meeting Assistant", + "Alaric" + ], + transcriptPath: Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"), + assistantContextPath: Path.Combine(vaultRoot, "Meetings", "Assistant Context", "20260519-context.md"), + summaryPath: Path.Combine(vaultRoot, "Meetings", "Summaries", "20260519-summary.md"), + userNotes: "Discuss rollout risks."); + + var saved = await store.SaveAsync(note, CancellationToken.None); + var content = await File.ReadAllTextAsync(saved.Path); + var loaded = await store.ReadAsync(saved.Path, CancellationToken.None); + + Assert.EndsWith(".md", saved.Path, StringComparison.Ordinal); + Assert.Contains("attendees:", content); + Assert.Contains("- Mike ", content); + Assert.Contains("- Ada", content); + Assert.Contains("projects:", content); + Assert.Contains("- Meeting Assistant", content); + Assert.DoesNotContain("purpose:", content); + Assert.Contains("start_time: \"2026-05-19T10:00:00.0000000+02:00\"", content); + Assert.Contains("end_time: \"2026-05-19T10:30:00.0000000+02:00\"", content); + Assert.Contains("transcript: \"[[../Transcripts/20260519-transcript|Transcript]]\"", content); + Assert.Contains("assistant_context: \"[[../Assistant Context/20260519-context|Assistant Context]]\"", content); + Assert.Contains("summary: \"[[../Summaries/20260519-summary|Summary]]\"", content); + Assert.DoesNotContain("decisions:", content); + Assert.DoesNotContain("next_steps:", content); + Assert.Equal("Discuss rollout risks.", loaded.UserNotes); + Assert.Equal(["Mike ", "Ada"], loaded.Frontmatter.Attendees); + Assert.Equal(["Meeting Assistant", "Alaric"], loaded.Frontmatter.Projects); + Assert.Equal(DateTimeOffset.Parse("2026-05-19T10:00:00+02:00"), loaded.Frontmatter.StartTime); + Assert.Equal(DateTimeOffset.Parse("2026-05-19T10:30:00+02:00"), loaded.Frontmatter.EndTime); + Assert.Equal("[[../Transcripts/20260519-transcript|Transcript]]", loaded.Frontmatter.Transcript); + Assert.Equal("[[../Assistant Context/20260519-context|Assistant Context]]", loaded.Frontmatter.AssistantContext); + Assert.Equal("[[../Summaries/20260519-summary|Summary]]", loaded.Frontmatter.Summary); + } + + [Fact] + public async Task FrontmatterUpdatePreservesExistingUserNotesBody() + { + var (vaultRoot, store) = CreateStore(); + var userNotes = """ + ## User notes + + - First point + - Second point + + ```text + Keep this block exactly. + ``` + + """; + var saved = await store.SaveAsync( + MeetingNoteTemplate.Create( + title: "Body Preservation", + attendees: [], + projects: [], + transcriptPath: Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"), + assistantContextPath: Path.Combine(vaultRoot, "Meetings", "Assistant Context", "20260519-context.md"), + summaryPath: Path.Combine(vaultRoot, "Meetings", "Summaries", "20260519-summary.md"), + userNotes: userNotes), + CancellationToken.None); + + var loaded = await store.ReadAsync(saved.Path, CancellationToken.None); + loaded.Frontmatter.Title = "Updated title"; + await store.SaveAsync(loaded, CancellationToken.None); + var reloaded = await store.ReadAsync(saved.Path, CancellationToken.None); + + Assert.Equal("Updated title", reloaded.Frontmatter.Title); + Assert.Equal(userNotes, reloaded.UserNotes); + } + + [Fact] + public void ActionLinkEscapesSummaryFileName() + { + var link = MeetingNoteActionLinks.CreateSummaryRetryLink( + "http://localhost:5090/", + "C:\\Vault\\Meetings\\Summaries\\summary with spaces.md"); + + Assert.Equal( + "[Retry summary generation](http://localhost:5090/meetings/summary/retry?summaryPath=summary%20with%20spaces.md)", + link); + } + + private static (string VaultRoot, MarkdownMeetingNoteStore Store) CreateStore() + { + var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var store = new MarkdownMeetingNoteStore( + Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions + { + MeetingNotesFolder = Path.Combine(vaultRoot, "Meetings", "Notes"), + TranscriptsFolder = Path.Combine(vaultRoot, "Meetings", "Transcripts"), + AssistantContextFolder = Path.Combine(vaultRoot, "Meetings", "Assistant Context"), + SummariesFolder = Path.Combine(vaultRoot, "Meetings", "Summaries"), + ProjectsFolder = Path.Combine(vaultRoot, "Projects"), + DictationWordsPath = Path.Combine(vaultRoot, "Meetings", "dictation-words.md") + } + }), + NullLogger.Instance); + + return (vaultRoot, store); + } +} diff --git a/MeetingAssistant.Tests/MeetingSummaryArtifactResolverTests.cs b/MeetingAssistant.Tests/MeetingSummaryArtifactResolverTests.cs new file mode 100644 index 0000000..b83a545 --- /dev/null +++ b/MeetingAssistant.Tests/MeetingSummaryArtifactResolverTests.cs @@ -0,0 +1,76 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingSummaryArtifactResolverTests +{ + [Fact] + public async Task ResolverFindsMeetingArtifactsForSummaryPath() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var notes = Path.Combine(root, "Meetings", "Notes"); + var summaries = Path.Combine(root, "Meetings", "Summaries"); + Directory.CreateDirectory(notes); + Directory.CreateDirectory(summaries); + var notePath = Path.Combine(notes, "meeting.md"); + await File.WriteAllTextAsync( + notePath, + """ + --- + title: Meeting + attendees: + projects: + transcript: "[[../Transcripts/transcript|Transcript]]" + assistant_context: "[[../Assistant Context/context|Assistant Context]]" + summary: "[[../Summaries/summary|Summary]]" + --- + + User notes. + """); + var options = Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions + { + MeetingNotesFolder = notes, + SummariesFolder = summaries + } + }); + var resolver = new MeetingSummaryArtifactResolver( + options, + new MarkdownMeetingNoteStore(options, NullLogger.Instance)); + + var artifacts = await resolver.ResolveBySummaryPathAsync("summary.md", CancellationToken.None); + + Assert.NotNull(artifacts); + Assert.Equal(notePath, artifacts.MeetingNotePath); + Assert.Equal(Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), artifacts.TranscriptPath); + Assert.Equal(Path.Combine(root, "Meetings", "Assistant Context", "context.md"), artifacts.AssistantContextPath); + Assert.Equal(Path.Combine(summaries, "summary.md"), artifacts.SummaryPath); + } + + [Fact] + public async Task ResolverRejectsSummaryPathOutsideConfiguredSummaryFolder() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var options = Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions + { + MeetingNotesFolder = Path.Combine(root, "Notes"), + SummariesFolder = Path.Combine(root, "Summaries") + } + }); + var resolver = new MeetingSummaryArtifactResolver( + options, + new MarkdownMeetingNoteStore(options, NullLogger.Instance)); + + var artifacts = await resolver.ResolveBySummaryPathAsync( + Path.Combine(root, "Other", "summary.md"), + CancellationToken.None); + + Assert.Null(artifacts); + } +} diff --git a/MeetingAssistant.Tests/MeetingSummaryFailureWriterTests.cs b/MeetingAssistant.Tests/MeetingSummaryFailureWriterTests.cs new file mode 100644 index 0000000..c648003 --- /dev/null +++ b/MeetingAssistant.Tests/MeetingSummaryFailureWriterTests.cs @@ -0,0 +1,65 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingSummaryFailureWriterTests +{ + [Fact] + public async Task WriterOverwritesSummaryWithFailureDetails() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = new MeetingSessionArtifacts( + Path.Combine(root, "Notes", "meeting.md"), + Path.Combine(root, "Transcripts", "transcript.md"), + Path.Combine(root, "Assistant Context", "context.md"), + Path.Combine(root, "Summaries", "summary.md")); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!); + await File.WriteAllTextAsync( + artifacts.MeetingNotePath, + """ + --- + title: Failure Meeting + start_time: "2026-05-20T10:00:00.0000000+02:00" + end_time: "2026-05-20T10:30:00.0000000+02:00" + attendees: [] + projects: [] + transcript: "[[../Transcripts/transcript|Transcript]]" + assistant_context: "[[../Assistant Context/context|Assistant Context]]" + summary: "[[../Summaries/summary|Summary]]" + --- + """); + await File.WriteAllTextAsync(artifacts.SummaryPath, "# Old Summary"); + var writer = new MeetingSummaryFailureWriter( + Options.Create(new MeetingAssistantOptions + { + Api = new ApiOptions { PublicBaseUrl = "http://localhost:5090" } + }), + new MarkdownMeetingNoteStore( + Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions { MeetingNotesFolder = Path.GetDirectoryName(artifacts.MeetingNotePath)! } + }), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance)); + + var result = await writer.WriteAsync( + artifacts, + new InvalidOperationException("LiteLLM Responses request failed with 500"), + CancellationToken.None); + + var content = await File.ReadAllTextAsync(artifacts.SummaryPath); + Assert.False(result.Succeeded); + Assert.Equal(artifacts.SummaryPath, result.SummaryPath); + Assert.Contains("Summary Generation Failed", content); + Assert.Contains("title: Failure Meeting", content); + Assert.Contains("meeting: \"[[../Notes/meeting|Meeting Note]]\"", content); + Assert.Contains( + "[Retry summary generation](http://localhost:5090/meetings/summary/retry?summaryPath=summary.md)", + content); + Assert.Contains("LiteLLM Responses request failed with 500", content); + Assert.Contains(artifacts.TranscriptPath, content); + Assert.DoesNotContain("# Old Summary", content); + } +} diff --git a/MeetingAssistant.Tests/MeetingSummaryToolTests.cs b/MeetingAssistant.Tests/MeetingSummaryToolTests.cs new file mode 100644 index 0000000..292ba1b --- /dev/null +++ b/MeetingAssistant.Tests/MeetingSummaryToolTests.cs @@ -0,0 +1,145 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; + +namespace MeetingAssistant.Tests; + +public sealed class MeetingSummaryToolTests +{ + [Fact] + public async Task ToolsReadMeetingInputsAndWriteSummaryNote() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), + AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"), + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md")); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.TranscriptPath)!); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + await File.WriteAllTextAsync( + artifacts.MeetingNotePath, + """ + --- + title: Meeting + start_time: "2026-05-20T10:00:00.0000000+02:00" + end_time: "2026-05-20T10:30:00.0000000+02:00" + attendees: + - Ada + projects: + - Meeting Assistant + transcript: "[[../Transcripts/transcript|Transcript]]" + assistant_context: "[[../Assistant Context/context|Assistant Context]]" + summary: "[[../Summaries/summary|Summary]]" + --- + + User note line. + """); + await File.WriteAllTextAsync(artifacts.TranscriptPath, "Ada: Transcript line."); + await File.WriteAllTextAsync(artifacts.AssistantContextPath, "Context line."); + var tools = new MeetingSummaryTools(artifacts); + + Assert.Equal("Ada: Transcript line.", await tools.ReadTranscript()); + Assert.Contains("title: Meeting", await tools.ReadMeetingNote()); + Assert.Equal("Context line.", await tools.ReadContext()); + Assert.Equal("User note line.", await tools.ReadUserNotes()); + Assert.Equal("", await tools.ReadGlossary()); + + var result = await tools.WriteSummary("# Summary\n\n- Done."); + + Assert.Equal(artifacts.SummaryPath, result); + var summary = await File.ReadAllTextAsync(artifacts.SummaryPath); + Assert.Contains("title: Meeting", summary); + Assert.Contains("start_time: \"2026-05-20T10:00:00.0000000+02:00\"", summary); + Assert.Contains("end_time: \"2026-05-20T10:30:00.0000000+02:00\"", summary); + Assert.Contains("meeting: \"[[../Notes/meeting|Meeting Note]]\"", summary); + Assert.Contains("transcript: \"[[../Transcripts/transcript|Transcript]]\"", summary); + Assert.Contains("assistant_context: \"[[../Assistant Context/context|Assistant Context]]\"", summary); + Assert.Contains("# Summary\n\n- Done.", summary); + } + + [Fact] + public async Task ReadToolsSupportClampedLineRanges() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), + AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"), + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md")); + var glossaryPath = Path.Combine(root, "Meetings", "dictation-words.md"); + var projectRoot = Path.Combine(root, "Projects", "Project A"); + var projectFile = Path.Combine(projectRoot, "notes.md"); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.TranscriptPath)!); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + Directory.CreateDirectory(Path.GetDirectoryName(glossaryPath)!); + Directory.CreateDirectory(projectRoot); + await File.WriteAllTextAsync( + artifacts.MeetingNotePath, + """ + --- + projects: + - Project A + --- + + note one + note two + note three + """); + await File.WriteAllTextAsync(artifacts.TranscriptPath, "transcript one\ntranscript two\ntranscript three"); + await File.WriteAllTextAsync(artifacts.AssistantContextPath, "context one\ncontext two\ncontext three"); + await File.WriteAllTextAsync(glossaryPath, "word one\nword two\nword three"); + await File.WriteAllTextAsync(projectFile, "project one\nproject two\nproject three"); + var tools = new MeetingSummaryTools( + artifacts, + new MeetingAssistantOptions + { + Vault = + { + DictationWordsPath = glossaryPath, + ProjectsFolder = Path.Combine(root, "Projects") + } + }); + + Assert.Equal("transcript two", await tools.ReadTranscript(from: 2, to: 2)); + Assert.Equal("context two\ncontext three", await tools.ReadContext(from: 2, to: 9)); + Assert.Equal("note two\nnote three", await tools.ReadUserNotes(from: 2, to: 99)); + Assert.Equal("word two", await tools.ReadGlossary(from: 2, to: 2)); + Assert.Equal("project two", await tools.ReadProjectFile("Project A", "notes.md", from: 2, to: 2)); + Assert.Equal("", await tools.ReadTranscript(from: 3, to: 2)); + } + + [Fact] + public async Task ToolsWriteAssistantContextBodyWithLineModes() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var artifacts = new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), + AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"), + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md")); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + await File.WriteAllTextAsync( + artifacts.AssistantContextPath, + """ + --- + meeting: "[[../Notes/meeting|Meeting Note]]" + transcript: "[[../Transcripts/transcript|Transcript]]" + summary: "[[../Summaries/summary|Summary]]" + state: summarizing + --- + + line one + line two + """); + var tools = new MeetingSummaryTools(artifacts); + + await tools.WriteContext("inserted", insert: 2); + await tools.WriteContext("replacement", from: 1, to: 1); + + var context = await File.ReadAllTextAsync(artifacts.AssistantContextPath); + Assert.Contains("state: summarizing", context); + Assert.Contains("replacement\ninserted\nline two", context); + } +} diff --git a/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs new file mode 100644 index 0000000..307369e --- /dev/null +++ b/MeetingAssistant.Tests/OutlookClassicMeetingMetadataProviderTests.cs @@ -0,0 +1,25 @@ +using MeetingAssistant.MeetingNotes; + +namespace MeetingAssistant.Tests; + +#if WINDOWS +public sealed class OutlookClassicMeetingMetadataProviderTests +{ + [Fact] + public void ExtractAgendaStopsBeforeTeamsJoinInformation() + { + var agenda = OutlookClassicMeetingMetadataProvider.ExtractAgenda( + """ + Review current prototype + Decide next backend + + ________________________________________________________________________________ + Microsoft Teams Need help? + Join the meeting now + https://teams.microsoft.com/l/meetup-join/... + """); + + Assert.Equal("Review current prototype\nDecide next backend", agenda.Replace("\r\n", "\n", StringComparison.Ordinal)); + } +} +#endif diff --git a/MeetingAssistant.Tests/OutlookMeetingCandidateSelectorTests.cs b/MeetingAssistant.Tests/OutlookMeetingCandidateSelectorTests.cs new file mode 100644 index 0000000..c8fac0c --- /dev/null +++ b/MeetingAssistant.Tests/OutlookMeetingCandidateSelectorTests.cs @@ -0,0 +1,72 @@ +using MeetingAssistant.MeetingNotes; + +namespace MeetingAssistant.Tests; + +public sealed class OutlookMeetingCandidateSelectorTests +{ + [Fact] + public void SelectIgnoresOverlappingMeetingThatEndsInLessThanFiveMinutes() + { + var now = new DateTime(2026, 5, 20, 10, 0, 0); + var selected = OutlookMeetingCandidateSelector.Select( + [ + new Candidate(now.AddMinutes(-25), now.AddMinutes(4)) + ], + now, + candidate => candidate.Start, + candidate => candidate.End); + + Assert.Null(selected); + } + + [Fact] + public void SelectPrefersSingleGoodOverlappingMeetingOverUpcomingMeeting() + { + var now = new DateTime(2026, 5, 20, 10, 0, 0); + var overlap = new Candidate(now.AddMinutes(-10), now.AddMinutes(30)); + var upcoming = new Candidate(now.AddMinutes(3), now.AddMinutes(33)); + + var selected = OutlookMeetingCandidateSelector.Select( + [upcoming, overlap], + now, + candidate => candidate.Start, + candidate => candidate.End); + + Assert.Same(overlap, selected); + } + + [Fact] + public void SelectUsesSingleTeamsMeetingStartingWithinFiveMinutesWhenNoGoodOverlapExists() + { + var now = new DateTime(2026, 5, 20, 10, 0, 0); + var endingOverlap = new Candidate(now.AddMinutes(-25), now.AddMinutes(2)); + var upcoming = new Candidate(now.AddMinutes(5), now.AddMinutes(35)); + + var selected = OutlookMeetingCandidateSelector.Select( + [endingOverlap, upcoming], + now, + candidate => candidate.Start, + candidate => candidate.End); + + Assert.Same(upcoming, selected); + } + + [Fact] + public void SelectRejectsAmbiguousUpcomingMeetings() + { + var now = new DateTime(2026, 5, 20, 10, 0, 0); + + var selected = OutlookMeetingCandidateSelector.Select( + [ + new Candidate(now.AddMinutes(2), now.AddMinutes(32)), + new Candidate(now.AddMinutes(4), now.AddMinutes(34)) + ], + now, + candidate => candidate.Start, + candidate => candidate.End); + + Assert.Null(selected); + } + + private sealed record Candidate(DateTime Start, DateTime End); +} diff --git a/MeetingAssistant.Tests/ProjectKnowledgeToolTests.cs b/MeetingAssistant.Tests/ProjectKnowledgeToolTests.cs new file mode 100644 index 0000000..fa89ad3 --- /dev/null +++ b/MeetingAssistant.Tests/ProjectKnowledgeToolTests.cs @@ -0,0 +1,125 @@ +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; + +namespace MeetingAssistant.Tests; + +public sealed class ProjectKnowledgeToolTests +{ + [Fact] + public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var projectsRoot = Path.Combine(root, "Projects"); + var meetingAssistantRoot = Path.Combine(projectsRoot, "MeetingAssistant"); + var ignoredRoot = Path.Combine(projectsRoot, "IgnoredProject"); + Directory.CreateDirectory(Path.Combine(meetingAssistantRoot, "notes")); + Directory.CreateDirectory(ignoredRoot); + await File.WriteAllTextAsync( + Path.Combine(meetingAssistantRoot, "README.md"), + "First line\nSecond alpha line\nThird beta line"); + await File.WriteAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "context.md"), "Project context"); + await File.WriteAllTextAsync(Path.Combine(ignoredRoot, "ignored.md"), "alpha should not be searched"); + + var artifacts = CreateArtifacts(root); + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!); + await File.WriteAllTextAsync( + artifacts.MeetingNotePath, + """ + --- + title: Project Meeting + attendees: [] + projects: + - MeetingAssistant + - MissingProject + transcript: "" + assistant_context: "" + summary: "" + --- + + User notes. + """); + var tools = new MeetingSummaryTools( + artifacts, + new MeetingAssistantOptions + { + Vault = + { + ProjectsFolder = projectsRoot + } + }); + + Assert.Equal("MeetingAssistant", await tools.ListProjects()); + Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant")); + Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99)); + + var writeResult = await tools.WriteProjectFile("MeetingAssistant", "notes/summary.md", "# Project Update"); + + Assert.Equal("MeetingAssistant/notes/summary.md", writeResult); + Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md"))); + Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed")); + Assert.Equal("changed", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md"))); + Assert.Equal( + "README.md:2 Second alpha line", + await tools.Search("alpha")); + } + + [Fact] + public async Task WriteProjectFileSupportsOverwriteReplaceInsertAndCreate() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var projectsRoot = Path.Combine(root, "Projects"); + var projectRoot = Path.Combine(projectsRoot, "MeetingAssistant"); + Directory.CreateDirectory(projectRoot); + var projectFile = Path.Combine(projectRoot, "notes.md"); + await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour"); + var tools = new MeetingSummaryTools( + CreateArtifacts(root), + new MeetingAssistantOptions + { + Vault = + { + ProjectsFolder = projectsRoot + } + }); + + await tools.WriteProjectFile("MeetingAssistant", "notes.md", "TWO\nTHREE", from: 2, to: 3); + Assert.Equal("one\nTWO\nTHREE\nfour", await File.ReadAllTextAsync(projectFile)); + + await tools.WriteProjectFile("MeetingAssistant", "notes.md", "inserted", insert: 2); + Assert.Equal("one\ninserted\nTWO\nTHREE\nfour", await File.ReadAllTextAsync(projectFile)); + + await tools.WriteProjectFile("MeetingAssistant", "created/new.md", "created content"); + Assert.Equal("created content", await File.ReadAllTextAsync(Path.Combine(projectRoot, "created", "new.md"))); + } + + [Fact] + public async Task WriteProjectFileRefusesMissingProjectsEscapingPathsAndAmbiguousLineArguments() + { + var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var projectsRoot = Path.Combine(root, "Projects"); + Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant")); + var tools = new MeetingSummaryTools( + CreateArtifacts(root), + new MeetingAssistantOptions + { + Vault = + { + ProjectsFolder = projectsRoot + } + }); + + Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content")); + Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content")); + Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1)); + Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1)); + } + + private static MeetingSessionArtifacts CreateArtifacts(string root) + { + return new MeetingSessionArtifacts( + MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"), + TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"), + AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"), + SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md")); + } +} diff --git a/MeetingAssistant.Tests/PyannoteTranscriptFinalizerTests.cs b/MeetingAssistant.Tests/PyannoteTranscriptFinalizerTests.cs new file mode 100644 index 0000000..96c61f0 --- /dev/null +++ b/MeetingAssistant.Tests/PyannoteTranscriptFinalizerTests.cs @@ -0,0 +1,265 @@ +using MeetingAssistant; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Tests; + +public sealed class PyannoteTranscriptFinalizerTests +{ + [Fact] + public async Task FinalizerAssignsPyannoteSpeakersToLiveWhisperSegmentsByOverlap() + { + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + var commandRunner = new CapturingCommandRunner( + """ + install noise + __MEETING_ASSISTANT_PYANNOTE_JSON_START__ + [{"start":0.0,"end":1.5,"speaker":"SPEAKER_00"},{"start":1.5,"end":3.0,"speaker":"SPEAKER_01"}] + __MEETING_ASSISTANT_PYANNOTE_JSON_END__ + """); + var finalizer = CreateFinalizer( + commandRunner, + token: "hf_test", + alignmentMode: PyannoteAlignmentMode.BestOverlap); + var liveSegments = new[] + { + new TranscriptionSegment(TimeSpan.FromSeconds(0.2), TimeSpan.FromSeconds(1.2), "Unknown", "hello"), + new TranscriptionSegment(TimeSpan.FromSeconds(1.7), TimeSpan.FromSeconds(2.4), "Unknown", "there") + }; + + var segments = await finalizer.FinalizeAsync( + audioPath, + liveSegments, + SpeechRecognitionPipelineOptions.Default, + CancellationToken.None); + + Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("inspect")); + Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("--format")); + Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("build")); + Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote:local")); + Assert.Equal("docker", commandRunner.Commands.Last().FileName); + Assert.Contains("run", commandRunner.Commands.Last().Arguments); + Assert.Contains("meeting-assistant-pyannote:local", commandRunner.Commands.Last().Arguments); + Assert.Contains("HF_TOKEN", commandRunner.Commands.Last().Arguments); + Assert.DoesNotContain("HF_TOKEN=hf_test", commandRunner.Commands.Last().Arguments); + Assert.Equal("hf_test", commandRunner.Environment["HF_TOKEN"]); + Assert.Contains("sh", commandRunner.Commands.Last().Arguments); + Assert.DoesNotContain("bash", commandRunner.Commands.Last().Arguments); + Assert.DoesNotContain(commandRunner.Commands.Last().Arguments, argument => argument.Contains("pip install", StringComparison.Ordinal)); + Assert.DoesNotContain(commandRunner.Commands.Last().Arguments, argument => argument.Contains("apt-get install", StringComparison.Ordinal)); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("PIP_CACHE_DIR=/workspace/cache/pip", StringComparison.Ordinal)); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("HF_HOME=/workspace/cache/huggingface", StringComparison.Ordinal)); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("TORCH_HOME=/workspace/cache/torch", StringComparison.Ordinal)); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("speaker_diarization", StringComparison.Ordinal)); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("token=os.environ.get('HF_TOKEN')", StringComparison.Ordinal)); + Assert.DoesNotContain(commandRunner.Commands.Last().Arguments, argument => argument.Contains("use_auth_token", StringComparison.Ordinal)); + Assert.Collection( + segments, + first => + { + Assert.Equal("SPEAKER_00", first.Speaker); + Assert.Equal("hello", first.Text); + }, + second => + { + Assert.Equal("SPEAKER_01", second.Speaker); + Assert.Equal("there", second.Text); + }); + } + + [Fact] + public async Task FinalizerCanBuildTranscriptSegmentsFromPyannoteTurns() + { + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + var commandRunner = new CapturingCommandRunner( + """ + __MEETING_ASSISTANT_PYANNOTE_JSON_START__ + [{"start":0.0,"end":1.0,"speaker":"SPEAKER_00"},{"start":1.0,"end":2.0,"speaker":"SPEAKER_01"}] + __MEETING_ASSISTANT_PYANNOTE_JSON_END__ + """); + var finalizer = CreateFinalizer( + commandRunner, + token: "hf_test", + annotationSource: PyannoteAnnotationSource.ExclusiveSpeakerDiarization, + alignmentMode: PyannoteAlignmentMode.PyannoteTurns); + var liveSegments = new[] + { + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(2), "Unknown", "alpha beta gamma delta") + }; + + var segments = await finalizer.FinalizeAsync( + audioPath, + liveSegments, + SpeechRecognitionPipelineOptions.Default, + CancellationToken.None); + + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("exclusive_speaker_diarization", StringComparison.Ordinal)); + Assert.Collection( + segments, + first => + { + Assert.Equal(TimeSpan.Zero, first.Start); + Assert.Equal(TimeSpan.FromSeconds(1), first.End); + Assert.Equal("SPEAKER_00", first.Speaker); + Assert.Equal("alpha beta", first.Text); + }, + second => + { + Assert.Equal(TimeSpan.FromSeconds(1), second.Start); + Assert.Equal(TimeSpan.FromSeconds(2), second.End); + Assert.Equal("SPEAKER_01", second.Speaker); + Assert.Equal("gamma delta", second.Text); + }); + } + + [Fact] + public async Task FinalizerPassesConfiguredNumSpeakersToPyannote() + { + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + var commandRunner = new CapturingCommandRunner( + """ + __MEETING_ASSISTANT_PYANNOTE_JSON_START__ + [{"start":0.0,"end":1.0,"speaker":"SPEAKER_00"}] + __MEETING_ASSISTANT_PYANNOTE_JSON_END__ + """); + var finalizer = CreateFinalizer(commandRunner, token: "hf_test"); + + await finalizer.FinalizeAsync( + audioPath, + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")], + new SpeechRecognitionPipelineOptions(5), + CancellationToken.None); + + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("num_speakers=5", StringComparison.Ordinal)); + } + + [Fact] + public async Task FinalizerCanUseExplicitDiarizationOptions() + { + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + var commandRunner = new CapturingCommandRunner( + """ + __MEETING_ASSISTANT_PYANNOTE_JSON_START__ + [{"start":0.0,"end":1.0,"speaker":"SPEAKER_00"}] + __MEETING_ASSISTANT_PYANNOTE_JSON_END__ + """); + var finalizer = new PyannoteTranscriptFinalizer( + commandRunner, + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + var explicitDiarization = new PyannoteDiarizationOptions + { + Enabled = true, + DockerCommand = "docker", + BaseImage = "python:3.11-slim", + Image = "meeting-assistant-pyannote-azure:local", + ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models"), + Model = "custom/diarization", + Token = "hf_azure", + TokenEnv = "", + CommandTimeout = TimeSpan.FromMinutes(1) + }; + + await finalizer.FinalizeAsync( + audioPath, + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")], + explicitDiarization, + SpeechRecognitionPipelineOptions.Default, + CancellationToken.None); + + Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-azure:local")); + Assert.Contains(commandRunner.Commands.Last().Arguments, argument => argument.Contains("custom/diarization", StringComparison.Ordinal)); + Assert.Equal("hf_azure", commandRunner.Environment["HF_TOKEN"]); + } + + [Fact] + public async Task FinalizerSkipsPyannoteWhenTokenIsMissing() + { + var commandRunner = new CapturingCommandRunner(""); + var finalizer = CreateFinalizer(commandRunner, token: null); + var audioPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "meeting.wav"); + Directory.CreateDirectory(Path.GetDirectoryName(audioPath)!); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3, 4]); + + var segments = await finalizer.FinalizeAsync( + audioPath, + [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")], + SpeechRecognitionPipelineOptions.Default, + CancellationToken.None); + + Assert.Empty(segments); + Assert.Empty(commandRunner.Commands); + } + + private static PyannoteTranscriptFinalizer CreateFinalizer( + CapturingCommandRunner commandRunner, + string? token, + PyannoteAnnotationSource annotationSource = PyannoteAnnotationSource.SpeakerDiarization, + PyannoteAlignmentMode alignmentMode = PyannoteAlignmentMode.PyannoteTurns) + { + return new PyannoteTranscriptFinalizer( + commandRunner, + Options.Create(new MeetingAssistantOptions + { + WhisperLocal = new WhisperLocalOptions + { + Diarization = new PyannoteDiarizationOptions + { + Enabled = true, + DockerCommand = "docker", + BaseImage = "python:3.11-slim", + Image = "meeting-assistant-pyannote:local", + ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models"), + AnnotationSource = annotationSource, + AlignmentMode = alignmentMode, + Token = token, + TokenEnv = "", + CommandTimeout = TimeSpan.FromMinutes(1) + } + } + }), + NullLogger.Instance); + } + + private sealed class CapturingCommandRunner : ICommandRunner + { + private readonly string output; + + public CapturingCommandRunner(string output) + { + this.output = output; + } + + public IReadOnlyList Commands { get; private set; } = []; + + public IReadOnlyDictionary Environment { get; private set; } = + new Dictionary(); + + public Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + Commands = Commands.Append(new CapturedCommand(fileName, arguments)).ToList(); + Environment = environment ?? new Dictionary(); + if (arguments.Contains("inspect")) + { + return Task.FromResult(new CommandResult(1, "", "image missing")); + } + + return Task.FromResult(new CommandResult(0, output, "")); + } + } + + private sealed record CapturedCommand(string FileName, IReadOnlyList Arguments); +} diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs new file mode 100644 index 0000000..fe1f3ac --- /dev/null +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -0,0 +1,930 @@ +using MeetingAssistant.Recording; +using MeetingAssistant.Transcription; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using System.Threading.Channels; + +namespace MeetingAssistant.Tests; + +public sealed class RecordingCoordinatorTests +{ + [Fact] + public async Task ToggleStartsStreamingTranscriptionAndSecondToggleStopsIt() + { + var audioSource = new ControlledAudioSource(); + var provider = new EchoStreamingTranscriptionProvider(); + var store = new InMemoryTranscriptStore(); + var noteStore = new InMemoryMeetingNoteStore(); + var noteOpener = new CapturingMeetingNoteOpener(); + var artifactStore = new InMemoryMeetingArtifactStore(); + var audioArchive = new InMemoryRecordedAudioStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + store, + noteStore, + noteOpener, + artifactStore, + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + var started = await coordinator.ToggleAsync(CancellationToken.None); + await audioSource.WriteAsync(new AudioChunk(new byte[] { 1, 0 }, 16000, 1), CancellationToken.None); + + await store.WaitForTextAsync("chunk:2"); + var stopped = await coordinator.ToggleAsync(CancellationToken.None); + + Assert.True(started.IsRecording); + Assert.Equal(noteStore.SavedNote?.Path, noteOpener.OpenedPath); + Assert.Equal(started.MeetingNotePath, noteOpener.OpenedPath); + Assert.Equal(started.MeetingNotePath, artifactStore.CreatedArtifacts?.MeetingNotePath); + Assert.True(provider.FirstChunkWasObservedBeforeSourceCompleted); + Assert.False(stopped.IsRecording); + } + + [Fact] + public async Task StartCreatesMeetingNoteLinkedToTranscriptAndOpensIt() + { + var audioSource = new ControlledAudioSource(); + var transcriptStore = new InMemoryTranscriptStore("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md"); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md"); + var noteOpener = new CapturingMeetingNoteOpener(); + var artifactStore = new InMemoryMeetingArtifactStore(); + var audioArchive = new InMemoryRecordedAudioStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + transcriptStore, + noteStore, + noteOpener, + artifactStore, + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions + { + AssistantContextFolder = "C:\\Vault\\Meetings\\Assistant Context", + SummariesFolder = "C:\\Vault\\Meetings\\Summaries" + } + }), + NullLogger.Instance); + + var status = await coordinator.StartAsync(CancellationToken.None); + + Assert.True(status.IsRecording); + Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", status.MeetingNotePath); + Assert.Equal("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md", noteStore.SavedNote?.Frontmatter.Transcript); + Assert.StartsWith("C:\\Vault\\Meetings\\Assistant Context\\", noteStore.SavedNote?.Frontmatter.AssistantContext, StringComparison.Ordinal); + Assert.EndsWith("-assistant-context.md", noteStore.SavedNote?.Frontmatter.AssistantContext, StringComparison.Ordinal); + Assert.StartsWith("C:\\Vault\\Meetings\\Summaries\\", noteStore.SavedNote?.Frontmatter.Summary, StringComparison.Ordinal); + Assert.EndsWith("-summary.md", noteStore.SavedNote?.Frontmatter.Summary, StringComparison.Ordinal); + Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", noteOpener.OpenedPath); + Assert.Equal("C:\\Vault\\Meetings\\Notes\\20260519-meeting.md", artifactStore.CreatedArtifacts?.MeetingNotePath); + Assert.Equal("C:\\Vault\\Meetings\\Transcripts\\20260519-transcript.md", artifactStore.CreatedArtifacts?.TranscriptPath); + Assert.StartsWith("C:\\Vault\\Meetings\\Assistant Context\\", artifactStore.CreatedArtifacts?.AssistantContextPath, StringComparison.Ordinal); + Assert.StartsWith("C:\\Vault\\Meetings\\Summaries\\", artifactStore.CreatedArtifacts?.SummaryPath, StringComparison.Ordinal); + + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartUsesCurrentOutlookMeetingMetadataWhenAvailable() + { + var audioSource = new ControlledAudioSource(); + var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\metadata-meeting.md"); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance, + new FixedMeetingMetadataProvider(new MeetingMetadata( + "Architecture Sync", + ["Ada ", "Grace"], + "Review API shape", + DateTimeOffset.Parse("2026-05-19T11:00:00+02:00")))); + + await coordinator.StartAsync(CancellationToken.None); + + Assert.Equal("Architecture Sync", noteStore.SavedNote?.Frontmatter.Title); + Assert.Equal(["Ada ", "Grace"], noteStore.SavedNote?.Frontmatter.Attendees); + Assert.Equal("Review API shape", artifactStore.Agenda); + Assert.Equal(DateTimeOffset.Parse("2026-05-19T11:00:00+02:00"), artifactStore.ScheduledEnd); + + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopCompletesAudioCaptureAndDrainsFinalTranscriptionWindow() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk(new byte[] { 1, 0 }, 16000, 1)); + var provider = new FinalSegmentOnAudioCompletionProvider(); + var store = new InMemoryTranscriptStore(); + var noteStore = new InMemoryMeetingNoteStore(); + var noteOpener = new CapturingMeetingNoteOpener(); + var artifactStore = new InMemoryMeetingArtifactStore(); + var audioArchive = new InMemoryRecordedAudioStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + store, + noteStore, + noteOpener, + artifactStore, + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + var stopped = await coordinator.StopAsync(CancellationToken.None); + + Assert.False(stopped.IsRecording); + await store.WaitForTextAsync("final:2"); + } + + [Fact] + public async Task CaptureStartsEvenWhenTranscriptionProviderIsStillWarmingUp() + { + var audioSource = new ControlledAudioSource(); + var provider = new BlockingBeforeTranscriptionProvider(); + var store = new InMemoryTranscriptStore(); + var audioArchive = new InMemoryRecordedAudioStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(provider), + store, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await provider.WaitUntilWaitingForBackendAsync(); + await audioSource.WriteAsync(new AudioChunk(new byte[] { 1, 0, 2, 0 }, 16000, 1), CancellationToken.None); + + await audioArchive.WaitForAppendAsync(); + Assert.Equal([4], audioArchive.AppendedChunkSizes); + + provider.MarkBackendReady(); + await store.WaitForTextAsync("chunk:4"); + await coordinator.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopRewritesTranscriptWithFinalDiarizedSegmentsWhenAvailable() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk(new byte[] { 1, 0, 2, 0 }, 16000, 1)); + var transcriptStore = new InMemoryTranscriptStore(); + var finalizer = new CapturingTranscriptFinalizer( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Speaker 0", "hello"), + new TranscriptionSegment(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), "Speaker 1", "there") + ]); + var audioArchive = new InMemoryRecordedAudioStore("memory-recording.wav"); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider(), finalizer.FinalizeAsync), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + audioArchive, + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal("memory-recording.wav", finalizer.AudioPath); + Assert.Collection( + finalizer.LiveSegments, + segment => Assert.Equal("final:4", segment.Text)); + Assert.Equal([4], audioArchive.AppendedChunkSizes); + Assert.True(audioArchive.Completed); + Assert.True(audioArchive.Deleted); + Assert.Collection( + transcriptStore.ReplacedSegments, + first => + { + Assert.Equal("Speaker 0", first.Speaker); + Assert.Equal("hello", first.Text); + }, + second => + { + Assert.Equal("Speaker 1", second.Speaker); + Assert.Equal("there", second.Text); + }); + } + + [Theory] + [InlineData(0, null)] + [InlineData(1, null)] + [InlineData(2, 2)] + [InlineData(5, 5)] + public async Task StopUsesMeetingNoteAttendeeCountAsSpeakerHintWhenThereAreMultipleAttendees( + int attendeeCount, + int? expectedNumSpeakers) + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var noteStore = new InMemoryMeetingNoteStore(); + var finalizer = new CapturingTranscriptFinalizer( + [ + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Speaker 0", "hello") + ]); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider(), finalizer.FinalizeAsync), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + noteStore.UpdateAttendees(Enumerable.Range(1, attendeeCount).Select(index => $"Person {index}")); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal(expectedNumSpeakers, finalizer.Options?.NumSpeakers); + } + + [Fact] + public async Task StopAddsMeetingEndTimeAndRunsSummaryAfterFinishedTranscript() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var noteStore = new InMemoryMeetingNoteStore(); + var artifactStore = new InMemoryMeetingArtifactStore(); + var summaryPipeline = new CapturingMeetingSummaryPipeline(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + new InMemoryTranscriptStore(), + noteStore, + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + summaryPipeline, + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + var startTime = noteStore.SavedNote?.Frontmatter.StartTime; + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.NotNull(startTime); + Assert.NotNull(noteStore.SavedNote?.Frontmatter.EndTime); + Assert.True(noteStore.SavedNote?.Frontmatter.EndTime >= startTime); + Assert.Equal(artifactStore.CreatedArtifacts, summaryPipeline.Artifacts); + } + + [Fact] + public async Task StopUpdatesTranscriptMetadataWithMeetingEndTime() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var transcriptStore = new InMemoryTranscriptStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + transcriptStore, + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + new InMemoryMeetingArtifactStore(), + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.NotNull(transcriptStore.MetadataMeetingNote?.Frontmatter.EndTime); + } + + [Fact] + public async Task StopUpdatesAssistantContextStateThroughSummaryLifecycle() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(CreateOptionsWithoutFinalizer()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal( + [AssistantContextState.Summarizing, AssistantContextState.Finished], + artifactStore.States); + } + + [Fact] + public async Task StopMarksAssistantContextErrorWhenSummaryFails() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(succeeded: false), + Options.Create(CreateOptionsWithoutFinalizer()), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal( + [AssistantContextState.Summarizing, AssistantContextState.Error], + artifactStore.States); + } + + [Fact] + public async Task StopMarksSpeakerRecognitionBeforeSummaryWhenFinalizerIsConfigured() + { + var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1)); + var artifactStore = new InMemoryMeetingArtifactStore(); + var coordinator = new MeetingRecordingCoordinator( + audioSource, + new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()), + new InMemoryTranscriptStore(), + new InMemoryMeetingNoteStore(), + new CapturingMeetingNoteOpener(), + artifactStore, + new InMemoryRecordedAudioStore(), + new CapturingMeetingSummaryPipeline(), + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions { TranscriptionProvider = "whisper-local" }, + WhisperLocal = new WhisperLocalOptions + { + Diarization = new PyannoteDiarizationOptions { Enabled = true } + } + }), + NullLogger.Instance); + + await coordinator.StartAsync(CancellationToken.None); + await audioSource.WaitUntilCapturedAsync(); + + await coordinator.StopAsync(CancellationToken.None); + + Assert.Equal( + [AssistantContextState.SpeakerRecognition, AssistantContextState.Summarizing, AssistantContextState.Finished], + artifactStore.States); + } + + [Fact] + public async Task VaultTranscriptStoreCreatesConfiguredFolderAndAppendsSegments() + { + var vaultFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var store = new VaultTranscriptStore( + Options.Create(new MeetingAssistantOptions + { + Vault = new VaultOptions { TranscriptsFolder = vaultFolder } + }), + NullLogger.Instance); + + var session = await store.CreateSessionAsync(CancellationToken.None); + await store.AppendAsync( + session, + new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello vault"), + CancellationToken.None); + + Assert.True(Directory.Exists(vaultFolder)); + Assert.EndsWith(".md", session.TranscriptPath, StringComparison.Ordinal); + Assert.Contains("hello vault", await File.ReadAllTextAsync(session.TranscriptPath)); + } + + [Fact] + public async Task TemporaryRecordedAudioStoreCreatesConfiguredFolderAndWritesPcmWav() + { + var recordingFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var store = new TemporaryRecordedAudioStore( + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions + { + SampleRate = 16000, + Channels = 1, + TemporaryRecordingsFolder = recordingFolder + } + }), + NullLogger.Instance); + + await using var session = await store.CreateSessionAsync(CancellationToken.None); + await session.AppendAsync(new AudioChunk([1, 0, 2, 0], 16000, 1), CancellationToken.None); + await session.CompleteAsync(CancellationToken.None); + + Assert.True(Directory.Exists(recordingFolder)); + Assert.EndsWith(".wav", session.AudioPath, StringComparison.Ordinal); + Assert.True(new FileInfo(session.AudioPath).Length > 44); + } + + [Fact] + public async Task TemporaryRecordedAudioStoreDeletesStaleRecordingsOnStartup() + { + var recordingFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(recordingFolder); + var staleRecording = Path.Combine(recordingFolder, "stale.wav"); + var unrelatedFile = Path.Combine(recordingFolder, "keep.txt"); + await File.WriteAllTextAsync(staleRecording, "stale"); + await File.WriteAllTextAsync(unrelatedFile, "keep"); + var store = new TemporaryRecordedAudioStore( + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions { TemporaryRecordingsFolder = recordingFolder } + }), + NullLogger.Instance); + + await store.DeleteStaleRecordingsAsync(CancellationToken.None); + + Assert.False(File.Exists(staleRecording)); + Assert.True(File.Exists(unrelatedFile)); + } + + private static MeetingAssistantOptions CreateOptionsWithoutFinalizer() + { + return new MeetingAssistantOptions + { + Recording = new RecordingOptions { TranscriptionProvider = "whisper-local" }, + WhisperLocal = new WhisperLocalOptions + { + Diarization = new PyannoteDiarizationOptions { Enabled = false } + } + }; + } + + private sealed class InMemoryTranscriptStore : ITranscriptStore + { + private readonly List segments = []; + private readonly TaskCompletionSource segmentWritten = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly string transcriptPath; + + public InMemoryTranscriptStore(string transcriptPath = "memory-transcript.md") + { + this.transcriptPath = transcriptPath; + } + + public Task CreateSessionAsync(CancellationToken cancellationToken) + { + return Task.FromResult(new TranscriptSession(transcriptPath)); + } + + public Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken) + { + segments.Add(segment); + segmentWritten.TrySetResult(); + return Task.CompletedTask; + } + + public async Task WaitForTextAsync(string text) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(5); + while (DateTimeOffset.UtcNow < deadline) + { + if (segments.Any(segment => segment.Text.Contains(text, StringComparison.Ordinal))) + { + return; + } + + await segmentWritten.Task.WaitAsync(TimeSpan.FromMilliseconds(100)); + } + + throw new TimeoutException($"Segment containing '{text}' was not written."); + } + + public IReadOnlyList ReplacedSegments { get; private set; } = []; + + public MeetingNote? MetadataMeetingNote { get; private set; } + + public Task ReplaceAsync( + TranscriptSession session, + IReadOnlyList replacementSegments, + CancellationToken cancellationToken) + { + ReplacedSegments = replacementSegments; + return Task.CompletedTask; + } + + public Task UpdateMetadataAsync( + TranscriptSession session, + MeetingSessionArtifacts artifacts, + MeetingNote meetingNote, + CancellationToken cancellationToken) + { + MetadataMeetingNote = meetingNote; + return Task.CompletedTask; + } + } + + private sealed class InMemoryMeetingNoteStore : IMeetingNoteStore + { + private readonly string notePath; + + public InMemoryMeetingNoteStore(string notePath = "memory-meeting.md") + { + this.notePath = notePath; + } + + public MeetingNote? SavedNote { get; private set; } + + public Task SaveAsync(MeetingNote note, CancellationToken cancellationToken) + { + SavedNote = note with { Path = notePath }; + return Task.FromResult(SavedNote); + } + + public Task ReadAsync(string path, CancellationToken cancellationToken) + { + return Task.FromResult(SavedNote ?? throw new FileNotFoundException(path)); + } + + public void UpdateAttendees(IEnumerable attendees) + { + if (SavedNote is null) + { + throw new InvalidOperationException("No meeting note has been saved."); + } + + SavedNote.Frontmatter.Attendees = attendees.ToList(); + } + } + + private sealed class CapturingMeetingNoteOpener : IMeetingNoteOpener + { + public string? OpenedPath { get; private set; } + + public Task OpenAsync(string notePath, CancellationToken cancellationToken) + { + OpenedPath = notePath; + return Task.CompletedTask; + } + } + + private sealed class InMemoryMeetingArtifactStore : IMeetingArtifactStore + { + public MeetingSessionArtifacts? CreatedArtifacts { get; private set; } + + public List States { get; } = []; + + public string? Agenda { get; private set; } + + public DateTimeOffset? ScheduledEnd { get; private set; } + + public Task CreateAssistantContextAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken) + { + CreatedArtifacts = artifacts; + Agenda = agenda; + ScheduledEnd = scheduledEnd; + return Task.CompletedTask; + } + + public Task UpdateAssistantContextStateAsync( + MeetingSessionArtifacts artifacts, + AssistantContextState state, + CancellationToken cancellationToken) + { + States.Add(state); + return Task.CompletedTask; + } + } + + private sealed class FixedMeetingMetadataProvider : IMeetingMetadataProvider + { + private readonly MeetingMetadata? metadata; + + public FixedMeetingMetadataProvider(MeetingMetadata? metadata) + { + this.metadata = metadata; + } + + public Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + return Task.FromResult(metadata); + } + } + + private sealed class CapturingMeetingSummaryPipeline : IMeetingSummaryPipeline + { + private readonly bool succeeded; + + public CapturingMeetingSummaryPipeline(bool succeeded = true) + { + this.succeeded = succeeded; + } + + public MeetingSessionArtifacts? Artifacts { get; private set; } + + public Task RunAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + Artifacts = artifacts; + return Task.FromResult(new MeetingSummaryRunResult( + artifacts.SummaryPath, + succeeded ? "summary ok" : "summary failed", + succeeded, + succeeded ? null : "error")); + } + } + + private sealed class InMemoryRecordedAudioStore : IRecordedAudioStore + { + private readonly string audioPath; + + public InMemoryRecordedAudioStore(string audioPath = "memory-recording.wav") + { + this.audioPath = audioPath; + } + + public List AppendedChunkSizes { get; } = []; + + public bool Completed { get; private set; } + + public bool Deleted { get; private set; } + + private TaskCompletionSource AppendObserved { get; set; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitForAppendAsync() + { + return AppendObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public Task CreateSessionAsync(CancellationToken cancellationToken) + { + return Task.FromResult(new Sink(this, audioPath)); + } + + public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + private sealed class Sink : IRecordedAudioSink + { + private readonly InMemoryRecordedAudioStore store; + + public Sink(InMemoryRecordedAudioStore store, string audioPath) + { + this.store = store; + AudioPath = audioPath; + } + + public string AudioPath { get; } + + public Task AppendAsync(AudioChunk chunk, CancellationToken cancellationToken) + { + store.AppendedChunkSizes.Add(chunk.Pcm.Length); + store.AppendObserved.TrySetResult(); + return Task.CompletedTask; + } + + public Task CompleteAsync(CancellationToken cancellationToken) + { + store.Completed = true; + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + + public Task DeleteAsync(CancellationToken cancellationToken) + { + store.Deleted = true; + return Task.CompletedTask; + } + } + } + + private sealed class TestSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory + { + private readonly IStreamingTranscriptionProvider provider; + private readonly Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize; + + public TestSpeechRecognitionPipelineFactory( + IStreamingTranscriptionProvider provider, + Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>>? finalize = null) + { + this.provider = provider; + this.finalize = finalize ?? ((_, _, _, _) => Task.FromResult>([])); + } + + public ISpeechRecognitionPipeline Create() + { + return new TestSpeechRecognitionPipeline(provider, finalize); + } + } + + private sealed class TestSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline + { + private readonly Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize; + + public TestSpeechRecognitionPipeline( + IStreamingTranscriptionProvider provider, + Func, SpeechRecognitionPipelineOptions, CancellationToken, Task>> finalize) + : base(provider) + { + this.finalize = finalize; + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return finalize(audioPath, liveSegments, options, cancellationToken); + } + } + + private sealed class CapturingTranscriptFinalizer + { + private readonly IReadOnlyList segments; + + public CapturingTranscriptFinalizer(IReadOnlyList segments) + { + this.segments = segments; + } + + public string? AudioPath { get; private set; } + + public IReadOnlyList LiveSegments { get; private set; } = []; + + public SpeechRecognitionPipelineOptions? Options { get; private set; } + + public Task> FinalizeAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + AudioPath = audioPath; + LiveSegments = liveSegments; + Options = options; + return Task.FromResult(segments); + } + } + + private sealed class ControlledAudioSource : IMeetingAudioSource + { + private readonly Channel chunks = Channel.CreateUnbounded(); + + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + return chunks.Reader.ReadAllAsync(cancellationToken); + } + + public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken) + { + return chunks.Writer.WriteAsync(chunk, cancellationToken); + } + } + + private sealed class CapturedChunkThenCancelAudioSource : IMeetingAudioSource + { + private readonly AudioChunk chunk; + private readonly TaskCompletionSource captured = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public CapturedChunkThenCancelAudioSource(AudioChunk chunk) + { + this.chunk = chunk; + } + + public async IAsyncEnumerable CaptureAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + yield return chunk; + captured.TrySetResult(); + + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + captured.TrySetResult(); + } + } + + public Task WaitUntilCapturedAsync() + { + return captured.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + } + + private sealed class EchoStreamingTranscriptionProvider : IStreamingTranscriptionProvider + { + public bool FirstChunkWasObservedBeforeSourceCompleted { get; private set; } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + FirstChunkWasObservedBeforeSourceCompleted = true; + yield return new TranscriptionSegment(TimeSpan.Zero, TimeSpan.Zero, "Unknown", $"chunk:{chunk.Pcm.Length}"); + } + } + } + + private sealed class FinalSegmentOnAudioCompletionProvider : IStreamingTranscriptionProvider + { + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var byteCount = 0; + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + byteCount += chunk.Pcm.Length; + } + + yield return new TranscriptionSegment(TimeSpan.Zero, TimeSpan.Zero, "Unknown", $"final:{byteCount}"); + } + } + + private sealed class BlockingBeforeTranscriptionProvider : IStreamingTranscriptionProvider + { + private readonly TaskCompletionSource waitingForBackend = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource backendReady = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilWaitingForBackendAsync() + { + return waitingForBackend.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public void MarkBackendReady() + { + backendReady.TrySetResult(); + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + waitingForBackend.TrySetResult(); + await backendReady.Task.WaitAsync(cancellationToken); + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + yield return new TranscriptionSegment(TimeSpan.Zero, TimeSpan.Zero, "Unknown", $"chunk:{chunk.Pcm.Length}"); + } + } + } +} + + diff --git a/MeetingAssistant.Tests/SampleWavFixtureTests.cs b/MeetingAssistant.Tests/SampleWavFixtureTests.cs new file mode 100644 index 0000000..ca293ce --- /dev/null +++ b/MeetingAssistant.Tests/SampleWavFixtureTests.cs @@ -0,0 +1,25 @@ +namespace MeetingAssistant.Tests; + +public sealed class SampleWavFixtureTests +{ + [Fact] + public async Task SampleWavFixtureIsSmallMonoPcmAudio() + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "sample-16khz-mono.wav"); + var bytes = await File.ReadAllBytesAsync(path); + + Assert.True(bytes.Length < 16_000); + Assert.Equal("RIFF", ReadAscii(bytes, 0, 4)); + Assert.Equal("WAVE", ReadAscii(bytes, 8, 4)); + Assert.Equal("fmt ", ReadAscii(bytes, 12, 4)); + Assert.Equal((short)1, BitConverter.ToInt16(bytes, 20)); + Assert.Equal((short)1, BitConverter.ToInt16(bytes, 22)); + Assert.Equal(16000, BitConverter.ToInt32(bytes, 24)); + Assert.Equal((short)16, BitConverter.ToInt16(bytes, 34)); + } + + private static string ReadAscii(byte[] bytes, int start, int length) + { + return System.Text.Encoding.ASCII.GetString(bytes, start, length); + } +} diff --git a/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs b/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs new file mode 100644 index 0000000..8f583d2 --- /dev/null +++ b/MeetingAssistant.Tests/SpeechRecognitionPipelineHostedServiceTests.cs @@ -0,0 +1,128 @@ +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Logging.Abstractions; + +namespace MeetingAssistant.Tests; + +public sealed class SpeechRecognitionPipelineHostedServiceTests +{ + [Fact] + public async Task HostedServiceInitializesAndWarmsConfiguredPipelineWithoutBlockingStartup() + { + var pipeline = new CapturingSpeechRecognitionPipeline { BlockReadinessUntilCancelled = true }; + var service = new SpeechRecognitionPipelineHostedService( + new CapturingSpeechRecognitionPipelineFactory(pipeline), + NullLogger.Instance); + + var startTask = service.StartAsync(CancellationToken.None); + + await startTask.WaitAsync(TimeSpan.FromSeconds(1)); + await pipeline.WaitForInitializeAsync(); + await pipeline.WaitForReadinessAsync(); + + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, pipeline.InitializeCount); + Assert.Equal(1, pipeline.ReadinessCount); + Assert.True(pipeline.ReadinessCancellationWasObserved); + Assert.True(pipeline.Disposed); + } + + private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory + { + private readonly ISpeechRecognitionPipeline pipeline; + + public CapturingSpeechRecognitionPipelineFactory(ISpeechRecognitionPipeline pipeline) + { + this.pipeline = pipeline; + } + + public ISpeechRecognitionPipeline Create() + { + return pipeline; + } + } + + private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline + { + private readonly TaskCompletionSource initialized = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource readinessChecked = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int InitializeCount { get; private set; } + + public int ReadinessCount { get; private set; } + + public bool BlockReadinessUntilCancelled { get; init; } + + public bool ReadinessCancellationWasObserved { get; private set; } + + public bool Disposed { get; private set; } + + public Task WaitForInitializeAsync() + { + return initialized.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public Task WaitForReadinessAsync() + { + return readinessChecked.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + public Task InitializeAsync(CancellationToken cancellationToken) + { + InitializeCount++; + initialized.TrySetResult(); + return Task.CompletedTask; + } + + public async Task WaitUntilReadyAsync(CancellationToken cancellationToken) + { + ReadinessCount++; + readinessChecked.TrySetResult(); + if (!BlockReadinessUntilCancelled) + { + return; + } + + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) + { + ReadinessCancellationWasObserved = true; + throw; + } + } + + public ValueTask WriteAsync(MeetingAssistant.Recording.AudioChunk chunk, CancellationToken cancellationToken) + { + return ValueTask.CompletedTask; + } + + public Task CompleteAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async IAsyncEnumerable ReadLiveTranscriptAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.CompletedTask; + yield break; + } + + public Task> ReadFinishedTranscriptAsync( + string audioPath, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult>([]); + } + + public ValueTask DisposeAsync() + { + Disposed = true; + return ValueTask.CompletedTask; + } + } +} diff --git a/MeetingAssistant/Hotkeys/GlobalHotkeyService.cs b/MeetingAssistant/Hotkeys/GlobalHotkeyService.cs new file mode 100644 index 0000000..aaddc60 --- /dev/null +++ b/MeetingAssistant/Hotkeys/GlobalHotkeyService.cs @@ -0,0 +1,150 @@ +using System.Runtime.InteropServices; +using System.Threading; +using MeetingAssistant.Recording; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Hotkeys; + +public sealed class GlobalHotkeyService : BackgroundService +{ + private const int HotkeyId = 0x4D41; + private const int WmHotkey = 0x0312; + private const int WmQuit = 0x0012; + private readonly MeetingAssistantOptions options; + private readonly MeetingRecordingCoordinator coordinator; + private readonly ILogger logger; + private TaskCompletionSource? messageLoopCompletion; + private uint messageThreadId; + + public GlobalHotkeyService( + IOptions options, + MeetingRecordingCoordinator coordinator, + ILogger logger) + { + this.options = options.Value; + this.coordinator = coordinator; + this.logger = logger; + } + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!OperatingSystem.IsWindows()) + { + logger.LogInformation("Global hotkey registration is only supported on Windows"); + return Task.CompletedTask; + } + + messageLoopCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var messageThread = new Thread(() => + { + try + { + RunMessageLoop(stoppingToken); + messageLoopCompletion.TrySetResult(); + } + catch (Exception exception) + { + messageLoopCompletion.TrySetException(exception); + } + }) + { + IsBackground = true, + Name = "Meeting Assistant Hotkey Loop" + }; + + messageThread.Start(); + return messageLoopCompletion.Task; + } + + private void RunMessageLoop(CancellationToken stoppingToken) + { + var hotkey = HotkeyDefinition.Parse(options.Hotkey.Toggle); + messageThreadId = GetCurrentThreadId(); + using var stoppingRegistration = stoppingToken.Register(() => PostQuitToMessageThread()); + + if (!RegisterHotKey(IntPtr.Zero, HotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey)) + { + logger.LogWarning("Could not register global hotkey {Hotkey}", options.Hotkey.Toggle); + return; + } + + logger.LogInformation("Registered global hotkey {Hotkey}", options.Hotkey.Toggle); + + try + { + while (!stoppingToken.IsCancellationRequested && GetMessage(out var message, IntPtr.Zero, 0, 0) > 0) + { + if (message.Message == WmHotkey) + { + _ = Task.Run(ToggleRecordingAsync, CancellationToken.None); + } + } + } + finally + { + UnregisterHotKey(IntPtr.Zero, HotkeyId); + messageThreadId = 0; + } + } + + public override Task StopAsync(CancellationToken cancellationToken) + { + PostQuitToMessageThread(); + return base.StopAsync(cancellationToken); + } + + private async Task ToggleRecordingAsync() + { + try + { + var status = await coordinator.ToggleAsync(CancellationToken.None); + logger.LogInformation("Hotkey toggled recording. IsRecording={IsRecording}", status.IsRecording); + } + catch (Exception exception) + { + logger.LogError(exception, "Hotkey toggle failed"); + } + } + + private void PostQuitToMessageThread() + { + var threadId = messageThreadId; + if (threadId != 0) + { + PostThreadMessage(threadId, WmQuit, UIntPtr.Zero, IntPtr.Zero); + } + } + + [DllImport("user32.dll", SetLastError = true)] + private static extern bool RegisterHotKey(IntPtr hWnd, int id, uint fsModifiers, uint vk); + + [DllImport("user32.dll", SetLastError = true)] + private static extern bool UnregisterHotKey(IntPtr hWnd, int id); + + [DllImport("user32.dll")] + private static extern int GetMessage(out NativeMessage lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax); + + [DllImport("kernel32.dll")] + private static extern uint GetCurrentThreadId(); + + [DllImport("user32.dll", SetLastError = true)] + private static extern bool PostThreadMessage(uint idThread, int msg, UIntPtr wParam, IntPtr lParam); + + [StructLayout(LayoutKind.Sequential)] + private struct NativeMessage + { + public IntPtr Hwnd; + public int Message; + public UIntPtr WParam; + public IntPtr LParam; + public uint Time; + public NativePoint Point; + } + + [StructLayout(LayoutKind.Sequential)] + private struct NativePoint + { + public int X; + public int Y; + } +} diff --git a/MeetingAssistant/Hotkeys/HotkeyDefinition.cs b/MeetingAssistant/Hotkeys/HotkeyDefinition.cs new file mode 100644 index 0000000..2ca326e --- /dev/null +++ b/MeetingAssistant/Hotkeys/HotkeyDefinition.cs @@ -0,0 +1,72 @@ +namespace MeetingAssistant.Hotkeys; + +[Flags] +public enum HotkeyModifiers : uint +{ + None = 0, + Alt = 0x0001, + Control = 0x0002, + Shift = 0x0004, + Windows = 0x0008 +} + +public sealed record HotkeyDefinition(HotkeyModifiers Modifiers, uint VirtualKey) +{ + public static HotkeyDefinition Parse(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Hotkey cannot be empty.", nameof(value)); + } + + var modifiers = HotkeyModifiers.None; + uint? virtualKey = null; + + foreach (var rawPart in value.Split('+', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var part = rawPart.ToUpperInvariant(); + modifiers |= part switch + { + "CTRL" or "CONTROL" => HotkeyModifiers.Control, + "ALT" => HotkeyModifiers.Alt, + "SHIFT" => HotkeyModifiers.Shift, + "WIN" or "WINDOWS" => HotkeyModifiers.Windows, + _ => HotkeyModifiers.None + }; + + if (modifiers.HasFlagForToken(part)) + { + continue; + } + + virtualKey = ParseVirtualKey(part); + } + + return virtualKey is null + ? throw new FormatException($"Hotkey '{value}' does not contain a key.") + : new HotkeyDefinition(modifiers, virtualKey.Value); + } + + private static uint ParseVirtualKey(string part) + { + if (part.Length == 1 && char.IsLetterOrDigit(part[0])) + { + return part[0]; + } + + if (part.Length is 2 or 3 && part[0] == 'F' && int.TryParse(part[1..], out var functionKey) && functionKey is >= 1 and <= 24) + { + return (uint)(0x70 + functionKey - 1); + } + + throw new FormatException($"Unsupported hotkey key '{part}'."); + } +} + +file static class HotkeyModifierExtensions +{ + public static bool HasFlagForToken(this HotkeyModifiers _, string token) + { + return token is "CTRL" or "CONTROL" or "ALT" or "SHIFT" or "WIN" or "WINDOWS"; + } +} diff --git a/MeetingAssistant/MeetingAssistant.csproj b/MeetingAssistant/MeetingAssistant.csproj index a3a34b6..82abf5d 100644 --- a/MeetingAssistant/MeetingAssistant.csproj +++ b/MeetingAssistant/MeetingAssistant.csproj @@ -1,9 +1,28 @@ - net10.0 + net10.0;net10.0-windows enable enable + + + + + + + + + + + + + + + + + + + diff --git a/MeetingAssistant/MeetingAssistantOptions.cs b/MeetingAssistant/MeetingAssistantOptions.cs new file mode 100644 index 0000000..b43c5db --- /dev/null +++ b/MeetingAssistant/MeetingAssistantOptions.cs @@ -0,0 +1,236 @@ +namespace MeetingAssistant; + +public sealed class MeetingAssistantOptions +{ + public HotkeyOptions Hotkey { get; set; } = new(); + + public VaultOptions Vault { get; set; } = new(); + + public RecordingOptions Recording { get; set; } = new(); + + public WhisperLocalOptions WhisperLocal { get; set; } = new(); + + public AzureSpeechOptions AzureSpeech { get; set; } = new(); + + public FunAsrOptions FunAsr { get; set; } = new(); + + public AgentOptions Agent { get; set; } = new(); + + public ApiOptions Api { get; set; } = new(); +} + +public sealed class HotkeyOptions +{ + public string Toggle { get; set; } = "Ctrl+Alt+M"; +} + +public sealed class VaultOptions +{ + public string TranscriptsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Transcripts"; + + public string MeetingNotesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Notes"; + + public string AssistantContextFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Assistant Context"; + + public string SummariesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Summaries"; + + public string ProjectsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Projects"; + + public string DictationWordsPath { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\dictation-words.md"; +} + +public sealed class RecordingOptions +{ + public string TranscriptionProvider { get; set; } = "funasr"; + + public int SampleRate { get; set; } = 16000; + + public int Channels { get; set; } = 1; + + public TimeSpan StopProcessingTimeout { get; set; } = TimeSpan.FromMinutes(10); + + public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings"; +} + +public sealed class WhisperLocalOptions +{ + public string ModelPath { get; set; } = @"models\ggml-base.bin"; + + public string Language { get; set; } = "auto"; + + public double WindowSeconds { get; set; } = 15; + + public PyannoteDiarizationOptions Diarization { get; set; } = new(); +} + +public sealed class PyannoteDiarizationOptions +{ + public bool Enabled { get; set; } = true; + + public string DockerCommand { get; set; } = "docker"; + + public string BaseImage { get; set; } = "python:3.11-slim"; + + public string Image { get; set; } = "meeting-assistant-pyannote:local"; + + public bool BuildImage { get; set; } = true; + + public string ModelsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Pyannote\models"; + + public string Model { get; set; } = "pyannote/speaker-diarization-3.1"; + + public PyannoteAnnotationSource AnnotationSource { get; set; } = PyannoteAnnotationSource.SpeakerDiarization; + + public PyannoteAlignmentMode AlignmentMode { get; set; } = PyannoteAlignmentMode.PyannoteTurns; + + public string TokenEnv { get; set; } = "HF_TOKEN"; + + public string? Token { get; set; } + + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30); +} + +public sealed class AzureSpeechOptions +{ + public string Endpoint { get; set; } = "https://germanywestcentral.api.cognitive.microsoft.com/"; + + public string Region { get; set; } = "germanywestcentral"; + + public string Language { get; set; } = "en-US"; + + public string? Key { get; set; } + + public string KeyEnv { get; set; } = "AZURE_SPEECH_KEY"; + + public TimeSpan RecognitionStopTimeout { get; set; } = TimeSpan.FromSeconds(10); + + public bool DiarizeIntermediateResults { get; set; } = true; +} + +public enum PyannoteAnnotationSource +{ + SpeakerDiarization, + ExclusiveSpeakerDiarization +} + +public enum PyannoteAlignmentMode +{ + BestOverlap, + PyannoteTurns +} + +public sealed class FunAsrOptions +{ + public string Endpoint { get; set; } = "ws://127.0.0.1:10095"; + + public string Mode { get; set; } = "2pass"; + + public int[] ChunkSize { get; set; } = [5, 10, 5]; + + public int ChunkInterval { get; set; } = 10; + + public int EncoderChunkLookBack { get; set; } = 4; + + public int DecoderChunkLookBack { get; set; } = 1; + + public bool Itn { get; set; } = true; + + public bool EmitOnlinePartials { get; set; } + + public string WavName { get; set; } = "meeting"; + + public TimeSpan FinalResultTimeout { get; set; } = TimeSpan.FromSeconds(10); + + public FunAsrBackendOptions Backend { get; set; } = new(); + + public FunAsrDiarizationOptions Diarization { get; set; } = new(); +} + +public sealed class FunAsrBackendOptions +{ + public bool Enabled { get; set; } = true; + + public string DockerCommand { get; set; } = "docker"; + + public string Image { get; set; } = "registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.13"; + + public string ContainerName { get; set; } = "meeting-assistant-funasr"; + + public string ModelsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\FunASR\models"; + + public int ContainerPort { get; set; } = 10095; + + public bool Privileged { get; set; } = true; + + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15); + + public TimeSpan StartupTimeout { get; set; } = TimeSpan.FromMinutes(10); + + public bool StopOnDispose { get; set; } = true; + + public string ServerCommand { get; set; } = "cd /workspace/FunASR/runtime && bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --itn-dir thuduj12/fst_itn_zh --certfile 0 --hotword /workspace/models/hotwords.txt && tail -f /dev/null"; +} + +public sealed class FunAsrDiarizationOptions +{ + public bool Enabled { get; set; } = true; + + public string AsrModel { get; set; } = "paraformer-zh"; + + public string VadModel { get; set; } = "fsmn-vad"; + + public string PunctuationModel { get; set; } = "ct-punc"; + + public string SpeakerModel { get; set; } = "cam++"; + + public double BatchSizeSeconds { get; set; } = 300; + + public double BatchSizeThresholdSeconds { get; set; } = 60; + + public int? PresetSpeakerCount { get; set; } + + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30); +} + +public sealed class AgentOptions +{ + public string Endpoint { get; set; } = "https://litellm.schweigert.cloud"; + + public string? Key { get; set; } + + public string KeyEnv { get; set; } = "LITELLM_API_KEY"; + + public string Model { get; set; } = "copilot-gpt-5.5"; + + public bool EnableThinking { get; set; } = true; + + public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium; + + public int ReconnectionAttempts { get; set; } = 2; + + public TimeSpan ReconnectionDelay { get; set; } = TimeSpan.FromSeconds(2); + + public int ContextWindowTokens { get; set; } = 128000; + + public int MaxOutputTokens { get; set; } = 8192; + + public bool EnableCompaction { get; set; } = true; + + public double CompactionRemainingRatio { get; set; } = 0.10; + + public string ResponsesCompactPath { get; set; } = "responses/compact"; +} + +public sealed class ApiOptions +{ + public string PublicBaseUrl { get; set; } = "http://localhost:5090"; +} + +public enum ReasoningEffortOption +{ + None, + Low, + Medium, + High, + ExtraHigh +} diff --git a/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs b/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs new file mode 100644 index 0000000..a34dc3f --- /dev/null +++ b/MeetingAssistant/MeetingNotes/IMeetingArtifactStore.cs @@ -0,0 +1,24 @@ +namespace MeetingAssistant.MeetingNotes; + +public interface IMeetingArtifactStore +{ + Task CreateAssistantContextAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken); + + Task UpdateAssistantContextStateAsync( + MeetingSessionArtifacts artifacts, + AssistantContextState state, + CancellationToken cancellationToken); +} + +public enum AssistantContextState +{ + Transcribing, + SpeakerRecognition, + Summarizing, + Finished, + Error +} diff --git a/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs b/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs new file mode 100644 index 0000000..a7890f3 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/IMeetingMetadataProvider.cs @@ -0,0 +1,24 @@ +namespace MeetingAssistant.MeetingNotes; + +public interface IMeetingMetadataProvider +{ + Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken); +} + +public sealed record MeetingMetadata( + string Title, + IReadOnlyList Attendees, + string Agenda, + DateTimeOffset? ScheduledEnd = null); + +public sealed class NoopMeetingMetadataProvider : IMeetingMetadataProvider +{ + public Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + return Task.FromResult(null); + } +} diff --git a/MeetingAssistant/MeetingNotes/IMeetingNoteOpener.cs b/MeetingAssistant/MeetingNotes/IMeetingNoteOpener.cs new file mode 100644 index 0000000..4ff72c9 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/IMeetingNoteOpener.cs @@ -0,0 +1,6 @@ +namespace MeetingAssistant.MeetingNotes; + +public interface IMeetingNoteOpener +{ + Task OpenAsync(string notePath, CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/MeetingNotes/IMeetingNoteStore.cs b/MeetingAssistant/MeetingNotes/IMeetingNoteStore.cs new file mode 100644 index 0000000..f2a9a70 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/IMeetingNoteStore.cs @@ -0,0 +1,8 @@ +namespace MeetingAssistant.MeetingNotes; + +public interface IMeetingNoteStore +{ + Task SaveAsync(MeetingNote note, CancellationToken cancellationToken); + + Task ReadAsync(string path, CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/MeetingNotes/MarkdownDocument.cs b/MeetingAssistant/MeetingNotes/MarkdownDocument.cs new file mode 100644 index 0000000..662afe0 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MarkdownDocument.cs @@ -0,0 +1,76 @@ +namespace MeetingAssistant.MeetingNotes; + +internal readonly record struct MarkdownDocument( + bool HasFrontmatter, + string Frontmatter, + string Body); + +internal static class MarkdownDocumentParser +{ + public static MarkdownDocument SplitOptional(string content) + { + using var reader = new StringReader(content); + if (reader.ReadLine() != "---") + { + return new MarkdownDocument(false, "", content); + } + + var yaml = new List(); + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (line == "---") + { + return new MarkdownDocument( + true, + string.Join(Environment.NewLine, yaml), + RemoveBodySeparator(reader.ReadToEnd())); + } + + yaml.Add(line); + } + + return new MarkdownDocument(false, "", content); + } + + public static MarkdownDocument SplitRequired( + string content, + string missingFrontmatterMessage, + string unclosedFrontmatterMessage) + { + using var reader = new StringReader(content); + if (reader.ReadLine() != "---") + { + throw new InvalidDataException(missingFrontmatterMessage); + } + + var yaml = new List(); + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (line == "---") + { + return new MarkdownDocument( + true, + string.Join(Environment.NewLine, yaml), + RemoveBodySeparator(reader.ReadToEnd())); + } + + yaml.Add(line); + } + + throw new InvalidDataException(unclosedFrontmatterMessage); + } + + private static string RemoveBodySeparator(string body) + { + if (body.StartsWith("\r\n", StringComparison.Ordinal)) + { + return body[2..]; + } + + return body.StartsWith('\n') + ? body[1..] + : body; + } +} diff --git a/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs b/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs new file mode 100644 index 0000000..f4a5c15 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MarkdownMeetingArtifactStore.cs @@ -0,0 +1,123 @@ +namespace MeetingAssistant.MeetingNotes; + +using YamlDotNet.Serialization; + +public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore +{ + private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + + private readonly ILogger logger; + + public MarkdownMeetingArtifactStore(ILogger logger) + { + this.logger = logger; + } + + public async Task CreateAssistantContextAsync( + MeetingSessionArtifacts artifacts, + string agenda, + DateTimeOffset? scheduledEnd, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + + var content = Render( + artifacts, + AssistantContextState.Transcribing, + agenda, + scheduledEnd, + "# Assistant Context" + Environment.NewLine + Environment.NewLine + "## Live Context" + Environment.NewLine); + + await File.WriteAllTextAsync(artifacts.AssistantContextPath, content, cancellationToken); + logger.LogInformation("Created assistant context note {AssistantContextPath}", artifacts.AssistantContextPath); + } + + public async Task UpdateAssistantContextStateAsync( + MeetingSessionArtifacts artifacts, + AssistantContextState state, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + var existing = File.Exists(artifacts.AssistantContextPath) + ? MarkdownDocumentParser.SplitOptional(await File.ReadAllTextAsync(artifacts.AssistantContextPath, cancellationToken)) + : new MarkdownDocument(false, "", "## Live Context" + Environment.NewLine); + var body = existing.Body; + var existingFrontmatter = ReadFrontmatter(existing.Frontmatter); + var agenda = existingFrontmatter.Agenda ?? ""; + var scheduledEnd = ParseDateTime(existingFrontmatter.ScheduledEnd); + + await File.WriteAllTextAsync( + artifacts.AssistantContextPath, + Render(artifacts, state, agenda, scheduledEnd, body), + cancellationToken); + logger.LogInformation( + "Updated assistant context note {AssistantContextPath} state to {State}", + artifacts.AssistantContextPath, + ToYamlState(state)); + } + + private static string Render( + MeetingSessionArtifacts artifacts, + AssistantContextState state, + string? agenda, + DateTimeOffset? scheduledEnd, + string body) + { + var frontmatter = new MeetingArtifactFrontmatter + { + Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, artifacts.AssistantContextPath, "Meeting Note"), + Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, artifacts.AssistantContextPath, "Transcript"), + Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, artifacts.AssistantContextPath, "Summary"), + State = ToYamlState(state), + Agenda = agenda ?? "", + ScheduledEnd = scheduledEnd + }; + + return MeetingArtifactFrontmatterRenderer.Render( + frontmatter, + body); + } + + private static AssistantContextFrontmatterYaml ReadFrontmatter(string frontmatter) + { + if (string.IsNullOrWhiteSpace(frontmatter)) + { + return new AssistantContextFrontmatterYaml(); + } + + return YamlDeserializer.Deserialize(frontmatter) + ?? new AssistantContextFrontmatterYaml(); + } + + private static DateTimeOffset? ParseDateTime(string? value) + { + return DateTimeOffset.TryParse(value, out var parsed) + ? parsed + : null; + } + + private sealed class AssistantContextFrontmatterYaml + { + [YamlMember(Alias = "agenda")] + public string? Agenda { get; set; } + + [YamlMember(Alias = "scheduled_end")] + public string? ScheduledEnd { get; set; } + } + + private static string ToYamlState(AssistantContextState state) + { + return state switch + { + AssistantContextState.Transcribing => "transcribing", + AssistantContextState.SpeakerRecognition => "speaker recognition", + AssistantContextState.Summarizing => "summarizing", + AssistantContextState.Finished => "finished", + AssistantContextState.Error => "error", + _ => "error" + }; + } + +} diff --git a/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs b/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs new file mode 100644 index 0000000..5f3e57f --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MarkdownMeetingNoteStore.cs @@ -0,0 +1,176 @@ +using Microsoft.Extensions.Options; +using YamlDotNet.Core; +using YamlDotNet.Serialization; + +namespace MeetingAssistant.MeetingNotes; + +public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore +{ + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + private readonly IDeserializer yamlDeserializer; + + public MarkdownMeetingNoteStore( + IOptions options, + ILogger logger) + { + this.options = options.Value; + this.logger = logger; + yamlDeserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + } + + public async Task SaveAsync(MeetingNote note, CancellationToken cancellationToken) + { + var folder = VaultPath.Resolve(options.Vault.MeetingNotesFolder); + Directory.CreateDirectory(folder); + + var path = string.IsNullOrWhiteSpace(note.Path) + ? Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Slugify(note.Frontmatter.Title)}.md") + : note.Path; + var frontmatter = PrepareFrontmatter(note.Frontmatter, path); + var content = Render(frontmatter, note.UserNotes); + + await File.WriteAllTextAsync(path, content, cancellationToken); + logger.LogInformation("Saved meeting note {MeetingNotePath}", path); + + return new MeetingNote(path, frontmatter, note.UserNotes); + } + + public async Task ReadAsync(string path, CancellationToken cancellationToken) + { + var content = await File.ReadAllTextAsync(path, cancellationToken); + var document = MarkdownDocumentParser.SplitRequired( + content, + "Meeting note does not start with YAML frontmatter.", + "Meeting note frontmatter is not closed."); + var yaml = yamlDeserializer.Deserialize(document.Frontmatter) ?? new MeetingNoteYaml(); + var frontmatter = new MeetingNoteFrontmatter + { + Title = yaml.Title ?? "", + StartTime = ParseDateTime(yaml.StartTime), + EndTime = ParseDateTime(yaml.EndTime), + Attendees = yaml.Attendees ?? [], + Projects = yaml.Projects ?? [], + Transcript = yaml.Transcript ?? "", + AssistantContext = yaml.AssistantContext ?? "", + Summary = yaml.Summary ?? "" + }; + + return new MeetingNote(path, frontmatter, document.Body); + } + + private MeetingNoteFrontmatter PrepareFrontmatter(MeetingNoteFrontmatter frontmatter, string notePath) + { + return new MeetingNoteFrontmatter + { + Title = frontmatter.Title, + StartTime = frontmatter.StartTime, + EndTime = frontmatter.EndTime, + Attendees = frontmatter.Attendees, + Projects = frontmatter.Projects, + Transcript = ObsidianLink.ToWikiLink(frontmatter.Transcript, notePath, "Transcript"), + AssistantContext = ObsidianLink.ToWikiLink(frontmatter.AssistantContext, notePath, "Assistant Context"), + Summary = ObsidianLink.ToWikiLink(frontmatter.Summary, notePath, "Summary") + }; + } + + private static string Render(MeetingNoteFrontmatter frontmatter, string userNotes) + { + var lines = new List + { + "---", + $"title: {EscapeScalar(frontmatter.Title)}", + $"start_time: {EscapeNullableDateTime(frontmatter.StartTime)}", + $"end_time: {EscapeNullableDateTime(frontmatter.EndTime)}", + "attendees:" + }; + + lines.AddRange(frontmatter.Attendees.Select(attendee => $"- {EscapeListItem(attendee)}")); + lines.Add("projects:"); + lines.AddRange(frontmatter.Projects.Select(project => $"- {EscapeListItem(project)}")); + lines.Add($"transcript: {EscapeQuoted(frontmatter.Transcript)}"); + lines.Add($"assistant_context: {EscapeQuoted(frontmatter.AssistantContext)}"); + lines.Add($"summary: {EscapeQuoted(frontmatter.Summary)}"); + lines.Add("---"); + lines.Add(""); + lines.Add(userNotes); + + return string.Join(Environment.NewLine, lines); + } + + private static string Slugify(string value) + { + var slug = new string(value + .ToLowerInvariant() + .Select(character => char.IsLetterOrDigit(character) ? character : '-') + .ToArray()); + slug = string.Join("-", slug.Split('-', StringSplitOptions.RemoveEmptyEntries)); + return string.IsNullOrWhiteSpace(slug) ? "meeting" : slug; + } + + private static string EscapeScalar(string value) + { + return string.IsNullOrWhiteSpace(value) ? "\"\"" : EscapeListItem(value); + } + + private static string EscapeQuoted(string value) + { + return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\""; + } + + private static string EscapeNullableDateTime(DateTimeOffset? value) + { + return value is null ? "\"\"" : EscapeQuoted(value.Value.ToString("O")); + } + + private static DateTimeOffset? ParseDateTime(string? value) + { + return DateTimeOffset.TryParse(value, out var parsed) + ? parsed + : null; + } + + private static string EscapeListItem(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "\"\""; + } + + if (value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal)) + { + return EscapeQuoted(value); + } + + return value; + } + + private sealed class MeetingNoteYaml + { + [YamlMember(Alias = "title")] + public string? Title { get; set; } + + [YamlMember(Alias = "start_time")] + public string? StartTime { get; set; } + + [YamlMember(Alias = "end_time")] + public string? EndTime { get; set; } + + [YamlMember(Alias = "attendees")] + public List? Attendees { get; set; } + + [YamlMember(Alias = "projects")] + public List? Projects { get; set; } + + [YamlMember(Alias = "transcript")] + public string? Transcript { get; set; } + + [YamlMember(Alias = "assistant_context")] + public string? AssistantContext { get; set; } + + [YamlMember(Alias = "summary")] + public string? Summary { get; set; } + } +} diff --git a/MeetingAssistant/MeetingNotes/MeetingArtifactFrontmatter.cs b/MeetingAssistant/MeetingNotes/MeetingArtifactFrontmatter.cs new file mode 100644 index 0000000..215051c --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingArtifactFrontmatter.cs @@ -0,0 +1,148 @@ +using System.Text; + +namespace MeetingAssistant.MeetingNotes; + +public sealed class MeetingArtifactFrontmatter +{ + public string Title { get; set; } = ""; + + public DateTimeOffset? StartTime { get; set; } + + public DateTimeOffset? EndTime { get; set; } + + public string Meeting { get; set; } = ""; + + public string Transcript { get; set; } = ""; + + public string AssistantContext { get; set; } = ""; + + public string Summary { get; set; } = ""; + + public string? State { get; set; } + + public string? Agenda { get; set; } + + public DateTimeOffset? ScheduledEnd { get; set; } +} + +public static class MeetingArtifactFrontmatterRenderer +{ + public static string Render( + MeetingArtifactFrontmatter frontmatter, + string body) + { + var builder = new StringBuilder(); + builder.AppendLine("---"); + AppendScalar(builder, "title", frontmatter.Title); + AppendDateTime(builder, "start_time", frontmatter.StartTime); + AppendDateTime(builder, "end_time", frontmatter.EndTime); + AppendQuoted(builder, "meeting", frontmatter.Meeting); + AppendQuoted(builder, "transcript", frontmatter.Transcript); + AppendQuoted(builder, "assistant_context", frontmatter.AssistantContext); + AppendQuoted(builder, "summary", frontmatter.Summary); + if (!string.IsNullOrWhiteSpace(frontmatter.State)) + { + AppendScalar(builder, "state", frontmatter.State); + } + + if (frontmatter.Agenda is not null) + { + AppendBlockScalar(builder, "agenda", frontmatter.Agenda); + } + + if (frontmatter.ScheduledEnd is not null) + { + AppendDateTime(builder, "scheduled_end", frontmatter.ScheduledEnd); + } + + builder.AppendLine("---"); + builder.AppendLine(); + builder.Append(body.TrimStart('\r', '\n')); + return builder.ToString(); + } + + public static (string Frontmatter, string Body) Split(string content) + { + var document = MarkdownDocumentParser.SplitOptional(content); + return (document.Frontmatter, document.Body); + } + + public static MeetingArtifactFrontmatter Create( + MeetingSessionArtifacts artifacts, + MeetingNote meetingNote, + string title, + string sourceNotePath, + string? state = null) + { + return new MeetingArtifactFrontmatter + { + Title = title, + StartTime = meetingNote.Frontmatter.StartTime, + EndTime = meetingNote.Frontmatter.EndTime, + Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, sourceNotePath, "Meeting Note"), + Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, sourceNotePath, "Transcript"), + AssistantContext = ObsidianLink.ToWikiLink(artifacts.AssistantContextPath, sourceNotePath, "Assistant Context"), + Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, sourceNotePath, "Summary"), + State = state + }; + } + + public static string DefaultTitle(MeetingNote meetingNote, string fallback) + { + return string.IsNullOrWhiteSpace(meetingNote.Frontmatter.Title) + ? fallback + : meetingNote.Frontmatter.Title; + } + + private static void AppendScalar(StringBuilder builder, string key, string? value) + { + builder.Append(key); + builder.Append(": "); + builder.AppendLine(string.IsNullOrWhiteSpace(value) ? "\"\"" : EscapeListItem(value)); + } + + private static void AppendQuoted(StringBuilder builder, string key, string value) + { + builder.Append(key); + builder.Append(": "); + builder.AppendLine(EscapeQuoted(value)); + } + + private static void AppendDateTime(StringBuilder builder, string key, DateTimeOffset? value) + { + builder.Append(key); + builder.Append(": "); + builder.AppendLine(value is null ? "\"\"" : EscapeQuoted(value.Value.ToString("O"))); + } + + private static void AppendBlockScalar(StringBuilder builder, string key, string value) + { + if (string.IsNullOrEmpty(value)) + { + builder.Append(key); + builder.AppendLine(": \"\""); + return; + } + + builder.Append(key); + builder.AppendLine(": |-"); + foreach (var line in value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n')) + { + builder.Append(" "); + builder.AppendLine(line); + } + } + + private static string EscapeQuoted(string value) + { + return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\""; + } + + private static string EscapeListItem(string value) + { + return value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal) + ? EscapeQuoted(value) + : value; + } + +} diff --git a/MeetingAssistant/MeetingNotes/MeetingNote.cs b/MeetingAssistant/MeetingNotes/MeetingNote.cs new file mode 100644 index 0000000..b6c4f87 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingNote.cs @@ -0,0 +1,25 @@ +namespace MeetingAssistant.MeetingNotes; + +public sealed record MeetingNote( + string Path, + MeetingNoteFrontmatter Frontmatter, + string UserNotes); + +public sealed class MeetingNoteFrontmatter +{ + public string Title { get; set; } = ""; + + public DateTimeOffset? StartTime { get; set; } + + public DateTimeOffset? EndTime { get; set; } + + public List Attendees { get; set; } = []; + + public List Projects { get; set; } = []; + + public string Transcript { get; set; } = ""; + + public string AssistantContext { get; set; } = ""; + + public string Summary { get; set; } = ""; +} diff --git a/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs b/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs new file mode 100644 index 0000000..9a8379b --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingNoteActionLinks.cs @@ -0,0 +1,17 @@ +namespace MeetingAssistant.MeetingNotes; + +public static class MeetingNoteActionLinks +{ + public static string CreateSummaryRetryLink( + string publicBaseUrl, + string summaryPath, + string label = "Retry summary generation") + { + var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl) + ? "http://localhost:5090" + : publicBaseUrl.TrimEnd('/'); + var summaryFileName = Path.GetFileName(summaryPath); + var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}"; + return $"[{label}]({url})"; + } +} diff --git a/MeetingAssistant/MeetingNotes/MeetingNoteTemplate.cs b/MeetingAssistant/MeetingNotes/MeetingNoteTemplate.cs new file mode 100644 index 0000000..19096ae --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingNoteTemplate.cs @@ -0,0 +1,31 @@ +namespace MeetingAssistant.MeetingNotes; + +public static class MeetingNoteTemplate +{ + public static MeetingNote Create( + string title, + DateTimeOffset? startTime = null, + DateTimeOffset? endTime = null, + IEnumerable? attendees = null, + IEnumerable? projects = null, + string transcriptPath = "", + string assistantContextPath = "", + string summaryPath = "", + string userNotes = "") + { + var notePath = ""; + var frontmatter = new MeetingNoteFrontmatter + { + Title = title, + StartTime = startTime, + EndTime = endTime, + Attendees = attendees?.Where(value => !string.IsNullOrWhiteSpace(value)).ToList() ?? [], + Projects = projects?.Where(value => !string.IsNullOrWhiteSpace(value)).ToList() ?? [], + Transcript = transcriptPath, + AssistantContext = assistantContextPath, + Summary = summaryPath + }; + + return new MeetingNote(notePath, frontmatter, userNotes); + } +} diff --git a/MeetingAssistant/MeetingNotes/MeetingSessionArtifacts.cs b/MeetingAssistant/MeetingNotes/MeetingSessionArtifacts.cs new file mode 100644 index 0000000..d681871 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/MeetingSessionArtifacts.cs @@ -0,0 +1,7 @@ +namespace MeetingAssistant.MeetingNotes; + +public sealed record MeetingSessionArtifacts( + string MeetingNotePath, + string TranscriptPath, + string AssistantContextPath, + string SummaryPath); diff --git a/MeetingAssistant/MeetingNotes/ObsidianLink.cs b/MeetingAssistant/MeetingNotes/ObsidianLink.cs new file mode 100644 index 0000000..147dbda --- /dev/null +++ b/MeetingAssistant/MeetingNotes/ObsidianLink.cs @@ -0,0 +1,22 @@ +namespace MeetingAssistant.MeetingNotes; + +public static class ObsidianLink +{ + public static string ToWikiLink(string pathOrLink, string sourceNotePath, string label) + { + if (string.IsNullOrWhiteSpace(pathOrLink) || pathOrLink.StartsWith("[[", StringComparison.Ordinal)) + { + return pathOrLink; + } + + var sourceFolder = Path.GetDirectoryName(sourceNotePath) ?? ""; + var relativePath = Path.GetRelativePath(sourceFolder, pathOrLink); + var withoutExtension = Path.Combine( + Path.GetDirectoryName(relativePath) ?? "", + Path.GetFileNameWithoutExtension(relativePath)) + .Replace(Path.DirectorySeparatorChar, '/') + .Replace(Path.AltDirectorySeparatorChar, '/'); + + return $"[[{withoutExtension}|{label}]]"; + } +} diff --git a/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs b/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs new file mode 100644 index 0000000..f5ebccb --- /dev/null +++ b/MeetingAssistant/MeetingNotes/ObsidianMeetingNoteOpener.cs @@ -0,0 +1,26 @@ +using System.Diagnostics; + +namespace MeetingAssistant.MeetingNotes; + +public sealed class ObsidianMeetingNoteOpener : IMeetingNoteOpener +{ + private readonly ILogger logger; + + public ObsidianMeetingNoteOpener(ILogger logger) + { + this.logger = logger; + } + + public Task OpenAsync(string notePath, CancellationToken cancellationToken) + { + var uri = $"obsidian://open?path={Uri.EscapeDataString(Path.GetFullPath(notePath))}"; + Process.Start(new ProcessStartInfo + { + FileName = uri, + UseShellExecute = true + }); + logger.LogInformation("Opened meeting note in Obsidian via {ObsidianUri}", uri); + + return Task.CompletedTask; + } +} diff --git a/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs new file mode 100644 index 0000000..6230681 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/OutlookClassicMeetingMetadataProvider.Windows.cs @@ -0,0 +1,289 @@ +using System.Runtime.InteropServices; + +namespace MeetingAssistant.MeetingNotes; + +public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProvider +{ + private const int OutlookCalendarFolder = 9; + private readonly ILogger logger; + + public OutlookClassicMeetingMetadataProvider(ILogger logger) + { + this.logger = logger; + } + + public Task GetCurrentMeetingAsync( + DateTimeOffset startedAt, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + return Task.FromResult(GetCurrentMeeting(startedAt)); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger.LogDebug(exception, "Outlook Classic meeting metadata was not available"); + return Task.FromResult(null); + } + } + + private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt) + { + object? application = null; + object? session = null; + object? calendar = null; + object? items = null; + object? restrictedItems = null; + + try + { + var applicationType = Type.GetTypeFromProgID("Outlook.Application"); + if (applicationType is null) + { + return null; + } + + application = Activator.CreateInstance(applicationType); + if (application is null) + { + return null; + } + + session = GetProperty(application, "Session"); + calendar = Invoke(session!, "GetDefaultFolder", OutlookCalendarFolder); + items = GetProperty(calendar!, "Items"); + SetProperty(items!, "IncludeRecurrences", true); + Invoke(items!, "Sort", "[Start]"); + + var startedLocal = startedAt.LocalDateTime; + var windowStart = startedLocal.AddMinutes(-1); + var windowEnd = startedLocal.AddMinutes(5); + var filter = + $"[Start] <= '{windowEnd:g}' AND [End] >= '{windowStart:g}'"; + restrictedItems = Invoke(items!, "Restrict", filter); + + var candidates = new List(); + var count = Convert.ToInt32(GetProperty(restrictedItems!, "Count")); + for (var index = 1; index <= count; index++) + { + var appointment = Invoke(restrictedItems!, "Item", index); + if (appointment is not null && IsTeamsAppointment(appointment)) + { + candidates.Add(new OutlookAppointmentCandidate( + appointment, + Convert.ToDateTime(GetProperty(appointment, "Start")), + Convert.ToDateTime(GetProperty(appointment, "End")))); + } + else + { + ReleaseComObject(appointment); + } + } + + var selected = OutlookMeetingCandidateSelector.Select( + candidates, + startedLocal, + candidate => candidate.Start, + candidate => candidate.End); + if (selected is null) + { + foreach (var candidate in candidates) + { + ReleaseComObject(candidate.Appointment); + } + + return null; + } + + try + { + var appointment = selected.Appointment; + var title = Convert.ToString(GetProperty(appointment, "Subject")) ?? ""; + var attendees = ReadAttendees(appointment); + var body = Convert.ToString(GetProperty(appointment, "Body")) ?? ""; + return new MeetingMetadata( + title.Trim(), + attendees, + ExtractAgenda(body), + ToLocalOffset(selected.End)); + } + finally + { + foreach (var candidate in candidates) + { + ReleaseComObject(candidate.Appointment); + } + } + } + finally + { + ReleaseComObject(restrictedItems); + ReleaseComObject(items); + ReleaseComObject(calendar); + ReleaseComObject(session); + ReleaseComObject(application); + } + } + + internal static string ExtractAgenda(string body) + { + if (string.IsNullOrWhiteSpace(body)) + { + return ""; + } + + var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'); + var agendaLines = new List(); + foreach (var line in lines) + { + if (IsTeamsSeparator(line) || IsTeamsJoinLine(line)) + { + break; + } + + agendaLines.Add(line); + } + + return string.Join(Environment.NewLine, agendaLines).Trim(); + } + + private static bool IsTeamsAppointment(object appointment) + { + var subject = Convert.ToString(GetProperty(appointment, "Subject")) ?? ""; + var location = Convert.ToString(GetProperty(appointment, "Location")) ?? ""; + var body = Convert.ToString(GetProperty(appointment, "Body")) ?? ""; + return ContainsTeamsMarker(subject) || + ContainsTeamsMarker(location) || + ContainsTeamsMarker(body); + } + + private static bool ContainsTeamsMarker(string value) + { + return value.Contains("teams.microsoft.com", StringComparison.OrdinalIgnoreCase) || + value.Contains("Microsoft Teams", StringComparison.OrdinalIgnoreCase) || + value.Contains("Join the meeting now", StringComparison.OrdinalIgnoreCase) || + value.Contains("Join Microsoft Teams Meeting", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsTeamsSeparator(string line) + { + var trimmed = line.Trim(); + return trimmed.Length >= 8 && + trimmed.All(character => character is '_' or '-' or '*' or ' '); + } + + private static bool IsTeamsJoinLine(string line) + { + return ContainsTeamsMarker(line); + } + + private static IReadOnlyList ReadAttendees(object appointment) + { + var attendees = new List(); + var organizer = Convert.ToString(GetProperty(appointment, "Organizer")); + if (!string.IsNullOrWhiteSpace(organizer)) + { + attendees.Add(organizer.Trim()); + } + + object? recipients = null; + try + { + recipients = GetProperty(appointment, "Recipients"); + var count = Convert.ToInt32(GetProperty(recipients!, "Count")); + for (var index = 1; index <= count; index++) + { + object? recipient = null; + try + { + recipient = Invoke(recipients!, "Item", index); + var formatted = FormatRecipient(recipient!); + if (!string.IsNullOrWhiteSpace(formatted)) + { + attendees.Add(formatted); + } + } + finally + { + ReleaseComObject(recipient); + } + } + } + finally + { + ReleaseComObject(recipients); + } + + return attendees + .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static DateTimeOffset ToLocalOffset(DateTime value) + { + return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local)); + } + + private static string FormatRecipient(object recipient) + { + var name = Convert.ToString(GetProperty(recipient, "Name"))?.Trim() ?? ""; + var email = Convert.ToString(GetProperty(recipient, "Address"))?.Trim() ?? ""; + if (string.IsNullOrWhiteSpace(name)) + { + return email; + } + + if (string.IsNullOrWhiteSpace(email) || + email.Contains('/')) + { + return name; + } + + return $"{name} <{email}>"; + } + + private static object? GetProperty(object target, string name) + { + return target.GetType().InvokeMember( + name, + System.Reflection.BindingFlags.GetProperty, + null, + target, + null); + } + + private static void SetProperty(object target, string name, object value) + { + target.GetType().InvokeMember( + name, + System.Reflection.BindingFlags.SetProperty, + null, + target, + [value]); + } + + private static object? Invoke(object target, string name, params object[] arguments) + { + return target.GetType().InvokeMember( + name, + System.Reflection.BindingFlags.InvokeMethod, + null, + target, + arguments); + } + + private static void ReleaseComObject(object? value) + { + if (value is not null && Marshal.IsComObject(value)) + { + Marshal.FinalReleaseComObject(value); + } + } + + private sealed record OutlookAppointmentCandidate( + object Appointment, + DateTime Start, + DateTime End); +} diff --git a/MeetingAssistant/MeetingNotes/OutlookMeetingCandidateSelector.cs b/MeetingAssistant/MeetingNotes/OutlookMeetingCandidateSelector.cs new file mode 100644 index 0000000..01c1930 --- /dev/null +++ b/MeetingAssistant/MeetingNotes/OutlookMeetingCandidateSelector.cs @@ -0,0 +1,38 @@ +namespace MeetingAssistant.MeetingNotes; + +internal static class OutlookMeetingCandidateSelector +{ + private static readonly TimeSpan MinimumRemainingOverlap = TimeSpan.FromMinutes(5); + private static readonly TimeSpan UpcomingStartWindow = TimeSpan.FromMinutes(5); + + public static T? Select( + IReadOnlyList candidates, + DateTime startedAt, + Func getStart, + Func getEnd) + where T : class + { + var goodOverlaps = candidates + .Where(candidate => + getStart(candidate) <= startedAt && + getEnd(candidate) >= startedAt.Add(MinimumRemainingOverlap)) + .ToList(); + if (goodOverlaps.Count == 1) + { + return goodOverlaps[0]; + } + + if (goodOverlaps.Count > 1) + { + return null; + } + + var upcoming = candidates + .Where(candidate => + getStart(candidate) > startedAt && + getStart(candidate) <= startedAt.Add(UpcomingStartWindow)) + .OrderBy(getStart) + .ToList(); + return upcoming.Count == 1 ? upcoming[0] : null; + } +} diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index 9b4adb5..23d0de7 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -1,4 +1,51 @@ +using MeetingAssistant; +using MeetingAssistant.Hotkeys; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Recording; +using MeetingAssistant.Summary; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Options; + var builder = WebApplication.CreateBuilder(args); +builder.Services.Configure(builder.Configuration.GetSection("MeetingAssistant")); +#if WINDOWS +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(services => new CompositeMeetingAudioSource( + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService>())); +builder.Services.AddSingleton(); +#else +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +#endif +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddTransient(); +builder.Services.AddTransient(); +builder.Services.AddTransient(); +builder.Services.AddTransient(); +builder.Services.AddTransient(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +builder.Services.AddHostedService(); +#if WINDOWS +builder.Services.AddHostedService(); +#endif + var app = builder.Build(); app.MapGet("/health", () => Results.Ok(new @@ -7,6 +54,138 @@ app.MapGet("/health", () => Results.Ok(new status = "ok" })); -app.Run(); +app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus)); +app.MapPost("/recording/toggle", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => + Results.Ok(await coordinator.ToggleAsync(cancellationToken))); +app.MapPost("/recording/start", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => + Results.Ok(await coordinator.StartAsync(cancellationToken))); +app.MapPost("/recording/stop", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) => + Results.Ok(await coordinator.StopAsync(cancellationToken))); +app.MapPost("/meetings/current/summary/run", async ( + MeetingRecordingCoordinator coordinator, + IMeetingSummaryPipeline summaryPipeline, + CancellationToken cancellationToken) => +{ + if (coordinator.CurrentArtifacts is null) + { + return Results.Conflict(new { error = "No meeting session has been started yet." }); + } + + return Results.Ok(await summaryPipeline.RunAsync(coordinator.CurrentArtifacts, cancellationToken)); +}); +app.MapPost("/meetings/summary/retry", async ( + SummaryRetryRequest request, + IMeetingSummaryArtifactResolver artifactResolver, + IMeetingSummaryPipeline summaryPipeline, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(request.SummaryPath)) + { + return Results.BadRequest(new { error = "A summary path is required." }); + } + + var artifacts = await artifactResolver.ResolveBySummaryPathAsync(request.SummaryPath, cancellationToken); + if (artifacts is null) + { + return Results.NotFound(new { error = $"No meeting note links to summary '{request.SummaryPath}'." }); + } + + return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken)); +}); +app.MapGet("/meetings/summary/retry", async ( + string summaryPath, + IMeetingSummaryArtifactResolver artifactResolver, + IMeetingSummaryPipeline summaryPipeline, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(summaryPath)) + { + return Results.BadRequest(new { error = "A summary path is required." }); + } + + var artifacts = await artifactResolver.ResolveBySummaryPathAsync(summaryPath, cancellationToken); + if (artifacts is null) + { + return Results.NotFound(new { error = $"No meeting note links to summary '{summaryPath}'." }); + } + + return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken)); +}); +app.MapPost("/asr/transcribe-file", async ( + AsrDiagnosticRequest request, + AsrDiagnosticService diagnostics, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(request.Path)) + { + return Results.BadRequest(new { error = "A WAV file path is required." }); + } + + try + { + return Results.Ok(await diagnostics.TranscribeWavAsync(request.Path, cancellationToken)); + } + catch (FileNotFoundException exception) + { + return Results.NotFound(new { error = exception.Message }); + } + catch (InvalidDataException exception) + { + return Results.BadRequest(new { error = exception.Message }); + } + catch (Exception exception) + { + return Results.Problem( + detail: exception.Message, + statusCode: StatusCodes.Status502BadGateway, + title: "ASR backend failed while processing the WAV file."); + } +}); +app.MapPost("/asr/diarize-file", async ( + AsrDiagnosticRequest request, + AsrDiagnosticService diagnostics, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(request.Path)) + { + return Results.BadRequest(new { error = "A WAV file path is required." }); + } + + var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(request.Path)); + if (!File.Exists(fullPath)) + { + return Results.NotFound(new { error = $"WAV file was not found at '{fullPath}'." }); + } + + try + { + return Results.Ok(await diagnostics.DiarizeWavAsync( + fullPath, + new SpeechRecognitionPipelineOptions(request.NumSpeakers), + cancellationToken)); + } + catch (Exception exception) + { + return Results.Problem( + detail: exception.Message, + statusCode: StatusCodes.Status502BadGateway, + title: "ASR diarization backend failed while processing the WAV file."); + } +}); + +try +{ + app.Run(); +} +catch (TimeoutException exception) +{ + Console.Error.WriteLine( + $"Meeting Assistant failed to start because a required operation timed out: {exception.Message}"); + Environment.ExitCode = 1; +} public partial class Program; + +public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null); + +public sealed record SummaryRetryRequest(string SummaryPath); diff --git a/MeetingAssistant/Recording/AudioChunk.cs b/MeetingAssistant/Recording/AudioChunk.cs new file mode 100644 index 0000000..04ea2ec --- /dev/null +++ b/MeetingAssistant/Recording/AudioChunk.cs @@ -0,0 +1,18 @@ +namespace MeetingAssistant.Recording; + +public sealed record AudioChunk(byte[] Pcm, int SampleRate, int Channels) +{ + public TimeSpan Duration + { + get + { + var bytesPerSampleFrame = Channels * sizeof(short); + if (bytesPerSampleFrame <= 0 || SampleRate <= 0) + { + return TimeSpan.Zero; + } + + return TimeSpan.FromSeconds((double)Pcm.Length / bytesPerSampleFrame / SampleRate); + } + } +} diff --git a/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs new file mode 100644 index 0000000..2945448 --- /dev/null +++ b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs @@ -0,0 +1,155 @@ +using System.Runtime.CompilerServices; +using System.Threading.Channels; + +namespace MeetingAssistant.Recording; + +public sealed class CompositeMeetingAudioSource : IMeetingAudioSource +{ + private readonly IMeetingAudioSource microphone; + private readonly IMeetingAudioSource systemAudio; + private readonly ILogger logger; + + public CompositeMeetingAudioSource( + IMeetingAudioSource microphone, + IMeetingAudioSource systemAudio, + ILogger logger) + { + this.microphone = microphone; + this.systemAudio = systemAudio; + this.logger = logger; + } + + public async IAsyncEnumerable CaptureAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var channel = Channel.CreateUnbounded(); + var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, channel.Writer, cancellationToken); + var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, channel.Writer, cancellationToken); + _ = Task.WhenAll(microphoneTask, systemTask) + .ContinueWith( + task => + { + if (task.Exception is not null) + { + channel.Writer.TryComplete(task.Exception); + return; + } + + channel.Writer.TryComplete(); + }, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + + AudioChunk? pendingMicrophone = null; + AudioChunk? pendingSystem = null; + + await foreach (var sourceChunk in channel.Reader.ReadAllAsync(cancellationToken)) + { + if (sourceChunk.Kind == AudioSourceKind.Microphone) + { + pendingMicrophone = sourceChunk.Chunk; + } + else + { + pendingSystem = sourceChunk.Chunk; + } + + if (pendingMicrophone is not null && pendingSystem is not null) + { + yield return Mix(pendingMicrophone, pendingSystem); + pendingMicrophone = null; + pendingSystem = null; + continue; + } + + using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token); + try + { + if (await channel.Reader.WaitToReadAsync(linked.Token)) + { + continue; + } + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + } + + if (pendingMicrophone is not null) + { + yield return pendingMicrophone; + pendingMicrophone = null; + } + + if (pendingSystem is not null) + { + yield return pendingSystem; + pendingSystem = null; + } + } + + if (pendingMicrophone is not null) + { + yield return pendingMicrophone; + } + + if (pendingSystem is not null) + { + yield return pendingSystem; + } + } + + private static async Task PumpAsync( + AudioSourceKind kind, + IMeetingAudioSource source, + ChannelWriter writer, + CancellationToken cancellationToken) + { + await foreach (var chunk in source.CaptureAsync(cancellationToken)) + { + await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken); + } + } + + private AudioChunk Mix(AudioChunk left, AudioChunk right) + { + if (left.SampleRate != right.SampleRate || left.Channels != right.Channels) + { + logger.LogWarning( + "Cannot mix audio chunks with different formats. Using microphone format {MicrophoneSampleRate}/{MicrophoneChannels} and dropping system chunk format {SystemSampleRate}/{SystemChannels}.", + left.SampleRate, + left.Channels, + right.SampleRate, + right.Channels); + return left; + } + + var mixed = new byte[Math.Max(left.Pcm.Length, right.Pcm.Length)]; + var sampleCount = mixed.Length / sizeof(short); + + for (var sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) + { + var byteIndex = sampleIndex * sizeof(short); + var leftSample = ReadSample(left.Pcm, byteIndex); + var rightSample = ReadSample(right.Pcm, byteIndex); + var sample = Math.Clamp(leftSample + rightSample, short.MinValue, short.MaxValue); + BitConverter.GetBytes((short)sample).CopyTo(mixed, byteIndex); + } + + return new AudioChunk(mixed, left.SampleRate, left.Channels); + } + + private static int ReadSample(byte[] pcm, int byteIndex) + { + return byteIndex + 1 < pcm.Length ? BitConverter.ToInt16(pcm, byteIndex) : 0; + } + + private sealed record SourceChunk(AudioSourceKind Kind, AudioChunk Chunk); + + private enum AudioSourceKind + { + Microphone, + System + } +} diff --git a/MeetingAssistant/Recording/IMeetingAudioSource.cs b/MeetingAssistant/Recording/IMeetingAudioSource.cs new file mode 100644 index 0000000..8519f95 --- /dev/null +++ b/MeetingAssistant/Recording/IMeetingAudioSource.cs @@ -0,0 +1,6 @@ +namespace MeetingAssistant.Recording; + +public interface IMeetingAudioSource +{ + IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Recording/IRecordedAudioStore.cs b/MeetingAssistant/Recording/IRecordedAudioStore.cs new file mode 100644 index 0000000..5bab82b --- /dev/null +++ b/MeetingAssistant/Recording/IRecordedAudioStore.cs @@ -0,0 +1,19 @@ +namespace MeetingAssistant.Recording; + +public interface IRecordedAudioStore +{ + Task CreateSessionAsync(CancellationToken cancellationToken); + + Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken); +} + +public interface IRecordedAudioSink : IAsyncDisposable +{ + string AudioPath { get; } + + Task AppendAsync(AudioChunk chunk, CancellationToken cancellationToken); + + Task CompleteAsync(CancellationToken cancellationToken); + + Task DeleteAsync(CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Recording/ITranscriptStore.cs b/MeetingAssistant/Recording/ITranscriptStore.cs new file mode 100644 index 0000000..025931e --- /dev/null +++ b/MeetingAssistant/Recording/ITranscriptStore.cs @@ -0,0 +1,24 @@ +using MeetingAssistant.Transcription; +using MeetingAssistant.MeetingNotes; + +namespace MeetingAssistant.Recording; + +public interface ITranscriptStore +{ + Task CreateSessionAsync(CancellationToken cancellationToken); + + Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken); + + Task ReplaceAsync( + TranscriptSession session, + IReadOnlyList replacementSegments, + CancellationToken cancellationToken); + + Task UpdateMetadataAsync( + TranscriptSession session, + MeetingSessionArtifacts artifacts, + MeetingNote meetingNote, + CancellationToken cancellationToken); +} + +public sealed record TranscriptSession(string TranscriptPath); diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs new file mode 100644 index 0000000..b78b682 --- /dev/null +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -0,0 +1,449 @@ +using System.Threading.Channels; +using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using MeetingAssistant.Transcription; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Recording; + +public sealed class MeetingRecordingCoordinator +{ + private readonly IMeetingAudioSource audioSource; + private readonly ISpeechRecognitionPipelineFactory speechRecognitionPipelineFactory; + private readonly ITranscriptStore transcriptStore; + private readonly IMeetingNoteStore meetingNoteStore; + private readonly IMeetingNoteOpener meetingNoteOpener; + private readonly IMeetingArtifactStore meetingArtifactStore; + private readonly IMeetingMetadataProvider meetingMetadataProvider; + private readonly IRecordedAudioStore recordedAudioStore; + private readonly IMeetingSummaryPipeline summaryPipeline; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + private readonly SemaphoreSlim gate = new(1, 1); + private RecordingRun? currentRun; + private TranscriptSession? currentSession; + private MeetingNote? currentMeetingNote; + private MeetingSessionArtifacts? currentArtifacts; + + public MeetingRecordingCoordinator( + IMeetingAudioSource audioSource, + ISpeechRecognitionPipelineFactory speechRecognitionPipelineFactory, + ITranscriptStore transcriptStore, + IMeetingNoteStore meetingNoteStore, + IMeetingNoteOpener meetingNoteOpener, + IMeetingArtifactStore meetingArtifactStore, + IRecordedAudioStore recordedAudioStore, + IMeetingSummaryPipeline summaryPipeline, + IOptions options, + ILogger logger, + IMeetingMetadataProvider? meetingMetadataProvider = null) + { + this.audioSource = audioSource; + this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory; + this.transcriptStore = transcriptStore; + this.meetingNoteStore = meetingNoteStore; + this.meetingNoteOpener = meetingNoteOpener; + this.meetingArtifactStore = meetingArtifactStore; + this.meetingMetadataProvider = meetingMetadataProvider ?? new NoopMeetingMetadataProvider(); + this.recordedAudioStore = recordedAudioStore; + this.summaryPipeline = summaryPipeline; + this.options = options.Value; + this.logger = logger; + } + + public RecordingStatus CurrentStatus => new( + IsRecording, + currentArtifacts?.TranscriptPath ?? currentSession?.TranscriptPath, + currentArtifacts?.MeetingNotePath ?? currentMeetingNote?.Path, + currentArtifacts?.AssistantContextPath, + currentArtifacts?.SummaryPath); + + public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts; + + private bool IsRecording => currentRun is { IsCaptureStopping: false }; + + public async Task ToggleAsync(CancellationToken cancellationToken) + { + return IsRecording + ? await StopAsync(cancellationToken) + : await StartAsync(cancellationToken); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + await gate.WaitAsync(cancellationToken); + try + { + if (IsRecording) + { + return CurrentStatus; + } + + currentSession = await transcriptStore.CreateSessionAsync(cancellationToken); + var recordedAudio = await recordedAudioStore.CreateSessionAsync(cancellationToken); + var startedAt = DateTimeOffset.Now; + var assistantContextPath = GetAssistantContextPath(startedAt); + var summaryPath = GetSummaryPath(startedAt); + var meetingMetadata = await meetingMetadataProvider.GetCurrentMeetingAsync(startedAt, cancellationToken); + var title = string.IsNullOrWhiteSpace(meetingMetadata?.Title) + ? $"Meeting {startedAt:yyyy-MM-dd HH:mm}" + : meetingMetadata.Title; + var attendees = meetingMetadata?.Attendees ?? []; + var agenda = meetingMetadata?.Agenda ?? ""; + var scheduledEnd = meetingMetadata?.ScheduledEnd; + currentMeetingNote = await meetingNoteStore.SaveAsync( + MeetingNoteTemplate.Create( + title: title, + startTime: startedAt, + attendees: attendees, + transcriptPath: currentSession.TranscriptPath, + assistantContextPath: assistantContextPath, + summaryPath: summaryPath), + cancellationToken); + currentArtifacts = new MeetingSessionArtifacts( + currentMeetingNote.Path, + currentSession.TranscriptPath, + assistantContextPath, + summaryPath); + await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, agenda, scheduledEnd, cancellationToken); + await transcriptStore.UpdateMetadataAsync( + currentSession, + currentArtifacts, + currentMeetingNote, + cancellationToken); + await meetingNoteOpener.OpenAsync(currentMeetingNote.Path, cancellationToken); + var captureCancellation = new CancellationTokenSource(); + var transcriptionCancellation = new CancellationTokenSource(); + var pipeline = speechRecognitionPipelineFactory.Create(); + var run = new RecordingRun(captureCancellation, transcriptionCancellation, recordedAudio, pipeline); + run.Task = Task.Run(() => RecordAsync(currentSession, run), CancellationToken.None); + currentRun = run; + logger.LogInformation("Meeting recording started"); + + return CurrentStatus; + } + finally + { + gate.Release(); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + RecordingRun run; + + await gate.WaitAsync(cancellationToken); + try + { + if (currentRun is null || currentRun.IsCaptureStopping) + { + return CurrentStatus; + } + + currentRun.StopCapture(); + run = currentRun; + } + finally + { + gate.Release(); + } + + try + { + await run.Task.WaitAsync(options.Recording.StopProcessingTimeout, cancellationToken); + } + catch (OperationCanceledException) + { + } + catch (TimeoutException) + { + logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation"); + run.CancelTranscription(); + await run.Task.WaitAsync(cancellationToken); + } + + await CompleteRunAsync(run); + logger.LogInformation("Meeting recording stopped"); + return CurrentStatus; + } + + private async Task RecordAsync( + TranscriptSession session, + RecordingRun run) + { + await run.Pipeline.InitializeAsync(run.TranscriptionCancellation); + var liveTranscriptTask = Task.Run( + () => StoreLiveTranscriptAsync(session, run.Pipeline, run.TranscriptionCancellation), + CancellationToken.None); + var captureTask = Task.Run( + () => CaptureAudioAsync(run.Pipeline, run.RecordedAudio, run.CaptureCancellation), + CancellationToken.None); + + try + { + await captureTask; + await run.Pipeline.CompleteAsync(run.TranscriptionCancellation); + await liveTranscriptTask; + var artifacts = currentArtifacts; + if (artifacts is not null && HasConfiguredFinalizer()) + { + await meetingArtifactStore.UpdateAssistantContextStateAsync( + artifacts, + AssistantContextState.SpeakerRecognition, + run.TranscriptionCancellation); + } + + await RewriteTranscriptWithFinishedSegmentsAsync(session, run.Pipeline, run.RecordedAudio.AudioPath, run.TranscriptionCancellation); + var completedMeetingNote = await CompleteMeetingNoteAsync(run.TranscriptionCancellation); + if (artifacts is not null && completedMeetingNote is not null) + { + await transcriptStore.UpdateMetadataAsync( + session, + artifacts, + completedMeetingNote, + run.TranscriptionCancellation); + } + + if (artifacts is not null) + { + await RunSummaryAsync(artifacts, run.TranscriptionCancellation); + } + } + catch (OperationCanceledException) when (run.TranscriptionCancellation.IsCancellationRequested) + { + } + catch (Exception exception) when (run.TranscriptionCancellation.IsCancellationRequested) + { + logger.LogInformation(exception, "Meeting recording stopped while the speech recognition pipeline was processing audio"); + } + catch (Exception exception) + { + logger.LogError(exception, "Meeting recording failed"); + } + finally + { + await run.RecordedAudio.DeleteAsync(CancellationToken.None); + await run.Pipeline.DisposeAsync(); + await CompleteRunAsync(run); + } + } + + private async Task CaptureAudioAsync( + ISpeechRecognitionPipeline pipeline, + IRecordedAudioSink recordedAudio, + CancellationToken cancellationToken) + { + try + { + await foreach (var chunk in audioSource.CaptureAsync(cancellationToken)) + { + await recordedAudio.AppendAsync(chunk, cancellationToken); + await pipeline.WriteAsync(chunk, cancellationToken); + } + + await recordedAudio.CompleteAsync(cancellationToken); + await pipeline.CompleteAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await recordedAudio.CompleteAsync(CancellationToken.None); + await pipeline.CompleteAsync(CancellationToken.None); + } + } + + private async Task StoreLiveTranscriptAsync( + TranscriptSession session, + ISpeechRecognitionPipeline pipeline, + CancellationToken cancellationToken) + { + await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken)) + { + await transcriptStore.AppendAsync(session, segment, cancellationToken); + } + } + + private async Task RewriteTranscriptWithFinishedSegmentsAsync( + TranscriptSession session, + ISpeechRecognitionPipeline pipeline, + string audioPath, + CancellationToken cancellationToken) + { + var finishedSegments = await pipeline.ReadFinishedTranscriptAsync( + audioPath, + await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken), + cancellationToken); + if (finishedSegments.Count == 0) + { + return; + } + + await transcriptStore.ReplaceAsync(session, finishedSegments, cancellationToken); + } + + private async Task CompleteMeetingNoteAsync(CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) + { + return null; + } + + var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + meetingNote.Frontmatter.EndTime = DateTimeOffset.Now; + currentMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, cancellationToken); + return currentMeetingNote; + } + + private async Task RunSummaryAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + try + { + await meetingArtifactStore.UpdateAssistantContextStateAsync( + artifacts, + AssistantContextState.Summarizing, + cancellationToken); + var result = await summaryPipeline.RunAsync(artifacts, cancellationToken); + await meetingArtifactStore.UpdateAssistantContextStateAsync( + artifacts, + result.Succeeded ? AssistantContextState.Finished : AssistantContextState.Error, + CancellationToken.None); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Meeting summary pipeline failed unexpectedly"); + await meetingArtifactStore.UpdateAssistantContextStateAsync( + artifacts, + AssistantContextState.Error, + CancellationToken.None); + } + } + + private bool HasConfiguredFinalizer() + { + return options.Recording.TranscriptionProvider switch + { + "funasr" => options.FunAsr.Diarization.Enabled, + "whisper-local" => options.WhisperLocal.Diarization.Enabled, + _ => false + }; + } + + private async Task BuildSpeechRecognitionPipelineOptionsAsync( + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path)) + { + return SpeechRecognitionPipelineOptions.Default; + } + + var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); + var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee)); + return attendeeCount > 1 + ? new SpeechRecognitionPipelineOptions(attendeeCount) + : SpeechRecognitionPipelineOptions.Default; + } + + private string GetAssistantContextPath(DateTimeOffset startedAt) + { + return GetMeetingArtifactPath(options.Vault.AssistantContextFolder, startedAt, "assistant-context"); + } + + private string GetSummaryPath(DateTimeOffset startedAt) + { + return GetMeetingArtifactPath(options.Vault.SummariesFolder, startedAt, "summary"); + } + + private static string GetMeetingArtifactPath(string configuredFolder, DateTimeOffset startedAt, string artifactName) + { + var folder = VaultPath.Resolve(configuredFolder); + return Path.Combine(folder, $"{startedAt:yyyyMMdd-HHmmss}-{artifactName}.md"); + } + + private async Task CompleteRunAsync(RecordingRun run) + { + if (!run.TryComplete()) + { + return; + } + + await gate.WaitAsync(CancellationToken.None); + try + { + if (ReferenceEquals(currentRun, run)) + { + currentRun = null; + } + } + finally + { + gate.Release(); + run.Dispose(); + } + } + + private sealed class RecordingRun : IDisposable + { + private int completed; + + public RecordingRun( + CancellationTokenSource captureCancellation, + CancellationTokenSource transcriptionCancellation, + IRecordedAudioSink recordedAudio, + ISpeechRecognitionPipeline pipeline) + { + CaptureCancellationSource = captureCancellation; + TranscriptionCancellationSource = transcriptionCancellation; + RecordedAudio = recordedAudio; + Pipeline = pipeline; + } + + public CancellationTokenSource CaptureCancellationSource { get; } + + public CancellationTokenSource TranscriptionCancellationSource { get; } + + public IRecordedAudioSink RecordedAudio { get; } + + public ISpeechRecognitionPipeline Pipeline { get; } + + public CancellationToken CaptureCancellation => CaptureCancellationSource.Token; + + public CancellationToken TranscriptionCancellation => TranscriptionCancellationSource.Token; + + public Task Task { get; set; } = Task.CompletedTask; + + public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested; + + public void StopCapture() + { + CaptureCancellationSource.Cancel(); + } + + public void CancelTranscription() + { + TranscriptionCancellationSource.Cancel(); + } + + public bool TryComplete() + { + return Interlocked.Exchange(ref completed, 1) == 0; + } + + public void Dispose() + { + CaptureCancellationSource.Dispose(); + TranscriptionCancellationSource.Dispose(); + } + } +} + +public sealed record RecordingStatus( + bool IsRecording, + string? TranscriptPath, + string? MeetingNotePath, + string? AssistantContextPath, + string? SummaryPath); diff --git a/MeetingAssistant/Recording/NaudioCaptureSource.cs b/MeetingAssistant/Recording/NaudioCaptureSource.cs new file mode 100644 index 0000000..d0eb214 --- /dev/null +++ b/MeetingAssistant/Recording/NaudioCaptureSource.cs @@ -0,0 +1,86 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using NAudio.CoreAudioApi; +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public sealed class MicrophoneAudioSource : IMeetingAudioSource +{ + private readonly MeetingAssistantOptions options; + + public MicrophoneAudioSource(IOptions options) + { + this.options = options.Value; + } + + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + return CaptureAsync(new WasapiCapture(), cancellationToken); + } + + private IAsyncEnumerable CaptureAsync(IWaveIn capture, CancellationToken cancellationToken) + { + capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels); + return CaptureWith(capture, cancellationToken); + } + + internal static async IAsyncEnumerable CaptureWith( + IWaveIn capture, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var channel = Channel.CreateUnbounded(); + + capture.DataAvailable += (_, args) => + { + var buffer = new byte[args.BytesRecorded]; + Buffer.BlockCopy(args.Buffer, 0, buffer, 0, args.BytesRecorded); + channel.Writer.TryWrite(new AudioChunk(buffer, capture.WaveFormat.SampleRate, capture.WaveFormat.Channels)); + }; + capture.RecordingStopped += (_, args) => + { + if (args.Exception is not null) + { + channel.Writer.TryComplete(args.Exception); + return; + } + + channel.Writer.TryComplete(); + }; + + try + { + using var registration = cancellationToken.Register(capture.StopRecording); + capture.StartRecording(); + + await foreach (var chunk in channel.Reader.ReadAllAsync(cancellationToken)) + { + yield return chunk; + } + } + finally + { + capture.Dispose(); + } + } +} + +public sealed class SystemAudioSource : IMeetingAudioSource +{ + private readonly MeetingAssistantOptions options; + + public SystemAudioSource(IOptions options) + { + this.options = options.Value; + } + + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + var capture = new WasapiLoopbackCapture + { + WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels) + }; + + return MicrophoneAudioSource.CaptureWith(capture, cancellationToken); + } +} diff --git a/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs b/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs new file mode 100644 index 0000000..b49cb28 --- /dev/null +++ b/MeetingAssistant/Recording/TemporaryRecordedAudioStore.cs @@ -0,0 +1,141 @@ +using Microsoft.Extensions.Options; +using NAudio.Wave; + +namespace MeetingAssistant.Recording; + +public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore +{ + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public TemporaryRecordedAudioStore( + IOptions options, + ILogger logger) + { + this.options = options.Value; + this.logger = logger; + } + + public Task CreateSessionAsync(CancellationToken cancellationToken) + { + var folder = GetTemporaryRecordingsFolder(); + Directory.CreateDirectory(folder); + + var path = Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Guid.NewGuid():N}-recording.wav"); + var format = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels); + logger.LogInformation("Created temporary meeting recording file {RecordingPath}", path); + + return Task.FromResult(new TemporaryRecordedAudioSink(path, format, logger)); + } + + public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken) + { + var folder = GetTemporaryRecordingsFolder(); + if (!Directory.Exists(folder)) + { + return Task.CompletedTask; + } + + foreach (var path in Directory.EnumerateFiles(folder, "*.wav", SearchOption.TopDirectoryOnly)) + { + cancellationToken.ThrowIfCancellationRequested(); + DeleteFile(path); + } + + return Task.CompletedTask; + } + + private string GetTemporaryRecordingsFolder() + { + return VaultPath.Resolve(options.Recording.TemporaryRecordingsFolder); + } + + private void DeleteFile(string path) + { + try + { + File.Delete(path); + logger.LogInformation("Deleted temporary meeting recording file {RecordingPath}", path); + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + catch (IOException exception) + { + logger.LogWarning(exception, "Could not delete temporary meeting recording file {RecordingPath}", path); + } + catch (UnauthorizedAccessException exception) + { + logger.LogWarning(exception, "Could not delete temporary meeting recording file {RecordingPath}", path); + } + } + + private sealed class TemporaryRecordedAudioSink : IRecordedAudioSink + { + private readonly WaveFileWriter writer; + private readonly ILogger logger; + private bool completed; + private bool disposed; + + public TemporaryRecordedAudioSink(string audioPath, WaveFormat format, ILogger logger) + { + AudioPath = audioPath; + this.logger = logger; + writer = new WaveFileWriter(audioPath, format); + } + + public string AudioPath { get; } + + public Task AppendAsync(AudioChunk chunk, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + writer.Write(chunk.Pcm, 0, chunk.Pcm.Length); + return Task.CompletedTask; + } + + public Task CompleteAsync(CancellationToken cancellationToken) + { + if (completed) + { + return Task.CompletedTask; + } + + cancellationToken.ThrowIfCancellationRequested(); + writer.Flush(); + completed = true; + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + if (disposed) + { + return ValueTask.CompletedTask; + } + + writer.Dispose(); + disposed = true; + return ValueTask.CompletedTask; + } + + public async Task DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await DisposeAsync(); + try + { + File.Delete(AudioPath); + logger.LogInformation("Deleted temporary meeting recording file {RecordingPath}", AudioPath); + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + } + } +} diff --git a/MeetingAssistant/Recording/TemporaryRecordingCleanupHostedService.cs b/MeetingAssistant/Recording/TemporaryRecordingCleanupHostedService.cs new file mode 100644 index 0000000..79aeec9 --- /dev/null +++ b/MeetingAssistant/Recording/TemporaryRecordingCleanupHostedService.cs @@ -0,0 +1,26 @@ +namespace MeetingAssistant.Recording; + +public sealed class TemporaryRecordingCleanupHostedService : IHostedService +{ + private readonly IRecordedAudioStore recordedAudioStore; + private readonly ILogger logger; + + public TemporaryRecordingCleanupHostedService( + IRecordedAudioStore recordedAudioStore, + ILogger logger) + { + this.recordedAudioStore = recordedAudioStore; + this.logger = logger; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + logger.LogInformation("Deleting stale temporary meeting recordings"); + await recordedAudioStore.DeleteStaleRecordingsAsync(cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } +} diff --git a/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs b/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs new file mode 100644 index 0000000..91f722d --- /dev/null +++ b/MeetingAssistant/Recording/UnavailableMeetingAudioSource.cs @@ -0,0 +1,10 @@ +namespace MeetingAssistant.Recording; + +public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource +{ + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + throw new PlatformNotSupportedException( + "Meeting audio capture is only implemented for the Windows target in this version."); + } +} diff --git a/MeetingAssistant/Recording/VaultTranscriptStore.cs b/MeetingAssistant/Recording/VaultTranscriptStore.cs new file mode 100644 index 0000000..fe76bf4 --- /dev/null +++ b/MeetingAssistant/Recording/VaultTranscriptStore.cs @@ -0,0 +1,113 @@ +using MeetingAssistant.Transcription; +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Recording; + +public sealed class VaultTranscriptStore : ITranscriptStore +{ + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public VaultTranscriptStore(IOptions options, ILogger logger) + { + this.options = options.Value; + this.logger = logger; + } + + public async Task CreateSessionAsync(CancellationToken cancellationToken) + { + var folder = VaultPath.Resolve(options.Vault.TranscriptsFolder); + Directory.CreateDirectory(folder); + + var fileName = $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-transcript.md"; + var path = Path.Combine(folder, fileName); + await File.WriteAllTextAsync(path, $"# Meeting Transcript{Environment.NewLine}{Environment.NewLine}", cancellationToken); + logger.LogInformation("Created meeting transcript file {TranscriptPath}", path); + + return new TranscriptSession(path); + } + + public Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken) + { + return AppendToBodyAsync(session.TranscriptPath, FormatSegment(segment), cancellationToken); + } + + public Task ReplaceAsync( + TranscriptSession session, + IReadOnlyList replacementSegments, + CancellationToken cancellationToken) + { + var existing = File.Exists(session.TranscriptPath) + ? File.ReadAllText(session.TranscriptPath) + : ""; + var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing); + var body = "# Meeting Transcript" + + Environment.NewLine + + Environment.NewLine + + string.Concat(replacementSegments.Select(FormatSegment)); + var content = string.IsNullOrWhiteSpace(frontmatter) + ? body + : "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body; + return File.WriteAllTextAsync(session.TranscriptPath, content, cancellationToken); + } + + public async Task UpdateMetadataAsync( + TranscriptSession session, + MeetingSessionArtifacts artifacts, + MeetingNote meetingNote, + CancellationToken cancellationToken) + { + var body = File.Exists(session.TranscriptPath) + ? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body + : "# Meeting Transcript" + Environment.NewLine + Environment.NewLine; + var frontmatter = MeetingArtifactFrontmatterRenderer.Create( + artifacts, + meetingNote, + MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"), + session.TranscriptPath); + + await File.WriteAllTextAsync( + session.TranscriptPath, + MeetingArtifactFrontmatterRenderer.Render(frontmatter, body), + cancellationToken); + } + + private static string FormatSegment(TranscriptionSegment segment) + { + var speaker = string.IsNullOrWhiteSpace(segment.Speaker) ? "Unknown" : segment.Speaker; + return $"[{segment.Start:hh\\:mm\\:ss}] {speaker}: {segment.Text}{Environment.NewLine}"; + } + + private static async Task AppendToBodyAsync( + string path, + string text, + CancellationToken cancellationToken) + { + if (!File.Exists(path)) + { + await File.AppendAllTextAsync(path, text, cancellationToken); + return; + } + + var content = await File.ReadAllTextAsync(path, cancellationToken); + var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content); + if (string.IsNullOrWhiteSpace(frontmatter)) + { + await File.AppendAllTextAsync(path, text, cancellationToken); + return; + } + + var updated = "---" + + Environment.NewLine + + frontmatter + + Environment.NewLine + + "---" + + Environment.NewLine + + Environment.NewLine + + body + + text; + await File.WriteAllTextAsync(path, updated, cancellationToken); + } + +} diff --git a/MeetingAssistant/Summary/IMeetingSummaryPipeline.cs b/MeetingAssistant/Summary/IMeetingSummaryPipeline.cs new file mode 100644 index 0000000..f83f935 --- /dev/null +++ b/MeetingAssistant/Summary/IMeetingSummaryPipeline.cs @@ -0,0 +1,14 @@ +using MeetingAssistant.MeetingNotes; + +namespace MeetingAssistant.Summary; + +public interface IMeetingSummaryPipeline +{ + Task RunAsync(MeetingSessionArtifacts artifacts, CancellationToken cancellationToken); +} + +public sealed record MeetingSummaryRunResult( + string SummaryPath, + string AgentResponse, + bool Succeeded = true, + string? Error = null); diff --git a/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs b/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs new file mode 100644 index 0000000..fc21d8d --- /dev/null +++ b/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs @@ -0,0 +1,618 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Agents.AI.Compaction; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace MeetingAssistant.Summary; + +#pragma warning disable MAAI001 +public sealed class LiteLlmResponsesChatClient : IChatClient +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient httpClient; + private readonly string model; + private readonly bool enableThinking; + private readonly string reasoningEffort; + private readonly int reconnectionAttempts; + private readonly TimeSpan reconnectionDelay; + private readonly LiteLlmResponsesCompactionOptions? compactionOptions; + private readonly ILogger? logger; + + public LiteLlmResponsesChatClient( + Uri endpoint, + string apiKey, + string model, + bool enableThinking, + string reasoningEffort, + int reconnectionAttempts, + TimeSpan reconnectionDelay, + LiteLlmResponsesCompactionOptions? compactionOptions = null, + ILogger? logger = null) + : this( + new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) }, + apiKey, + model, + enableThinking, + reasoningEffort, + reconnectionAttempts, + reconnectionDelay, + compactionOptions, + logger) + { + } + + internal LiteLlmResponsesChatClient( + HttpClient httpClient, + string apiKey, + string model, + bool enableThinking, + string reasoningEffort, + int reconnectionAttempts, + TimeSpan reconnectionDelay, + LiteLlmResponsesCompactionOptions? compactionOptions = null, + ILogger? logger = null) + { + this.httpClient = httpClient; + this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); + this.model = model; + this.enableThinking = enableThinking; + this.reasoningEffort = reasoningEffort; + this.reconnectionAttempts = Math.Max(0, reconnectionAttempts); + this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero; + this.compactionOptions = compactionOptions; + this.logger = logger; + } + + public void Dispose() + { + httpClient.Dispose(); + } + + public async Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false); + var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false); + + var response = ParseResponseJson(responseJson); + LogResponseUsage(response.Usage); + return response; + } + + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + foreach (var message in response.Messages) + { + yield return new ChatResponseUpdate(message.Role, message.Contents); + } + } + + public object? GetService(Type serviceType, object? serviceKey = null) + { + if (serviceKey is not null) + { + return null; + } + + return serviceType == typeof(ChatClientMetadata) + ? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model) + : null; + } + + public static ChatResponse ParseResponseJson(string responseJson) + { + using var document = JsonDocument.Parse(responseJson); + var root = document.RootElement; + var contents = new List(); + + if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array) + { + foreach (var item in output.EnumerateArray()) + { + var type = GetString(item, "type"); + if (type == "message") + { + AddMessageContent(contents, item); + } + else if (type == "function_call") + { + contents.Add(new FunctionCallContent( + GetRequiredString(item, "call_id"), + GetRequiredString(item, "name"), + ParseArguments(GetString(item, "arguments")))); + } + } + } + + var message = new ChatMessage + { + Role = ChatRole.Assistant, + Contents = contents + }; + + return new ChatResponse(message) + { + ResponseId = GetString(root, "id"), + ModelId = GetString(root, "model"), + CreatedAt = GetUnixTimestamp(root, "created_at"), + Usage = ParseUsage(root), + RawRepresentation = responseJson + }; + } + + private async Task CreateCompactedPayloadAsync( + IReadOnlyList messages, + ChatOptions? options, + CancellationToken cancellationToken) + { + var payload = CreatePayload(messages, options); + var estimate = EstimateTokens(payload); + var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue; + if (compactionOptions?.Enabled != true || estimate < triggerTokens) + { + LogContextWindow(estimate, compacted: false, source: "none"); + return payload; + } + + LogContextWindow(estimate, compacted: false, source: "threshold"); + + var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken) + .ConfigureAwait(false); + if (remotePayload is not null) + { + LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact"); + return remotePayload; + } + + if (compactionOptions.FallbackStrategy is null) + { + return payload; + } + + try + { + var compactedMessages = await CompactionProvider.CompactAsync( + compactionOptions.FallbackStrategy, + messages, + logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + cancellationToken) + .ConfigureAwait(false); + var fallbackPayload = CreatePayload(compactedMessages, options); + LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework"); + return fallbackPayload; + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger?.LogWarning( + exception, + "Agent Framework compaction fallback failed; sending un-compacted summary request"); + return payload; + } + } + + private async Task TryCompactWithResponsesEndpointAsync( + JsonObject payload, + int estimatedTokens, + CancellationToken cancellationToken) + { + try + { + var request = new JsonObject + { + ["model"] = payload["model"]?.DeepClone(), + ["input"] = payload["input"]?.DeepClone(), + ["instructions"] = payload["instructions"]?.DeepClone(), + ["context_window_tokens"] = compactionOptions!.ContextWindowTokens, + ["max_output_tokens"] = compactionOptions.MaxOutputTokens, + ["estimated_input_tokens"] = estimatedTokens, + ["target_input_tokens"] = compactionOptions.TriggerTokens, + ["strategy"] = new JsonArray + { + "collapse_tool_results", + "summarize_older_spans", + "keep_last_4_user_turns", + "drop_oldest_groups" + } + }; + + using var content = new StringContent(request.ToJsonString(JsonOptions), Encoding.UTF8, "application/json"); + using var response = await httpClient.PostAsync( + NormalizeRelativePath(compactionOptions.CompactPath), + content, + cancellationToken) + .ConfigureAwait(false); + var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + logger?.LogWarning( + "Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}", + (int)response.StatusCode, + response.ReasonPhrase, + responseJson); + return null; + } + + return ApplyCompactedInput(payload, responseJson); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction"); + return null; + } + } + + private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson) + { + using var document = JsonDocument.Parse(responseJson); + var root = document.RootElement; + var input = FindCompactedInput(root); + if (input is null) + { + return null; + } + + var compactedPayload = (JsonObject)payload.DeepClone(); + compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText()); + if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String) + { + compactedPayload["instructions"] = instructions.GetString(); + } + + return compactedPayload; + } + + private static JsonElement? FindCompactedInput(JsonElement root) + { + foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" }) + { + if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array) + { + return value; + } + } + + return null; + } + + private JsonObject CreatePayload(IEnumerable messages, ChatOptions? options) + { + var instructions = new StringBuilder(); + if (!string.IsNullOrWhiteSpace(options?.Instructions)) + { + instructions.AppendLine(options.Instructions); + } + + var input = new JsonArray(); + foreach (var message in messages) + { + AddInputItem(input, instructions, message); + } + + var payload = new JsonObject + { + ["model"] = options?.ModelId ?? model, + ["input"] = input, + ["store"] = false, + ["tool_choice"] = "auto" + }; + + if (instructions.Length > 0) + { + payload["instructions"] = instructions.ToString().Trim(); + } + + if (enableThinking) + { + payload["reasoning"] = new JsonObject + { + ["effort"] = reasoningEffort + }; + } + + var tools = CreateTools(options?.Tools); + if (tools.Count > 0) + { + payload["tools"] = tools; + payload["parallel_tool_calls"] = true; + } + + return payload; + } + + private async Task PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken) + { + var payloadJson = payload.ToJsonString(JsonOptions); + Exception? lastException = null; + + for (var attempt = 0; attempt <= reconnectionAttempts; attempt++) + { + try + { + using var content = new StringContent(payloadJson, Encoding.UTF8, "application/json"); + using var response = await httpClient.PostAsync("responses", content, cancellationToken).ConfigureAwait(false); + var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + return responseJson; + } + + var exception = new InvalidOperationException( + $"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}"); + if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts) + { + throw exception; + } + + lastException = exception; + } + catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts) + { + lastException = exception; + } + + await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false); + } + + throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed."); + } + + private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken) + { + if (reconnectionDelay > TimeSpan.Zero) + { + await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false); + } + } + + private static bool IsRetryableStatusCode(int statusCode) + { + return statusCode is 408 or 409 or 425 or 429 + || statusCode >= 500; + } + + private static bool IsRetryableException(Exception exception) + { + return exception is HttpRequestException or TaskCanceledException; + } + + private void LogContextWindow(int estimatedTokens, bool compacted, string source) + { + if (compactionOptions is null || logger is null) + { + return; + } + + var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); + var remainingTokens = Math.Max(0, contextWindow - estimatedTokens); + logger.LogInformation( + "Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}", + estimatedTokens, + contextWindow, + remainingTokens, + compacted, + source); + } + + private void LogResponseUsage(UsageDetails? usage) + { + if (usage is null || compactionOptions is null || logger is null) + { + return; + } + + var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); + var inputTokens = usage.InputTokenCount ?? 0; + var outputTokens = usage.OutputTokenCount ?? 0; + var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens; + logger.LogInformation( + "Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}", + inputTokens, + outputTokens, + totalTokens, + contextWindow); + } + + private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message) + { + var role = message.Role.Value; + var text = message.Text; + if (role == ChatRole.System.Value) + { + if (!string.IsNullOrWhiteSpace(text)) + { + instructions.AppendLine(text); + } + + return; + } + + if (!string.IsNullOrWhiteSpace(text)) + { + input.Add(new JsonObject + { + ["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user", + ["content"] = text + }); + } + + foreach (var content in message.Contents) + { + if (content is FunctionCallContent call) + { + input.Add(new JsonObject + { + ["type"] = "function_call", + ["call_id"] = call.CallId, + ["name"] = call.Name, + ["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions) + }); + } + else if (content is FunctionResultContent result) + { + input.Add(new JsonObject + { + ["type"] = "function_call_output", + ["call_id"] = result.CallId, + ["output"] = ResultToString(result.Result) + }); + } + } + } + + private static JsonArray CreateTools(IList? tools) + { + var result = new JsonArray(); + if (tools is null) + { + return result; + } + + foreach (var tool in tools.OfType()) + { + result.Add(new JsonObject + { + ["type"] = "function", + ["name"] = tool.Name, + ["description"] = tool.Description ?? string.Empty, + ["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText()) + }); + } + + return result; + } + + private static void AddMessageContent(List contents, JsonElement item) + { + if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array) + { + return; + } + + foreach (var content in messageContent.EnumerateArray()) + { + if (GetString(content, "type") == "output_text") + { + var text = GetString(content, "text"); + if (!string.IsNullOrEmpty(text)) + { + contents.Add(new TextContent(text)); + } + } + } + } + + private static Dictionary ParseArguments(string? arguments) + { + if (string.IsNullOrWhiteSpace(arguments)) + { + return []; + } + + using var document = JsonDocument.Parse(arguments); + return document.RootElement.EnumerateObject() + .ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value)); + } + + private static object? ConvertJsonValue(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString(), + JsonValueKind.Number when element.TryGetInt64(out var integer) => integer, + JsonValueKind.Number when element.TryGetDouble(out var number) => number, + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + _ => JsonSerializer.Deserialize(element.GetRawText(), JsonOptions) + }; + } + + private static string ResultToString(object? result) + { + return result switch + { + null => string.Empty, + string text => text, + _ => JsonSerializer.Serialize(result, JsonOptions) + }; + } + + private static UsageDetails? ParseUsage(JsonElement root) + { + if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object) + { + return null; + } + + var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens"); + var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens"); + var totalTokens = GetInt(usage, "total_tokens"); + return new UsageDetails + { + InputTokenCount = inputTokens, + OutputTokenCount = outputTokens, + TotalTokenCount = totalTokens + }; + } + + private static int EstimateTokens(JsonObject payload) + { + var json = payload.ToJsonString(JsonOptions); + return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0)); + } + + private static int? GetInt(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value) + ? value + : null; + } + + private static string GetRequiredString(JsonElement element, string propertyName) + { + return GetString(element, propertyName) + ?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'."); + } + + private static string? GetString(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String + ? property.GetString() + : null; + } + + private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value) + ? DateTimeOffset.FromUnixTimeSeconds(value) + : null; + } + + private static Uri NormalizeEndpoint(Uri endpoint) + { + var value = endpoint.ToString().TrimEnd('/'); + if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) + { + value += "/v1"; + } + + return new Uri(value + "/"); + } + + private static string NormalizeRelativePath(string value) + { + return value.TrimStart('/'); + } +} +#pragma warning restore MAAI001 diff --git a/MeetingAssistant/Summary/LiteLlmResponsesCompactionOptions.cs b/MeetingAssistant/Summary/LiteLlmResponsesCompactionOptions.cs new file mode 100644 index 0000000..87a6603 --- /dev/null +++ b/MeetingAssistant/Summary/LiteLlmResponsesCompactionOptions.cs @@ -0,0 +1,32 @@ +using Microsoft.Agents.AI.Compaction; + +namespace MeetingAssistant.Summary; + +#pragma warning disable MAAI001 +public sealed class LiteLlmResponsesCompactionOptions +{ + public bool Enabled { get; init; } + + public int ContextWindowTokens { get; init; } + + public int MaxOutputTokens { get; init; } + + public double RemainingRatio { get; init; } + + public string CompactPath { get; init; } = "responses/compact"; + + public CompactionStrategy? FallbackStrategy { get; init; } + + public int TriggerTokens + { + get + { + var contextWindow = Math.Max(1, ContextWindowTokens); + var outputReserve = Math.Clamp(MaxOutputTokens, 0, contextWindow - 1); + var inputBudget = contextWindow - outputReserve; + var remainingRatio = Math.Clamp(RemainingRatio, 0.01, 0.90); + return Math.Max(1, (int)Math.Floor(inputBudget * (1 - remainingRatio))); + } + } +} +#pragma warning restore MAAI001 diff --git a/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs b/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs new file mode 100644 index 0000000..9f69189 --- /dev/null +++ b/MeetingAssistant/Summary/MeetingSummaryArtifactResolver.cs @@ -0,0 +1,126 @@ +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Summary; + +public interface IMeetingSummaryArtifactResolver +{ + Task ResolveBySummaryPathAsync( + string summaryPath, + CancellationToken cancellationToken); +} + +public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactResolver +{ + private readonly MeetingAssistantOptions options; + private readonly IMeetingNoteStore meetingNoteStore; + + public MeetingSummaryArtifactResolver( + IOptions options, + IMeetingNoteStore meetingNoteStore) + { + this.options = options.Value; + this.meetingNoteStore = meetingNoteStore; + } + + public async Task ResolveBySummaryPathAsync( + string summaryPath, + CancellationToken cancellationToken) + { + var requestedSummaryPath = ResolveRequestedSummaryPath(summaryPath); + if (requestedSummaryPath is null) + { + return null; + } + + var meetingNotesFolder = VaultPath.Resolve(options.Vault.MeetingNotesFolder); + if (!Directory.Exists(meetingNotesFolder)) + { + return null; + } + + foreach (var meetingNotePath in Directory.EnumerateFiles(meetingNotesFolder, "*.md")) + { + var note = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken); + var noteSummaryPath = ResolveNoteLink(note.Frontmatter.Summary, note.Path); + if (!PathsEqual(requestedSummaryPath, noteSummaryPath)) + { + continue; + } + + return new MeetingSessionArtifacts( + note.Path, + ResolveNoteLink(note.Frontmatter.Transcript, note.Path), + ResolveNoteLink(note.Frontmatter.AssistantContext, note.Path), + requestedSummaryPath); + } + + return null; + } + + private string? ResolveRequestedSummaryPath(string summaryPath) + { + if (string.IsNullOrWhiteSpace(summaryPath)) + { + return null; + } + + var summariesFolder = VaultPath.Resolve(options.Vault.SummariesFolder); + var requested = Environment.ExpandEnvironmentVariables(summaryPath); + var fullPath = Path.IsPathRooted(requested) + ? Path.GetFullPath(requested) + : Path.GetFullPath(Path.Combine(summariesFolder, requested)); + + return IsWithinDirectory(summariesFolder, fullPath) ? fullPath : null; + } + + private static string ResolveNoteLink(string pathOrLink, string sourceNotePath) + { + if (string.IsNullOrWhiteSpace(pathOrLink)) + { + return ""; + } + + var sourceFolder = Path.GetDirectoryName(sourceNotePath) ?? ""; + var target = ExtractWikiLinkTarget(pathOrLink) ?? pathOrLink; + target = target.Replace('/', Path.DirectorySeparatorChar); + + if (!Path.HasExtension(target)) + { + target += ".md"; + } + + return Path.GetFullPath(Path.IsPathRooted(target) + ? target + : Path.Combine(sourceFolder, target)); + } + + private static string? ExtractWikiLinkTarget(string value) + { + if (!value.StartsWith("[[", StringComparison.Ordinal) || + !value.EndsWith("]]", StringComparison.Ordinal)) + { + return null; + } + + var inner = value[2..^2]; + var labelSeparator = inner.IndexOf('|', StringComparison.Ordinal); + return labelSeparator < 0 ? inner : inner[..labelSeparator]; + } + + private static bool PathsEqual(string left, string right) + { + return string.Equals( + Path.GetFullPath(left), + Path.GetFullPath(right), + StringComparison.OrdinalIgnoreCase); + } + + private static bool IsWithinDirectory(string directory, string path) + { + var normalizedDirectory = Path.GetFullPath(directory) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + var normalizedPath = Path.GetFullPath(path); + return normalizedPath.StartsWith(normalizedDirectory, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/MeetingAssistant/Summary/MeetingSummaryFailureWriter.cs b/MeetingAssistant/Summary/MeetingSummaryFailureWriter.cs new file mode 100644 index 0000000..5ff5bcd --- /dev/null +++ b/MeetingAssistant/Summary/MeetingSummaryFailureWriter.cs @@ -0,0 +1,83 @@ +using System.Text; +using MeetingAssistant.MeetingNotes; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Summary; + +public interface IMeetingSummaryFailureWriter +{ + Task WriteAsync( + MeetingSessionArtifacts artifacts, + Exception exception, + CancellationToken cancellationToken); +} + +public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter +{ + private readonly MeetingAssistantOptions options; + private readonly IMeetingNoteStore meetingNoteStore; + + public MeetingSummaryFailureWriter( + IOptions options, + IMeetingNoteStore meetingNoteStore) + { + this.options = options.Value; + this.meetingNoteStore = meetingNoteStore; + } + + public async Task WriteAsync( + MeetingSessionArtifacts artifacts, + Exception exception, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!); + var meetingNote = File.Exists(artifacts.MeetingNotePath) + ? await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken) + : new MeetingNote("", new MeetingNoteFrontmatter(), ""); + var frontmatter = MeetingArtifactFrontmatterRenderer.Create( + artifacts, + meetingNote, + MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Summary"), + artifacts.SummaryPath); + await File.WriteAllTextAsync( + artifacts.SummaryPath, + MeetingArtifactFrontmatterRenderer.Render( + frontmatter, + RenderFailureMarkdown(artifacts, exception)), + cancellationToken); + + return new MeetingSummaryRunResult( + artifacts.SummaryPath, + "Summary generation failed. See the summary file for error details.", + Succeeded: false, + Error: exception.Message); + } + + private string RenderFailureMarkdown(MeetingSessionArtifacts artifacts, Exception exception) + { + var builder = new StringBuilder(); + builder.AppendLine("# Meeting Summary"); + builder.AppendLine(); + builder.AppendLine("## Summary Generation Failed"); + builder.AppendLine(); + builder.AppendLine($"Summary generation failed at {DateTimeOffset.Now:O}."); + builder.AppendLine(); + builder.AppendLine(MeetingNoteActionLinks.CreateSummaryRetryLink( + options.Api.PublicBaseUrl, + artifacts.SummaryPath)); + builder.AppendLine(); + builder.AppendLine("## Error"); + builder.AppendLine(); + builder.AppendLine("```text"); + builder.AppendLine(exception.ToString().Replace("```", "` ` `", StringComparison.Ordinal)); + builder.AppendLine("```"); + builder.AppendLine(); + builder.AppendLine("## Meeting Artifacts"); + builder.AppendLine(); + builder.AppendLine($"- Meeting note: `{artifacts.MeetingNotePath}`"); + builder.AppendLine($"- Transcript: `{artifacts.TranscriptPath}`"); + builder.AppendLine($"- Assistant context: `{artifacts.AssistantContextPath}`"); + builder.AppendLine($"- Summary: `{artifacts.SummaryPath}`"); + return builder.ToString(); + } +} diff --git a/MeetingAssistant/Summary/MeetingSummaryTools.cs b/MeetingAssistant/Summary/MeetingSummaryTools.cs new file mode 100644 index 0000000..4b8eb53 --- /dev/null +++ b/MeetingAssistant/Summary/MeetingSummaryTools.cs @@ -0,0 +1,641 @@ +using MeetingAssistant.MeetingNotes; +using System.Diagnostics; +using System.Text.Json; +using YamlDotNet.Serialization; + +namespace MeetingAssistant.Summary; + +public sealed class MeetingSummaryTools +{ + private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + + private readonly MeetingSessionArtifacts artifacts; + private readonly MeetingAssistantOptions options; + + public MeetingSummaryTools(MeetingSessionArtifacts artifacts) + : this(artifacts, new MeetingAssistantOptions()) + { + } + + public MeetingSummaryTools(MeetingSessionArtifacts artifacts, MeetingAssistantOptions options) + { + this.artifacts = artifacts; + this.options = options; + } + + public Task ReadTranscript(int? @from = null, int? to = null) + { + return ReadIfExistsAsync(artifacts.TranscriptPath, @from, to); + } + + public Task ReadMeetingNote(int? @from = null, int? to = null) + { + return ReadIfExistsAsync(artifacts.MeetingNotePath, @from, to); + } + + public Task ReadContext(int? @from = null, int? to = null) + { + return ReadIfExistsAsync(artifacts.AssistantContextPath, @from, to); + } + + public async Task ReadUserNotes(int? @from = null, int? to = null) + { + if (!File.Exists(artifacts.MeetingNotePath)) + { + return ""; + } + + var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath); + var document = MarkdownDocumentParser.SplitOptional(content); + var body = document.HasFrontmatter ? document.Body.Trim() : content; + return ReadLines(body, @from, to); + } + + public Task ReadGlossary(int? @from = null, int? to = null) + { + return ReadIfExistsAsync(VaultPath.Resolve(options.Vault.DictationWordsPath), @from, to); + } + + public async Task WriteSummary(string markdown, string? title = null) + { + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!); + var meetingNote = await ReadMeetingNoteAsync(); + var summaryTitle = string.IsNullOrWhiteSpace(meetingNote.Frontmatter.Title) + ? title + : meetingNote.Frontmatter.Title; + var frontmatter = MeetingArtifactFrontmatterRenderer.Create( + artifacts, + meetingNote, + string.IsNullOrWhiteSpace(summaryTitle) ? "Meeting Summary" : summaryTitle, + artifacts.SummaryPath); + await File.WriteAllTextAsync( + artifacts.SummaryPath, + MeetingArtifactFrontmatterRenderer.Render(frontmatter, markdown)); + return artifacts.SummaryPath; + } + + public async Task WriteContext( + string content, + int? @from = null, + int? to = null, + int? insert = null) + { + var editMode = FileLineEditMode.Create(@from, to, insert); + if (editMode is null) + { + return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for overwrite."; + } + + Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!); + var existing = File.Exists(artifacts.AssistantContextPath) + ? await File.ReadAllTextAsync(artifacts.AssistantContextPath) + : ""; + var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(existing); + var updatedBody = ApplyLineEdit(body, content, editMode); + var updated = string.IsNullOrWhiteSpace(frontmatter) + ? updatedBody + : "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + updatedBody; + + await File.WriteAllTextAsync(artifacts.AssistantContextPath, updated); + return artifacts.AssistantContextPath; + } + + public async Task ListProjects() + { + var projects = await GetBoundProjectsAsync(); + return string.Join('\n', projects.Select(project => project.Name)); + } + + public async Task ListProjectFiles(string project) + { + var projectRoot = await ResolveBoundProjectRootAsync(project); + if (projectRoot is null) + { + return ""; + } + + var files = Directory.EnumerateFiles(projectRoot, "*", SearchOption.AllDirectories) + .Select(path => ToToolPath(Path.GetRelativePath(projectRoot, path))) + .Order(StringComparer.Ordinal); + return string.Join('\n', files); + } + + public async Task ReadProjectFile(string project, string path, int? @from = null, int? to = null) + { + var filePath = await ResolveBoundProjectFilePathAsync(project, path); + if (filePath is null || !File.Exists(filePath)) + { + return ""; + } + + return ReadLines(await File.ReadAllTextAsync(filePath), @from, to); + } + + public async Task WriteProjectFile( + string project, + string path, + string content, + int? @from = null, + int? to = null, + int? insert = null) + { + var editMode = FileLineEditMode.Create(@from, to, insert); + if (editMode is null) + { + return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for overwrite."; + } + + var target = ResolveExistingProjectFilePath(project, path); + if (target is null) + { + return "Refused: project does not exist or the path escapes the project folder."; + } + + Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!); + await WriteProjectFileContentAsync(target.Path, content, editMode); + return $"{target.Project.Name}/{ToToolPath(path)}"; + } + + public async Task Search(string keywords, string[]? projects = null) + { + if (string.IsNullOrWhiteSpace(keywords)) + { + return ""; + } + + var selectedProjects = await GetSearchProjectsAsync(projects); + if (selectedProjects.Count == 0) + { + return ""; + } + + var projectsRoot = GetProjectsRoot(); + var result = await RunRipgrepAsync(projectsRoot, selectedProjects, keywords); + return result is not null + ? FormatRipgrepJson(result, selectedProjects.Count == 1) + : SearchWithRegexFallback(selectedProjects, keywords, selectedProjects.Count == 1); + } + + private static async Task ReadIfExistsAsync(string path, int? from = null, int? to = null) + { + if (!File.Exists(path)) + { + return ""; + } + + var content = await File.ReadAllTextAsync(path); + return ReadLines(content, from, to); + } + + private async Task ReadMeetingNoteAsync() + { + if (!File.Exists(artifacts.MeetingNotePath)) + { + return new MeetingNote("", new MeetingNoteFrontmatter(), ""); + } + + var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath); + var document = MarkdownDocumentParser.SplitOptional(content); + var frontmatter = new MeetingNoteFrontmatter(); + if (document.HasFrontmatter) + { + var note = YamlDeserializer.Deserialize(document.Frontmatter) ?? new MeetingNoteFrontmatterYaml(); + frontmatter.Title = note.Title ?? ""; + frontmatter.StartTime = ParseDateTime(note.StartTime); + frontmatter.EndTime = ParseDateTime(note.EndTime); + frontmatter.Attendees = note.Attendees ?? []; + frontmatter.Projects = note.Projects ?? []; + frontmatter.Transcript = note.Transcript ?? ""; + frontmatter.AssistantContext = note.AssistantContext ?? ""; + frontmatter.Summary = note.Summary ?? ""; + } + + return new MeetingNote(artifacts.MeetingNotePath, frontmatter, await ReadUserNotes()); + } + + private async Task> GetSearchProjectsAsync(string[]? projects) + { + var boundProjects = await GetBoundProjectsAsync(); + if (projects is null || projects.Length == 0) + { + return boundProjects; + } + + var requested = projects + .Where(project => !string.IsNullOrWhiteSpace(project)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + return boundProjects + .Where(project => requested.Contains(project.Name)) + .ToList(); + } + + private async Task ResolveBoundProjectAsync(string project) + { + if (string.IsNullOrWhiteSpace(project)) + { + return null; + } + + return (await GetBoundProjectsAsync()) + .FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase)); + } + + private async Task ResolveBoundProjectRootAsync(string project) + { + return (await ResolveBoundProjectAsync(project))?.Path; + } + + private async Task ResolveBoundProjectFilePathAsync(string project, string path) + { + var projectRoot = await ResolveBoundProjectRootAsync(project); + if (projectRoot is null || string.IsNullOrWhiteSpace(path) || Path.IsPathRooted(path)) + { + return null; + } + + var fullPath = Path.GetFullPath(Path.Combine(projectRoot, path)); + return IsWithinDirectory(projectRoot, fullPath) ? fullPath : null; + } + + private ProjectFileTarget? ResolveExistingProjectFilePath(string project, string path) + { + if (string.IsNullOrWhiteSpace(project) || + string.IsNullOrWhiteSpace(path) || + Path.IsPathRooted(path)) + { + return null; + } + + var projectsRoot = GetProjectsRoot(); + if (!Directory.Exists(projectsRoot)) + { + return null; + } + + var projectFolder = Directory.EnumerateDirectories(projectsRoot) + .Select(candidate => new ProjectFolder(Path.GetFileName(candidate), candidate)) + .FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase)); + if (projectFolder is null) + { + return null; + } + + var fullPath = Path.GetFullPath(Path.Combine(projectFolder.Path, path)); + return IsWithinDirectory(projectFolder.Path, fullPath) + ? new ProjectFileTarget(projectFolder, fullPath) + : null; + } + + private static async Task WriteProjectFileContentAsync( + string filePath, + string content, + FileLineEditMode editMode) + { + if (editMode.Kind == FileLineEditKind.Overwrite) + { + await File.WriteAllTextAsync(filePath, content); + return; + } + + var lines = File.Exists(filePath) + ? (await File.ReadAllLinesAsync(filePath)).ToList() + : []; + var replacementLines = SplitLines(content); + + if (editMode.Kind == FileLineEditKind.Insert) + { + var insertIndex = Math.Clamp(editMode.Insert!.Value - 1, 0, lines.Count); + lines.InsertRange(insertIndex, replacementLines); + await File.WriteAllTextAsync(filePath, string.Join('\n', lines)); + return; + } + + var startIndex = Math.Clamp(editMode.From!.Value - 1, 0, lines.Count); + var endIndexExclusive = Math.Clamp(editMode.To!.Value, 0, lines.Count); + if (endIndexExclusive < startIndex) + { + endIndexExclusive = startIndex; + } + + lines.RemoveRange(startIndex, endIndexExclusive - startIndex); + lines.InsertRange(startIndex, replacementLines); + await File.WriteAllTextAsync(filePath, string.Join('\n', lines)); + } + + private static List SplitLines(string content) + { + if (content.Length == 0) + { + return []; + } + + return content + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n') + .ToList(); + } + + private static string ReadLines(string content, int? from = null, int? to = null) + { + if (!from.HasValue && !to.HasValue) + { + return content; + } + + var lines = SplitLines(content); + if (lines.Count == 0) + { + return ""; + } + + var start = Math.Clamp((from ?? 1) - 1, 0, lines.Count - 1); + var end = Math.Clamp((to ?? lines.Count) - 1, 0, lines.Count - 1); + if (end < start) + { + return ""; + } + + return string.Join('\n', lines.GetRange(start, end - start + 1)); + } + + private static string ApplyLineEdit( + string existingContent, + string replacementContent, + FileLineEditMode editMode) + { + if (editMode.Kind == FileLineEditKind.Overwrite) + { + return replacementContent; + } + + var lines = SplitLines(existingContent); + var replacementLines = SplitLines(replacementContent); + if (editMode.Kind == FileLineEditKind.Insert) + { + var insertIndex = Math.Clamp(editMode.Insert!.Value - 1, 0, lines.Count); + lines.InsertRange(insertIndex, replacementLines); + return string.Join('\n', lines); + } + + var startIndex = Math.Clamp(editMode.From!.Value - 1, 0, lines.Count); + var endIndexExclusive = Math.Clamp(editMode.To!.Value, 0, lines.Count); + if (endIndexExclusive < startIndex) + { + endIndexExclusive = startIndex; + } + + lines.RemoveRange(startIndex, endIndexExclusive - startIndex); + lines.InsertRange(startIndex, replacementLines); + return string.Join('\n', lines); + } + + private async Task> GetBoundProjectsAsync() + { + var projectsRoot = GetProjectsRoot(); + if (!Directory.Exists(projectsRoot)) + { + return []; + } + + var projectNames = await ReadMeetingProjectNamesAsync(); + if (projectNames.Count == 0) + { + return []; + } + + return Directory.EnumerateDirectories(projectsRoot) + .Select(path => new ProjectFolder(Path.GetFileName(path), path)) + .Where(project => projectNames.Contains(project.Name)) + .OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private async Task> ReadMeetingProjectNamesAsync() + { + if (!File.Exists(artifacts.MeetingNotePath)) + { + return []; + } + + var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath); + var document = MarkdownDocumentParser.SplitOptional(content); + if (!document.HasFrontmatter) + { + return []; + } + + var note = YamlDeserializer.Deserialize(document.Frontmatter) ?? new ProjectFrontmatter(); + return (note.Projects ?? []) + .Where(project => !string.IsNullOrWhiteSpace(project)) + .Select(project => project.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + private string GetProjectsRoot() + { + return VaultPath.Resolve(options.Vault.ProjectsFolder); + } + + private static async Task RunRipgrepAsync( + string projectsRoot, + IReadOnlyList projects, + string keywords) + { + try + { + var startInfo = new ProcessStartInfo + { + FileName = "rg", + WorkingDirectory = projectsRoot, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + startInfo.ArgumentList.Add("--json"); + startInfo.ArgumentList.Add("--line-number"); + startInfo.ArgumentList.Add("--color"); + startInfo.ArgumentList.Add("never"); + startInfo.ArgumentList.Add("--"); + startInfo.ArgumentList.Add(keywords); + foreach (var project in projects) + { + startInfo.ArgumentList.Add(project.Name); + } + + using var process = Process.Start(startInfo); + if (process is null) + { + return null; + } + + var output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + return process.ExitCode is 0 or 1 ? output : null; + } + catch + { + return null; + } + } + + private static string FormatRipgrepJson(string output, bool singleProject) + { + var matches = new List(); + using var reader = new StringReader(output); + string? line; + while ((line = reader.ReadLine()) is not null) + { + using var document = JsonDocument.Parse(line); + var root = document.RootElement; + if (!root.TryGetProperty("type", out var type) || + type.GetString() != "match" || + !root.TryGetProperty("data", out var data)) + { + continue; + } + + var path = data.GetProperty("path").GetProperty("text").GetString() ?? ""; + var lineNumber = data.GetProperty("line_number").GetInt32(); + var text = data.GetProperty("lines").GetProperty("text").GetString() ?? ""; + matches.Add($"{FormatSearchPath(path, singleProject)}:{lineNumber} {text.TrimEnd('\r', '\n')}"); + } + + return string.Join('\n', matches); + } + + private static string SearchWithRegexFallback(IReadOnlyList projects, string keywords, bool singleProject) + { + try + { + var regex = new System.Text.RegularExpressions.Regex(keywords); + var matches = new List(); + foreach (var project in projects) + { + foreach (var file in Directory.EnumerateFiles(project.Path, "*", SearchOption.AllDirectories)) + { + var lines = File.ReadAllLines(file); + for (var index = 0; index < lines.Length; index++) + { + if (regex.IsMatch(lines[index])) + { + var relative = Path.Combine(project.Name, Path.GetRelativePath(project.Path, file)); + matches.Add($"{FormatSearchPath(relative, singleProject)}:{index + 1} {lines[index]}"); + } + } + } + } + + return string.Join('\n', matches); + } + catch + { + return ""; + } + } + + private static string FormatSearchPath(string path, bool singleProject) + { + var formatted = ToToolPath(path); + if (!singleProject) + { + return formatted; + } + + var slashIndex = formatted.IndexOf('/', StringComparison.Ordinal); + return slashIndex < 0 ? formatted : formatted[(slashIndex + 1)..]; + } + + private static bool IsWithinDirectory(string directory, string path) + { + var normalizedDirectory = Path.GetFullPath(directory) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + var normalizedPath = Path.GetFullPath(path); + return normalizedPath.StartsWith(normalizedDirectory, StringComparison.OrdinalIgnoreCase); + } + + private static string ToToolPath(string path) + { + return path.Replace(Path.DirectorySeparatorChar.ToString(), "/", StringComparison.Ordinal) + .Replace(Path.AltDirectorySeparatorChar.ToString(), "/", StringComparison.Ordinal); + } + + private sealed record ProjectFolder(string Name, string Path); + + private sealed record ProjectFileTarget(ProjectFolder Project, string Path); + + private sealed record FileLineEditMode( + FileLineEditKind Kind, + int? From = null, + int? To = null, + int? Insert = null) + { + public static FileLineEditMode? Create(int? from, int? to, int? insert) + { + if (insert.HasValue) + { + return from.HasValue || to.HasValue + ? null + : new FileLineEditMode(FileLineEditKind.Insert, Insert: insert); + } + + if (from.HasValue || to.HasValue) + { + return from.HasValue && to.HasValue && from.Value <= to.Value + ? new FileLineEditMode(FileLineEditKind.Replace, From: from, To: to) + : null; + } + + return new FileLineEditMode(FileLineEditKind.Overwrite); + } + } + + private enum FileLineEditKind + { + Overwrite, + Replace, + Insert + } + + private sealed class ProjectFrontmatter + { + [YamlDotNet.Serialization.YamlMember(Alias = "projects")] + public List? Projects { get; set; } + } + + private sealed class MeetingNoteFrontmatterYaml + { + [YamlDotNet.Serialization.YamlMember(Alias = "title")] + public string? Title { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "start_time")] + public string? StartTime { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "end_time")] + public string? EndTime { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "attendees")] + public List? Attendees { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "projects")] + public List? Projects { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "transcript")] + public string? Transcript { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "assistant_context")] + public string? AssistantContext { get; set; } + + [YamlDotNet.Serialization.YamlMember(Alias = "summary")] + public string? Summary { get; set; } + } + + private static DateTimeOffset? ParseDateTime(string? value) + { + return DateTimeOffset.TryParse(value, out var parsed) + ? parsed + : null; + } +} diff --git a/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs b/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs new file mode 100644 index 0000000..41074f7 --- /dev/null +++ b/MeetingAssistant/Summary/OpenAiMeetingSummaryAgentPipeline.cs @@ -0,0 +1,296 @@ +using MeetingAssistant.MeetingNotes; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Compaction; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Summary; + +#pragma warning disable MAAI001 +public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline +{ + private const string Instructions = """ + You are the Meeting Assistant summary agent. + + Use the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files. + All read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines. + Then write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary. + Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time. + Use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note. + After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context. + Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files. + The summary note should contain concise sections for summary, decisions, open questions, and next steps. + Keep the output grounded in the source material and explicitly say when a section has no known items. + """; + + private readonly MeetingAssistantOptions options; + private readonly ILoggerFactory loggerFactory; + private readonly ILogger logger; + private readonly IServiceProvider services; + private readonly IMeetingSummaryFailureWriter failureWriter; + + public OpenAiMeetingSummaryAgentPipeline( + IOptions options, + ILoggerFactory loggerFactory, + ILogger logger, + IServiceProvider services, + IMeetingSummaryFailureWriter failureWriter) + { + this.options = options.Value; + this.loggerFactory = loggerFactory; + this.logger = logger; + this.services = services; + this.failureWriter = failureWriter; + } + + public async Task RunAsync( + MeetingSessionArtifacts artifacts, + CancellationToken cancellationToken) + { + var agentOptions = options.Agent; + var key = ResolveApiKey(agentOptions); + var tools = CreateTools(new MeetingSummaryTools(artifacts, options)); + using var compactionSummaryClient = new LiteLlmResponsesChatClient( + new Uri(agentOptions.Endpoint), + key, + agentOptions.Model, + agentOptions.EnableThinking, + ToReasoningEffortValue(agentOptions.ReasoningEffort), + agentOptions.ReconnectionAttempts, + agentOptions.ReconnectionDelay, + compactionOptions: null, + logger); + var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient); + using var chatClient = new LiteLlmResponsesChatClient( + new Uri(agentOptions.Endpoint), + key, + agentOptions.Model, + agentOptions.EnableThinking, + ToReasoningEffortValue(agentOptions.ReasoningEffort), + agentOptions.ReconnectionAttempts, + agentOptions.ReconnectionDelay, + compactionOptions, + logger); + var agent = chatClient + .AsBuilder() + .UseFunctionInvocation() + .Build() + .AsAIAgent( + Instructions, + "meeting_summary", + "Writes the markdown summary note for a captured meeting.", + tools, + loggerFactory: loggerFactory, + services: services); + + try + { + var response = await agent.RunAsync( + "Write the summary for this meeting. Use the tools; do not invent missing information.", + session: null, + options: new ChatClientAgentRunOptions(CreateChatOptions(agentOptions)), + cancellationToken: cancellationToken); + + return new MeetingSummaryRunResult(artifacts.SummaryPath, response.Text); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError( + exception, + "Meeting summary generation failed for {SummaryPath}; writing failure summary", + artifacts.SummaryPath); + return await failureWriter.WriteAsync(artifacts, exception, cancellationToken); + } + } + + private static List CreateTools(MeetingSummaryTools tools) + { + return + [ + AIFunctionFactory.Create( + tools.ReadMeetingNote, + "read_meetingnote", + "Read the meeting note, including YAML frontmatter and user-authored body notes. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."), + AIFunctionFactory.Create( + tools.ReadTranscript, + "read_transcript", + "Read the transcript markdown for the current meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."), + AIFunctionFactory.Create( + tools.ReadContext, + "read_context", + "Read the assistant context markdown for the current meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."), + AIFunctionFactory.Create( + tools.ReadUserNotes, + "read_usernotes", + "Read the user-authored body notes from the meeting note. With no line arguments, read the whole body. With from and to, read that clamped inclusive 1-based body line range."), + AIFunctionFactory.Create( + tools.WriteContext, + "write_context", + "Write internal assistant notes to the assistant context body. With no line arguments, overwrite the body. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."), + AIFunctionFactory.Create( + tools.ReadGlossary, + "read_glossary", + "Read the glossary or vocabulary context for this meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range. May be empty."), + AIFunctionFactory.Create( + tools.WriteSummary, + "write_summary", + "Write the finished summary markdown to the configured summary note. Provide title when the meeting note has no title."), + AIFunctionFactory.Create( + tools.ListProjects, + "list_projects", + "List the configured project folders bound to the current meeting note frontmatter."), + AIFunctionFactory.Create( + tools.ListProjectFiles, + "list_projectfiles", + "List markdown and other files in a bound project. The project parameter is a project folder name."), + AIFunctionFactory.Create( + tools.ReadProjectFile, + "read_projectfile", + "Read a bound project file, optionally clamping to from and to inclusive line numbers."), + AIFunctionFactory.Create( + tools.WriteProjectFile, + "write_projectfile", + "Write a file inside an existing project folder. With no line arguments, overwrite or create the file. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."), + AIFunctionFactory.Create( + tools.Search, + "search", + "Run ripgrep syntax keywords against the requested bound projects, or all projects bound to the meeting note.") + ]; + } + + private static ChatOptions CreateChatOptions(AgentOptions options) + { + var chatOptions = new ChatOptions + { + ModelId = options.Model, + AllowMultipleToolCalls = true, + ToolMode = ChatToolMode.Auto + }; + + if (options.EnableThinking) + { + chatOptions.Reasoning = new ReasoningOptions + { + Effort = ToReasoningEffort(options.ReasoningEffort) + }; + } + + return chatOptions; + } + + private static LiteLlmResponsesCompactionOptions CreateCompactionOptions( + AgentOptions options, + IChatClient? summaryClient) + { + return new LiteLlmResponsesCompactionOptions + { + Enabled = options.EnableCompaction, + ContextWindowTokens = options.ContextWindowTokens, + MaxOutputTokens = options.MaxOutputTokens, + RemainingRatio = options.CompactionRemainingRatio, + CompactPath = options.ResponsesCompactPath, + FallbackStrategy = CreateFallbackCompactionStrategy( + options.ContextWindowTokens, + options.MaxOutputTokens, + options.CompactionRemainingRatio, + summaryClient) + }; + } + + internal static CompactionStrategy CreateFallbackCompactionStrategyForTests( + int contextWindowTokens, + int maxOutputTokens, + double remainingRatio, + IChatClient? summaryClient) + { + return CreateFallbackCompactionStrategy( + contextWindowTokens, + maxOutputTokens, + remainingRatio, + summaryClient); + } + + private static CompactionStrategy CreateFallbackCompactionStrategy( + int contextWindowTokens, + int maxOutputTokens, + double remainingRatio, + IChatClient? summaryClient) + { + var contextWindow = Math.Max(1, contextWindowTokens); + var outputReserve = Math.Clamp(maxOutputTokens, 0, contextWindow - 1); + var inputBudget = contextWindow - outputReserve; + var triggerTokens = Math.Max(1, (int)Math.Floor(inputBudget * (1 - Math.Clamp(remainingRatio, 0.01, 0.90)))); + var target = CompactionTriggers.TokensBelow(Math.Max(1, triggerTokens - 1)); + var trigger = CompactionTriggers.TokensExceed(triggerTokens); + var strategies = new List + { + new ToolResultCompactionStrategy(trigger, minimumPreservedGroups: 4, target) + }; + + if (summaryClient is not null) + { + strategies.Add(new SummarizationCompactionStrategy( + summaryClient, + trigger, + minimumPreservedGroups: 4, + target: target)); + } + + strategies.Add(new SlidingWindowCompactionStrategy( + trigger, + minimumPreservedTurns: 4, + target)); + strategies.Add(new TruncationCompactionStrategy( + trigger, + minimumPreservedGroups: 1, + target)); + + return new PipelineCompactionStrategy(strategies); + } + + private static string ToReasoningEffortValue(ReasoningEffortOption effort) + { + return effort switch + { + ReasoningEffortOption.None => "none", + ReasoningEffortOption.Low => "low", + ReasoningEffortOption.High => "high", + ReasoningEffortOption.ExtraHigh => "xhigh", + _ => "medium" + }; + } + + private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort) + { + return effort switch + { + ReasoningEffortOption.None => ReasoningEffort.None, + ReasoningEffortOption.Low => ReasoningEffort.Low, + ReasoningEffortOption.High => ReasoningEffort.High, + ReasoningEffortOption.ExtraHigh => ReasoningEffort.ExtraHigh, + _ => ReasoningEffort.Medium + }; + } + + private static string ResolveApiKey(AgentOptions options) + { + if (!string.IsNullOrWhiteSpace(options.Key)) + { + return options.Key; + } + + var value = Environment.GetEnvironmentVariable(options.KeyEnv); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + + throw new InvalidOperationException( + $"No agent API key configured. Set MeetingAssistant:Agent:Key or environment variable '{options.KeyEnv}'."); + } +} +#pragma warning restore MAAI001 diff --git a/MeetingAssistant/Transcription/AsrDiagnosticService.cs b/MeetingAssistant/Transcription/AsrDiagnosticService.cs new file mode 100644 index 0000000..00864ce --- /dev/null +++ b/MeetingAssistant/Transcription/AsrDiagnosticService.cs @@ -0,0 +1,119 @@ +using MeetingAssistant.Recording; +using NAudio.Wave; + +namespace MeetingAssistant.Transcription; + +public sealed class AsrDiagnosticService +{ + private readonly ISpeechRecognitionPipelineFactory pipelineFactory; + + public AsrDiagnosticService(ISpeechRecognitionPipelineFactory pipelineFactory) + { + this.pipelineFactory = pipelineFactory; + } + + public async Task TranscribeWavAsync(string path, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("A WAV file path is required.", nameof(path)); + } + + var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path)); + if (!File.Exists(fullPath)) + { + throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath); + } + + await using var pipeline = pipelineFactory.Create(); + await pipeline.InitializeAsync(cancellationToken); + var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken); + await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken); + await pipeline.CompleteAsync(cancellationToken); + return new AsrDiagnosticResult(fullPath, await liveTranscriptTask); + } + + public async Task DiarizeWavAsync( + string path, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + var fullPath = ResolveExistingWavPath(path); + await using var pipeline = pipelineFactory.Create(); + await pipeline.InitializeAsync(cancellationToken); + var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken); + await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken); + await pipeline.CompleteAsync(cancellationToken); + await liveTranscriptTask; + var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(fullPath, options, cancellationToken); + + return new AsrDiagnosticResult(fullPath, finishedSegments); + } + + private static async Task> CollectLiveTranscriptAsync( + ISpeechRecognitionPipeline pipeline, + CancellationToken cancellationToken) + { + var segments = new List(); + await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken)) + { + segments.Add(segment); + } + + return segments; + } + + private static async Task WriteWavToPipelineAsync( + string fullPath, + ISpeechRecognitionPipeline pipeline, + bool paceAudio, + CancellationToken cancellationToken) + { + await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken)) + { + await pipeline.WriteAsync(chunk, cancellationToken); + if (paceAudio && chunk.Duration > TimeSpan.Zero) + { + await Task.Delay(chunk.Duration, cancellationToken); + } + } + } + + private static string ResolveExistingWavPath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("A WAV file path is required.", nameof(path)); + } + + var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path)); + if (!File.Exists(fullPath)) + { + throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath); + } + + return fullPath; + } + + private static async IAsyncEnumerable ReadPcmWavChunks( + string path, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using var reader = new WaveFileReader(path); + var format = reader.WaveFormat; + if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16) + { + throw new InvalidDataException( + $"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample."); + } + + var buffer = new byte[format.AverageBytesPerSecond]; + int read; + while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0) + { + yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels); + } + } +} + +public sealed record AsrDiagnosticResult(string Path, IReadOnlyList Segments); diff --git a/MeetingAssistant/Transcription/AzureSpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/AzureSpeechRecognitionPipeline.cs new file mode 100644 index 0000000..54ed983 --- /dev/null +++ b/MeetingAssistant/Transcription/AzureSpeechRecognitionPipeline.cs @@ -0,0 +1,23 @@ +namespace MeetingAssistant.Transcription; + +public sealed class AzureSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline +{ + public AzureSpeechRecognitionPipeline(AzureSpeechStreamingTranscriptionProvider transcriptionProvider) + : base(transcriptionProvider) + { + } + + internal AzureSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider) + : base(transcriptionProvider) + { + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return Task.FromResult(liveSegments); + } +} diff --git a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs new file mode 100644 index 0000000..6934881 --- /dev/null +++ b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs @@ -0,0 +1,190 @@ +using System.Threading.Channels; +using MeetingAssistant.Recording; +using Microsoft.CognitiveServices.Speech; +using Microsoft.CognitiveServices.Speech.Audio; +using Microsoft.CognitiveServices.Speech.Transcription; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTranscriptionProvider +{ + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public AzureSpeechStreamingTranscriptionProvider( + IOptions options, + ILogger logger) + { + this.options = options.Value; + this.logger = logger; + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await using var enumerator = audio.GetAsyncEnumerator(cancellationToken); + if (!await enumerator.MoveNextAsync()) + { + yield break; + } + + var firstChunk = enumerator.Current; + using var pushStream = AudioInputStream.CreatePushStream( + AudioStreamFormat.GetWaveFormatPCM( + (uint)firstChunk.SampleRate, + 16, + (byte)firstChunk.Channels)); + using var audioConfig = AudioConfig.FromStreamInput(pushStream); + var speechConfig = CreateSpeechConfig(); + using var recognizer = new ConversationTranscriber(speechConfig, audioConfig); + var segments = Channel.CreateUnbounded(); + + recognizer.Transcribed += (_, args) => + { + if (args.Result.Reason != ResultReason.RecognizedSpeech + || string.IsNullOrWhiteSpace(args.Result.Text)) + { + return; + } + + var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); + var segment = new TranscriptionSegment( + start, + start + args.Result.Duration, + NormalizeSpeakerId(args.Result.SpeakerId), + args.Result.Text.Trim()); + logger.LogInformation( + "Azure Speech emitted segment at {Start} from {Speaker}: {Text}", + segment.Start, + segment.Speaker, + segment.Text); + segments.Writer.TryWrite(segment); + }; + recognizer.Canceled += (_, args) => + { + if (args.Reason == CancellationReason.Error) + { + segments.Writer.TryComplete(new InvalidOperationException( + $"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}")); + return; + } + + segments.Writer.TryComplete(); + }; + + await recognizer.StartTranscribingAsync(); + var pumpTask = Task.Run( + () => PumpAudioAsync( + firstChunk, + enumerator, + pushStream, + recognizer, + segments.Writer, + options.AzureSpeech.RecognitionStopTimeout, + cancellationToken), + CancellationToken.None); + + await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) + { + yield return segment; + } + + await pumpTask.WaitAsync(cancellationToken); + } + + private SpeechConfig CreateSpeechConfig() + { + var azure = options.AzureSpeech; + var key = ResolveKey(azure); + if (string.IsNullOrWhiteSpace(key)) + { + throw new InvalidOperationException( + $"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}."); + } + + var speechConfig = string.IsNullOrWhiteSpace(azure.Region) + ? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key) + : SpeechConfig.FromSubscription(key, azure.Region); + speechConfig.SpeechRecognitionLanguage = azure.Language; + speechConfig.OutputFormat = OutputFormat.Detailed; + speechConfig.SetProperty( + PropertyId.SpeechServiceResponse_DiarizeIntermediateResults, + azure.DiarizeIntermediateResults ? "true" : "false"); + return speechConfig; + } + + private static async Task PumpAudioAsync( + AudioChunk firstChunk, + IAsyncEnumerator audio, + PushAudioInputStream pushStream, + ConversationTranscriber recognizer, + ChannelWriter segments, + TimeSpan recognitionStopTimeout, + CancellationToken cancellationToken) + { + try + { + WriteChunk(pushStream, firstChunk, firstChunk); + while (await audio.MoveNextAsync()) + { + cancellationToken.ThrowIfCancellationRequested(); + WriteChunk(pushStream, firstChunk, audio.Current); + } + + pushStream.Close(); + try + { + var stopTask = recognizer.StopTranscribingAsync(); + if (recognitionStopTimeout > TimeSpan.Zero) + { + await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken); + } + else + { + await stopTask.WaitAsync(cancellationToken); + } + } + catch (TimeoutException) + { + // The SDK can outlive short diagnostic streams while waiting for final service events. + } + + segments.TryComplete(); + } + catch (Exception exception) + { + segments.TryComplete(exception); + } + } + + private static void WriteChunk(PushAudioInputStream pushStream, AudioChunk expectedFormat, AudioChunk chunk) + { + if (chunk.SampleRate != expectedFormat.SampleRate || chunk.Channels != expectedFormat.Channels) + { + return; + } + + pushStream.Write(chunk.Pcm); + } + + private static string NormalizeSpeakerId(string? speakerId) + { + return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase) + ? "Unknown" + : speakerId; + } + + private static string ResolveKey(AzureSpeechOptions options) + { + if (!string.IsNullOrWhiteSpace(options.Key)) + { + return options.Key; + } + + return string.IsNullOrWhiteSpace(options.KeyEnv) + ? "" + : Environment.GetEnvironmentVariable(options.KeyEnv) ?? ""; + } +} diff --git a/MeetingAssistant/Transcription/ClientFunAsrWebSocketConnection.cs b/MeetingAssistant/Transcription/ClientFunAsrWebSocketConnection.cs new file mode 100644 index 0000000..6a17c54 --- /dev/null +++ b/MeetingAssistant/Transcription/ClientFunAsrWebSocketConnection.cs @@ -0,0 +1,78 @@ +using System.Net.WebSockets; +using System.Text; + +namespace MeetingAssistant.Transcription; + +public sealed class ClientFunAsrWebSocketConnectionFactory : IFunAsrWebSocketConnectionFactory +{ + public async Task ConnectAsync(Uri endpoint, CancellationToken cancellationToken) + { + var socket = new ClientWebSocket(); + await socket.ConnectAsync(endpoint, cancellationToken); + return new ClientFunAsrWebSocketConnection(socket); + } +} + +public sealed class ClientFunAsrWebSocketConnection : IFunAsrWebSocketConnection +{ + private readonly ClientWebSocket socket; + + public ClientFunAsrWebSocketConnection(ClientWebSocket socket) + { + this.socket = socket; + } + + public Task SendTextAsync(string message, CancellationToken cancellationToken) + { + return socket.SendAsync( + Encoding.UTF8.GetBytes(message), + WebSocketMessageType.Text, + endOfMessage: true, + cancellationToken); + } + + public async Task SendBinaryAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + await socket.SendAsync(message, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken); + } + + public async Task ReceiveTextAsync(CancellationToken cancellationToken) + { + var buffer = new byte[8192]; + using var message = new MemoryStream(); + + while (true) + { + var result = await socket.ReceiveAsync(buffer, cancellationToken); + if (result.MessageType == WebSocketMessageType.Close) + { + return null; + } + + if (result.MessageType != WebSocketMessageType.Text) + { + continue; + } + + message.Write(buffer, 0, result.Count); + if (result.EndOfMessage) + { + return Encoding.UTF8.GetString(message.ToArray()); + } + } + } + + public async Task CloseOutputAsync(CancellationToken cancellationToken) + { + if (socket.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + await socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "meeting assistant finished sending audio", cancellationToken); + } + } + + public ValueTask DisposeAsync() + { + socket.Dispose(); + return ValueTask.CompletedTask; + } +} diff --git a/MeetingAssistant/Transcription/CommandRunner.cs b/MeetingAssistant/Transcription/CommandRunner.cs new file mode 100644 index 0000000..14fc483 --- /dev/null +++ b/MeetingAssistant/Transcription/CommandRunner.cs @@ -0,0 +1,101 @@ +using System.Diagnostics; +using System.Text; + +namespace MeetingAssistant.Transcription; + +public interface ICommandRunner +{ + Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null); +} + +public sealed class ProcessCommandRunner : ICommandRunner +{ + private readonly ILogger logger; + + public ProcessCommandRunner(ILogger logger) + { + this.logger = logger; + } + + public async Task RunAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken, + IReadOnlyDictionary? environment = null) + { + var startInfo = new ProcessStartInfo + { + FileName = fileName, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + if (environment is not null) + { + foreach (var (key, value) in environment) + { + startInfo.Environment[key] = value; + } + } + + logger.LogInformation("Running command {Command} {Arguments}", fileName, string.Join(' ', arguments)); + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start command '{fileName}'."); + var outputTask = ReadLinesAsync(process.StandardOutput, line => logger.LogInformation("{Command}: {Line}", fileName, line), cancellationToken); + var errorTask = ReadLinesAsync(process.StandardError, line => logger.LogWarning("{Command}: {Line}", fileName, line), cancellationToken); + + try + { + await process.WaitForExitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + if (!process.HasExited) + { + logger.LogWarning("Cancelling command {Command}; killing process tree.", fileName); + process.Kill(entireProcessTree: true); + } + + throw; + } + + var result = new CommandResult(process.ExitCode, await outputTask, await errorTask); + if (result.ExitCode != 0) + { + logger.LogWarning( + "Command {Command} exited with {ExitCode}: {Error}", + fileName, + result.ExitCode, + result.StandardError); + } + + return result; + } + + private static async Task ReadLinesAsync( + StreamReader reader, + Action log, + CancellationToken cancellationToken) + { + var output = new StringBuilder(); + while (await reader.ReadLineAsync(cancellationToken) is { } line) + { + output.AppendLine(line); + log(line); + } + + return output.ToString(); + } +} + +public sealed record CommandResult(int ExitCode, string StandardOutput, string StandardError); diff --git a/MeetingAssistant/Transcription/ConfiguredSpeechRecognitionPipelineFactory.cs b/MeetingAssistant/Transcription/ConfiguredSpeechRecognitionPipelineFactory.cs new file mode 100644 index 0000000..5984946 --- /dev/null +++ b/MeetingAssistant/Transcription/ConfiguredSpeechRecognitionPipelineFactory.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class ConfiguredSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory +{ + private readonly IServiceProvider services; + private readonly MeetingAssistantOptions options; + + public ConfiguredSpeechRecognitionPipelineFactory( + IServiceProvider services, + IOptions options) + { + this.services = services; + this.options = options.Value; + } + + public ISpeechRecognitionPipeline Create() + { + return options.Recording.TranscriptionProvider switch + { + "funasr" => ActivatorUtilities.CreateInstance(services), + "whisper-local" => ActivatorUtilities.CreateInstance(services), + "azure-speech" => ActivatorUtilities.CreateInstance(services), + _ => throw new InvalidOperationException( + $"Unsupported speech recognition pipeline '{options.Recording.TranscriptionProvider}'.") + }; + } +} diff --git a/MeetingAssistant/Transcription/FunAsrDockerBackendLifecycle.cs b/MeetingAssistant/Transcription/FunAsrDockerBackendLifecycle.cs new file mode 100644 index 0000000..50e1d83 --- /dev/null +++ b/MeetingAssistant/Transcription/FunAsrDockerBackendLifecycle.cs @@ -0,0 +1,190 @@ +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public interface IFunAsrBackendLifecycle +{ + Task EnsureStartedAsync(CancellationToken cancellationToken); +} + +public sealed class FunAsrDockerBackendLifecycle : IFunAsrBackendLifecycle, IAsyncDisposable +{ + private readonly ICommandRunner commandRunner; + private readonly IFunAsrBackendReadinessProbe readinessProbe; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + private readonly SemaphoreSlim gate = new(1, 1); + private bool containerStarted; + private bool started; + private bool disposed; + + public FunAsrDockerBackendLifecycle( + ICommandRunner commandRunner, + IFunAsrBackendReadinessProbe readinessProbe, + IOptions options, + ILogger logger) + { + this.commandRunner = commandRunner; + this.readinessProbe = readinessProbe; + this.options = options.Value; + this.logger = logger; + } + + public async Task EnsureStartedAsync(CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(disposed, this); + var backend = options.FunAsr.Backend; + if (!backend.Enabled) + { + return; + } + + await gate.WaitAsync(cancellationToken); + try + { + if (started) + { + return; + } + + PrepareModelsFolder(backend); + await RunRequiredAsync(backend.DockerCommand, ["pull", backend.Image], cancellationToken); + await commandRunner.RunAsync( + backend.DockerCommand, + ["rm", "-f", backend.ContainerName], + cancellationToken); + await RunRequiredAsync(backend.DockerCommand, BuildDockerRunArguments(), cancellationToken); + containerStarted = true; + await readinessProbe.WaitUntilReadyAsync( + new Uri(options.FunAsr.Endpoint), + backend.StartupTimeout, + cancellationToken); + + started = true; + logger.LogInformation( + "Started managed FunASR backend container {ContainerName}; endpoint {Endpoint} is ready", + backend.ContainerName, + options.FunAsr.Endpoint); + } + finally + { + gate.Release(); + } + } + + public async ValueTask DisposeAsync() + { + if (disposed) + { + return; + } + + var backend = options.FunAsr.Backend; + if (!backend.Enabled || !backend.StopOnDispose) + { + disposed = true; + gate.Dispose(); + return; + } + + await gate.WaitAsync(); + try + { + if (containerStarted) + { + await commandRunner.RunAsync( + backend.DockerCommand, + ["stop", backend.ContainerName], + CancellationToken.None); + containerStarted = false; + started = false; + } + } + finally + { + disposed = true; + gate.Release(); + gate.Dispose(); + } + } + + private async Task RunRequiredAsync( + string fileName, + IReadOnlyList arguments, + CancellationToken cancellationToken) + { + var backend = options.FunAsr.Backend; + using var timeoutSource = backend.CommandTimeout > TimeSpan.Zero + ? new CancellationTokenSource(backend.CommandTimeout) + : null; + using var linkedSource = timeoutSource is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token); + var effectiveToken = linkedSource?.Token ?? cancellationToken; + CommandResult result; + try + { + result = await commandRunner.RunAsync(fileName, arguments, effectiveToken); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true) + { + throw new TimeoutException( + $"Command '{fileName} {string.Join(' ', arguments)}' timed out after {backend.CommandTimeout}."); + } + + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"Command '{fileName} {string.Join(' ', arguments)}' failed with exit code {result.ExitCode}: {result.StandardError}"); + } + } + + private string[] BuildDockerRunArguments() + { + var backend = options.FunAsr.Backend; + List arguments = + [ + "run", + "--rm", + "-d", + ]; + + if (backend.Privileged) + { + arguments.Add("--privileged=true"); + } + + arguments.AddRange( + [ + "--name", + backend.ContainerName, + "-p", + $"{GetEndpointPort()}:{backend.ContainerPort}", + "-v", + $"{VaultPath.Resolve(backend.ModelsFolder)}:/workspace/models", + backend.Image, + "bash", + "-lc", + backend.ServerCommand + ]); + + return [.. arguments]; + } + + private static void PrepareModelsFolder(FunAsrBackendOptions backend) + { + var modelsFolder = VaultPath.Resolve(backend.ModelsFolder); + Directory.CreateDirectory(modelsFolder); + + var hotwordsPath = Path.Combine(modelsFolder, "hotwords.txt"); + if (!File.Exists(hotwordsPath)) + { + File.WriteAllText(hotwordsPath, ""); + } + } + + private int GetEndpointPort() + { + return new Uri(options.FunAsr.Endpoint).Port; + } +} diff --git a/MeetingAssistant/Transcription/FunAsrMessageParser.cs b/MeetingAssistant/Transcription/FunAsrMessageParser.cs new file mode 100644 index 0000000..c096818 --- /dev/null +++ b/MeetingAssistant/Transcription/FunAsrMessageParser.cs @@ -0,0 +1,182 @@ +using System.Text.Json; + +namespace MeetingAssistant.Transcription; + +public sealed class FunAsrMessageParser +{ + private readonly FunAsrOptions options; + + public FunAsrMessageParser(FunAsrOptions options) + { + this.options = options; + } + + public FunAsrParsedMessage Parse(string message) + { + using var document = JsonDocument.Parse(message); + var root = document.RootElement; + var mode = GetString(root, "mode"); + var isFinal = GetBoolean(root, "is_final"); + var shouldEmit = options.EmitOnlinePartials || !IsOnlineMode(mode); + var segments = shouldEmit ? ReadSegments(root) : []; + + return new FunAsrParsedMessage(IsTerminal: isFinal && !IsOnlineMode(mode), segments); + } + + private static List ReadSegments(JsonElement root) + { + var sentenceSegments = ReadSentenceSegments(root, "sentence_info"); + if (sentenceSegments.Count > 0) + { + return sentenceSegments; + } + + var stampSegments = ReadSentenceSegments(root, "stamp_sents"); + if (stampSegments.Count > 0) + { + return stampSegments; + } + + var text = GetString(root, "text").Trim(); + if (string.IsNullOrWhiteSpace(text)) + { + return []; + } + + var (start, end) = ReadTimestampRange(root); + return [new TranscriptionSegment(start, end, ReadSpeaker(root), text)]; + } + + private static List ReadSentenceSegments(JsonElement root, string propertyName) + { + if (!root.TryGetProperty(propertyName, out var sentences) || sentences.ValueKind != JsonValueKind.Array) + { + return []; + } + + var segments = new List(); + foreach (var sentence in sentences.EnumerateArray()) + { + var text = (GetString(sentence, "text") + GetString(sentence, "text_seg") + GetString(sentence, "punc")).Trim(); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + segments.Add(new TranscriptionSegment( + TimeSpan.FromMilliseconds(GetDouble(sentence, "start")), + TimeSpan.FromMilliseconds(GetDouble(sentence, "end")), + ReadSpeaker(sentence), + text)); + } + + return segments; + } + + private static (TimeSpan Start, TimeSpan End) ReadTimestampRange(JsonElement root) + { + if (!root.TryGetProperty("timestamp", out var timestamp)) + { + return (TimeSpan.Zero, TimeSpan.Zero); + } + + if (timestamp.ValueKind == JsonValueKind.String) + { + var value = timestamp.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return (TimeSpan.Zero, TimeSpan.Zero); + } + + using var document = JsonDocument.Parse(value); + return ReadTimestampArray(document.RootElement); + } + + return ReadTimestampArray(timestamp); + } + + private static (TimeSpan Start, TimeSpan End) ReadTimestampArray(JsonElement timestamp) + { + if (timestamp.ValueKind != JsonValueKind.Array) + { + return (TimeSpan.Zero, TimeSpan.Zero); + } + + double? start = null; + double? end = null; + foreach (var item in timestamp.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Array) + { + continue; + } + + var values = item.EnumerateArray().ToArray(); + if (values.Length < 2) + { + continue; + } + + start ??= ReadNumber(values[0]); + end = ReadNumber(values[1]); + } + + return ( + TimeSpan.FromMilliseconds(start ?? 0), + TimeSpan.FromMilliseconds(end ?? start ?? 0)); + } + + private static string ReadSpeaker(JsonElement element) + { + var speaker = GetString(element, "spk_name"); + if (!string.IsNullOrWhiteSpace(speaker)) + { + return speaker; + } + + speaker = GetString(element, "speaker"); + if (!string.IsNullOrWhiteSpace(speaker)) + { + return speaker; + } + + if (element.TryGetProperty("spk", out var spk)) + { + return $"Speaker {spk}"; + } + + return "Unknown"; + } + + private static bool IsOnlineMode(string mode) + { + return mode.Equals("online", StringComparison.OrdinalIgnoreCase) + || mode.Equals("2pass-online", StringComparison.OrdinalIgnoreCase); + } + + private static string GetString(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String + ? property.GetString() ?? "" + : ""; + } + + private static bool GetBoolean(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind is JsonValueKind.True or JsonValueKind.False + && property.GetBoolean(); + } + + private static double GetDouble(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) ? ReadNumber(property) : 0; + } + + private static double ReadNumber(JsonElement element) + { + return element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0; + } +} + +public sealed record FunAsrParsedMessage(bool IsTerminal, IReadOnlyList Segments); diff --git a/MeetingAssistant/Transcription/FunAsrSpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/FunAsrSpeechRecognitionPipeline.cs new file mode 100644 index 0000000..f06ec0b --- /dev/null +++ b/MeetingAssistant/Transcription/FunAsrSpeechRecognitionPipeline.cs @@ -0,0 +1,31 @@ +namespace MeetingAssistant.Transcription; + +public sealed class FunAsrSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline +{ + private readonly IFunAsrBackendLifecycle backendLifecycle; + private readonly FunAsrTranscriptFinalizer transcriptFinalizer; + + public FunAsrSpeechRecognitionPipeline( + FunAsrStreamingTranscriptionProvider transcriptionProvider, + IFunAsrBackendLifecycle backendLifecycle, + FunAsrTranscriptFinalizer transcriptFinalizer) + : base(transcriptionProvider) + { + this.backendLifecycle = backendLifecycle; + this.transcriptFinalizer = transcriptFinalizer; + } + + public override Task WaitUntilReadyAsync(CancellationToken cancellationToken) + { + return backendLifecycle.EnsureStartedAsync(cancellationToken); + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, cancellationToken); + } +} diff --git a/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs new file mode 100644 index 0000000..1c172b5 --- /dev/null +++ b/MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs @@ -0,0 +1,184 @@ +using System.Text.Json; +using System.Threading.Channels; +using MeetingAssistant.Recording; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscriptionProvider, IAsyncDisposable +{ + private readonly IFunAsrWebSocketConnectionFactory connectionFactory; + private readonly IFunAsrBackendLifecycle backendLifecycle; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public FunAsrStreamingTranscriptionProvider( + IFunAsrWebSocketConnectionFactory connectionFactory, + IFunAsrBackendLifecycle backendLifecycle, + IOptions options, + ILogger logger) + { + this.connectionFactory = connectionFactory; + this.backendLifecycle = backendLifecycle; + this.options = options.Value; + this.logger = logger; + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var segments = Channel.CreateUnbounded(); + var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None); + + await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) + { + yield return segment; + } + + await session; + } + + private async Task RunSessionAsync( + IAsyncEnumerable audio, + ChannelWriter segments, + CancellationToken cancellationToken) + { + try + { + await backendLifecycle.EnsureStartedAsync(cancellationToken); + await using var enumerator = audio.GetAsyncEnumerator(cancellationToken); + if (!await enumerator.MoveNextAsync()) + { + segments.TryComplete(); + return; + } + + var firstChunk = enumerator.Current; + var endpoint = new Uri(options.FunAsr.Endpoint); + await using var connection = await connectionFactory.ConnectAsync(endpoint, cancellationToken); + var parser = new FunAsrMessageParser(options.FunAsr); + var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken); + + await SendStartAsync(connection, firstChunk, cancellationToken); + await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken); + + while (await enumerator.MoveNextAsync()) + { + await connection.SendBinaryAsync(enumerator.Current.Pcm, cancellationToken); + } + + await connection.SendTextAsync("""{"is_speaking":false}""", cancellationToken); + + try + { + await finalReceived.Task.WaitAsync(options.FunAsr.FinalResultTimeout, cancellationToken); + } + catch (TimeoutException) + { + logger.LogWarning( + "Timed out waiting {Timeout} for the final FunASR result from {Endpoint}", + options.FunAsr.FinalResultTimeout, + endpoint); + } + + await connection.CloseOutputAsync(cancellationToken); + await receiveTask; + segments.TryComplete(); + } + catch (Exception exception) + { + segments.TryComplete(exception); + } + } + + public async ValueTask DisposeAsync() + { + if (backendLifecycle is IAsyncDisposable disposable) + { + await disposable.DisposeAsync(); + } + } + + private async Task ReceiveAsync( + IFunAsrWebSocketConnection connection, + FunAsrMessageParser parser, + ChannelWriter segments, + TaskCompletionSource finalReceived, + CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + var message = await connection.ReceiveTextAsync(cancellationToken); + if (message is null) + { + finalReceived.TrySetResult(); + return; + } + + var parsed = parser.Parse(message); + foreach (var segment in parsed.Segments) + { + await segments.WriteAsync(segment, cancellationToken); + logger.LogInformation( + "FunASR emitted segment at {Start} for {Speaker}: {Text}", + segment.Start, + segment.Speaker, + segment.Text); + } + + if (parsed.IsTerminal) + { + finalReceived.TrySetResult(); + return; + } + } + } + + private async Task SendStartAsync( + IFunAsrWebSocketConnection connection, + AudioChunk firstChunk, + CancellationToken cancellationToken) + { + var message = new Dictionary + { + ["mode"] = options.FunAsr.Mode, + ["chunk_size"] = NormalizeChunkSize(options.FunAsr.ChunkSize), + ["chunk_interval"] = options.FunAsr.ChunkInterval, + ["encoder_chunk_look_back"] = options.FunAsr.EncoderChunkLookBack, + ["decoder_chunk_look_back"] = options.FunAsr.DecoderChunkLookBack, + ["audio_fs"] = firstChunk.SampleRate, + ["wav_name"] = options.FunAsr.WavName, + ["wav_format"] = "pcm", + ["is_speaking"] = true, + ["hotwords"] = await LoadHotwordsAsync(cancellationToken), + ["itn"] = options.FunAsr.Itn + }; + + await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken); + } + + private async Task LoadHotwordsAsync(CancellationToken cancellationToken) + { + var path = VaultPath.Resolve(options.Vault.DictationWordsPath); + if (!File.Exists(path)) + { + return ""; + } + + var words = await File.ReadAllLinesAsync(path, cancellationToken); + var hotwords = words + .Select(line => line.Trim().TrimStart('-', '*').Trim()) + .Where(line => !string.IsNullOrWhiteSpace(line)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToDictionary(word => word, _ => 20); + + return hotwords.Count == 0 ? "" : JsonSerializer.Serialize(hotwords); + } + + private static int[] NormalizeChunkSize(int[] configuredChunkSize) + { + return configuredChunkSize.Length == 3 ? configuredChunkSize : [5, 10, 5]; + } +} diff --git a/MeetingAssistant/Transcription/FunAsrTranscriptFinalizer.cs b/MeetingAssistant/Transcription/FunAsrTranscriptFinalizer.cs new file mode 100644 index 0000000..813e14e --- /dev/null +++ b/MeetingAssistant/Transcription/FunAsrTranscriptFinalizer.cs @@ -0,0 +1,207 @@ +using System.Text.Json; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class FunAsrTranscriptFinalizer +{ + private const string JsonStart = "__MEETING_ASSISTANT_DIARIZATION_JSON_START__"; + private const string JsonEnd = "__MEETING_ASSISTANT_DIARIZATION_JSON_END__"; + + private readonly ICommandRunner commandRunner; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public FunAsrTranscriptFinalizer( + ICommandRunner commandRunner, + IOptions options, + ILogger logger) + { + this.commandRunner = commandRunner; + this.options = options.Value; + this.logger = logger; + } + + public async Task> FinalizeAsync( + string audioPath, + IReadOnlyList liveSegments, + CancellationToken cancellationToken) + { + var diarization = options.FunAsr.Diarization; + if (!diarization.Enabled) + { + return []; + } + + var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath)); + if (!File.Exists(fullAudioPath)) + { + throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath); + } + + var result = await RunDiarizationAsync(fullAudioPath, cancellationToken); + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"FunASR diarization failed with exit code {result.ExitCode}: {result.StandardError}"); + } + + var json = ExtractJson(result.StandardOutput); + var segments = ParseSegments(json); + logger.LogInformation("FunASR final diarization produced {SegmentCount} sentence segments", segments.Count); + return segments; + } + + private async Task RunDiarizationAsync( + string fullAudioPath, + CancellationToken cancellationToken) + { + var backend = options.FunAsr.Backend; + var modelsFolder = VaultPath.Resolve(backend.ModelsFolder); + Directory.CreateDirectory(modelsFolder); + using var timeoutSource = options.FunAsr.Diarization.CommandTimeout > TimeSpan.Zero + ? new CancellationTokenSource(options.FunAsr.Diarization.CommandTimeout) + : null; + using var linkedSource = timeoutSource is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token); + + try + { + return await commandRunner.RunAsync( + backend.DockerCommand, + BuildDockerArguments(fullAudioPath, modelsFolder), + linkedSource?.Token ?? cancellationToken); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true) + { + throw new TimeoutException( + $"FunASR diarization timed out after {options.FunAsr.Diarization.CommandTimeout}."); + } + } + + private string[] BuildDockerArguments(string fullAudioPath, string modelsFolder) + { + var backend = options.FunAsr.Backend; + return + [ + "run", + "--rm", + "-v", + $"{fullAudioPath}:/workspace/input.wav:ro", + "-v", + $"{modelsFolder}:/workspace/models", + backend.Image, + "bash", + "-lc", + BuildPythonCommand() + ]; + } + + private string BuildPythonCommand() + { + var diarization = options.FunAsr.Diarization; + var presetSpeakerCount = diarization.PresetSpeakerCount is int value + ? $", preset_spk_num={value}" + : ""; + return + "cat > /tmp/meeting_assistant_diarize.py <<'PY'\n" + + "import json\n" + + "from funasr import AutoModel\n" + + "model = AutoModel(\n" + + $" model={JsonSerializer.Serialize(diarization.AsrModel)},\n" + + $" vad_model={JsonSerializer.Serialize(diarization.VadModel)},\n" + + $" punc_model={JsonSerializer.Serialize(diarization.PunctuationModel)},\n" + + $" spk_model={JsonSerializer.Serialize(diarization.SpeakerModel)},\n" + + " device='cpu',\n" + + " disable_update=True,\n" + + ")\n" + + "res = model.generate(\n" + + " input='/workspace/input.wav',\n" + + $" batch_size_s={diarization.BatchSizeSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)},\n" + + $" batch_size_threshold_s={diarization.BatchSizeThresholdSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)}" + + presetSpeakerCount + + "\n)\n" + + $"print('{JsonStart}')\n" + + "print(json.dumps(res, ensure_ascii=False, default=str))\n" + + $"print('{JsonEnd}')\n" + + "PY\n" + + "MODELSCOPE_CACHE=/workspace/models/modelscope python /tmp/meeting_assistant_diarize.py"; + } + + private static string ExtractJson(string output) + { + var start = output.IndexOf(JsonStart, StringComparison.Ordinal); + if (start < 0) + { + throw new InvalidOperationException("FunASR diarization output did not contain the JSON start marker."); + } + + start += JsonStart.Length; + var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal); + if (end < 0) + { + throw new InvalidOperationException("FunASR diarization output did not contain the JSON end marker."); + } + + return output[start..end].Trim(); + } + + private static IReadOnlyList ParseSegments(string json) + { + using var document = JsonDocument.Parse(json); + var segments = new List(); + + foreach (var result in document.RootElement.EnumerateArray()) + { + if (!result.TryGetProperty("sentence_info", out var sentences) + || sentences.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var sentence in sentences.EnumerateArray()) + { + var text = ReadString(sentence, "text").Trim(); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + segments.Add(new TranscriptionSegment( + TimeSpan.FromMilliseconds(ReadDouble(sentence, "start")), + TimeSpan.FromMilliseconds(ReadDouble(sentence, "end")), + $"Speaker {ReadInt(sentence, "spk")}", + text)); + } + } + + return segments; + } + + private static string ReadString(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.String + ? property.GetString() ?? "" + : ""; + } + + private static double ReadDouble(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.Number + && property.TryGetDouble(out var value) + ? value + : 0; + } + + private static int ReadInt(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.Number + && property.TryGetInt32(out var value) + ? value + : 0; + } +} diff --git a/MeetingAssistant/Transcription/IFunAsrBackendReadinessProbe.cs b/MeetingAssistant/Transcription/IFunAsrBackendReadinessProbe.cs new file mode 100644 index 0000000..f02e440 --- /dev/null +++ b/MeetingAssistant/Transcription/IFunAsrBackendReadinessProbe.cs @@ -0,0 +1,84 @@ +using System.Net.WebSockets; + +namespace MeetingAssistant.Transcription; + +public interface IFunAsrBackendReadinessProbe +{ + Task WaitUntilReadyAsync(Uri endpoint, TimeSpan timeout, CancellationToken cancellationToken); +} + +public sealed class FunAsrWebSocketBackendReadinessProbe : IFunAsrBackendReadinessProbe +{ + private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(1); + + public async Task WaitUntilReadyAsync( + Uri endpoint, + TimeSpan timeout, + CancellationToken cancellationToken) + { + if (timeout <= TimeSpan.Zero) + { + return; + } + + using var timeoutSource = new CancellationTokenSource(timeout); + using var linkedSource = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutSource.Token); + + while (!linkedSource.IsCancellationRequested) + { + try + { + using var client = new ClientWebSocket(); + await client.ConnectAsync(endpoint, linkedSource.Token).ConfigureAwait(false); + await CloseQuietlyAsync(client, linkedSource.Token).ConfigureAwait(false); + return; + } + catch (OperationCanceledException) + { + break; + } + catch (Exception exception) when (IsTransientReadinessFailure(exception)) + { + try + { + await Task.Delay(RetryDelay, linkedSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + + throw new TimeoutException( + $"Timed out waiting {timeout} for managed FunASR backend WebSocket readiness at {endpoint}."); + } + + private static async Task CloseQuietlyAsync(ClientWebSocket client, CancellationToken cancellationToken) + { + if (client.State != WebSocketState.Open) + { + return; + } + + try + { + await client.CloseOutputAsync( + WebSocketCloseStatus.NormalClosure, + "readiness probe complete", + cancellationToken).ConfigureAwait(false); + } + catch (WebSocketException) + { + } + } + + private static bool IsTransientReadinessFailure(Exception exception) + { + return exception is WebSocketException + or HttpRequestException + or IOException; + } +} diff --git a/MeetingAssistant/Transcription/IFunAsrWebSocketConnection.cs b/MeetingAssistant/Transcription/IFunAsrWebSocketConnection.cs new file mode 100644 index 0000000..e737f02 --- /dev/null +++ b/MeetingAssistant/Transcription/IFunAsrWebSocketConnection.cs @@ -0,0 +1,17 @@ +namespace MeetingAssistant.Transcription; + +public interface IFunAsrWebSocketConnectionFactory +{ + Task ConnectAsync(Uri endpoint, CancellationToken cancellationToken); +} + +public interface IFunAsrWebSocketConnection : IAsyncDisposable +{ + Task SendTextAsync(string message, CancellationToken cancellationToken); + + Task SendBinaryAsync(ReadOnlyMemory message, CancellationToken cancellationToken); + + Task ReceiveTextAsync(CancellationToken cancellationToken); + + Task CloseOutputAsync(CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs new file mode 100644 index 0000000..7a716d1 --- /dev/null +++ b/MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs @@ -0,0 +1,31 @@ +using MeetingAssistant.Recording; + +namespace MeetingAssistant.Transcription; + +public interface ISpeechRecognitionPipeline : IAsyncDisposable +{ + Task InitializeAsync(CancellationToken cancellationToken); + + Task WaitUntilReadyAsync(CancellationToken cancellationToken); + + ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken); + + Task CompleteAsync(CancellationToken cancellationToken); + + IAsyncEnumerable ReadLiveTranscriptAsync(CancellationToken cancellationToken); + + Task> ReadFinishedTranscriptAsync( + string audioPath, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken); +} + +public interface ISpeechRecognitionPipelineFactory +{ + ISpeechRecognitionPipeline Create(); +} + +public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null) +{ + public static SpeechRecognitionPipelineOptions Default { get; } = new(); +} diff --git a/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs new file mode 100644 index 0000000..aac211e --- /dev/null +++ b/MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs @@ -0,0 +1,10 @@ +using MeetingAssistant.Recording; + +namespace MeetingAssistant.Transcription; + +public interface IStreamingTranscriptionProvider +{ + IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + CancellationToken cancellationToken); +} diff --git a/MeetingAssistant/Transcription/PyannoteTranscriptFinalizer.cs b/MeetingAssistant/Transcription/PyannoteTranscriptFinalizer.cs new file mode 100644 index 0000000..e7781f2 --- /dev/null +++ b/MeetingAssistant/Transcription/PyannoteTranscriptFinalizer.cs @@ -0,0 +1,420 @@ +using System.Text.Json; +using Microsoft.Extensions.Options; + +namespace MeetingAssistant.Transcription; + +public sealed class PyannoteTranscriptFinalizer +{ + private const string JsonStart = "__MEETING_ASSISTANT_PYANNOTE_JSON_START__"; + private const string JsonEnd = "__MEETING_ASSISTANT_PYANNOTE_JSON_END__"; + + private readonly ICommandRunner commandRunner; + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public PyannoteTranscriptFinalizer( + ICommandRunner commandRunner, + IOptions options, + ILogger logger) + { + this.commandRunner = commandRunner; + this.options = options.Value; + this.logger = logger; + } + + public async Task> FinalizeAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions pipelineOptions, + CancellationToken cancellationToken) + { + return await FinalizeAsync( + audioPath, + liveSegments, + options.WhisperLocal.Diarization, + pipelineOptions, + cancellationToken); + } + + public async Task> FinalizeAsync( + string audioPath, + IReadOnlyList liveSegments, + PyannoteDiarizationOptions diarization, + SpeechRecognitionPipelineOptions pipelineOptions, + CancellationToken cancellationToken) + { + if (!diarization.Enabled || liveSegments.Count == 0) + { + return []; + } + + var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath)); + if (!File.Exists(fullAudioPath)) + { + throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath); + } + + var token = ResolveToken(diarization); + if (string.IsNullOrWhiteSpace(token)) + { + logger.LogWarning( + "Pyannote diarization is enabled but no Hugging Face token is configured directly or via {TokenEnv}", + diarization.TokenEnv); + return []; + } + + var result = await RunDiarizationAsync(fullAudioPath, token, diarization, pipelineOptions, cancellationToken); + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"pyannote diarization failed with exit code {result.ExitCode}: {result.StandardError}"); + } + + var turns = ParseTurns(ExtractJson(result.StandardOutput)); + var segments = diarization.AlignmentMode == PyannoteAlignmentMode.PyannoteTurns + ? BuildTurnSegments(liveSegments, turns) + : AssignSpeakers(liveSegments, turns); + logger.LogInformation( + "pyannote diarization produced {TurnCount} speaker turns and built {SegmentCount} transcript segments using {AlignmentMode}", + turns.Count, + segments.Count, + diarization.AlignmentMode); + return segments; + } + + private async Task RunDiarizationAsync( + string fullAudioPath, + string token, + PyannoteDiarizationOptions diarization, + SpeechRecognitionPipelineOptions pipelineOptions, + CancellationToken cancellationToken) + { + var modelsFolder = VaultPath.Resolve(diarization.ModelsFolder); + Directory.CreateDirectory(modelsFolder); + using var timeoutSource = diarization.CommandTimeout > TimeSpan.Zero + ? new CancellationTokenSource(diarization.CommandTimeout) + : null; + using var linkedSource = timeoutSource is null + ? null + : CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token); + + try + { + if (diarization.BuildImage) + { + await EnsureDockerImageAsync(diarization, modelsFolder, linkedSource?.Token ?? cancellationToken); + } + + return await commandRunner.RunAsync( + diarization.DockerCommand, + BuildDockerArguments(diarization, fullAudioPath, modelsFolder, pipelineOptions), + linkedSource?.Token ?? cancellationToken, + new Dictionary { ["HF_TOKEN"] = token }); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true) + { + throw new TimeoutException( + $"pyannote diarization timed out after {diarization.CommandTimeout}."); + } + } + + private async Task EnsureDockerImageAsync( + PyannoteDiarizationOptions diarization, + string modelsFolder, + CancellationToken cancellationToken) + { + var inspectResult = await commandRunner.RunAsync( + diarization.DockerCommand, + ["image", "inspect", "--format", "{{.Id}}", diarization.Image], + cancellationToken); + if (inspectResult.ExitCode == 0) + { + return; + } + + var dockerfilePath = Path.Combine(modelsFolder, "docker", $"{SanitizeFileName(diarization.Image)}.Dockerfile"); + Directory.CreateDirectory(Path.GetDirectoryName(dockerfilePath)!); + await File.WriteAllTextAsync(dockerfilePath, BuildDockerfile(diarization), cancellationToken); + var result = await commandRunner.RunAsync( + diarization.DockerCommand, + ["build", "-t", diarization.Image, "-f", dockerfilePath, Path.GetDirectoryName(dockerfilePath)!], + cancellationToken); + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"pyannote Docker image build failed with exit code {result.ExitCode}: {result.StandardError}"); + } + } + + private static string BuildDockerfile(PyannoteDiarizationOptions diarization) + { + return + $"FROM {diarization.BaseImage}\n" + + "ENV DEBIAN_FRONTEND=noninteractive\n" + + "RUN apt-get update \\\n" + + " && apt-get install -y --no-install-recommends ffmpeg libsndfile1 \\\n" + + " && rm -rf /var/lib/apt/lists/*\n" + + "RUN python -m pip install --upgrade pip \\\n" + + " && python -m pip install pyannote.audio soundfile\n"; + } + + private string[] BuildDockerArguments( + PyannoteDiarizationOptions diarization, + string fullAudioPath, + string modelsFolder, + SpeechRecognitionPipelineOptions pipelineOptions) + { + return + [ + "run", + "--rm", + "-e", + "HF_TOKEN", + "-v", + $"{fullAudioPath}:/workspace/input.wav:ro", + "-v", + $"{modelsFolder}:/workspace/cache", + "-e", + "HF_HOME=/workspace/cache/huggingface", + "-e", + "XDG_CACHE_HOME=/workspace/cache", + "-e", + "PIP_CACHE_DIR=/workspace/cache/pip", + "-e", + "TORCH_HOME=/workspace/cache/torch", + diarization.Image, + "sh", + "-lc", + BuildPythonCommand(diarization, pipelineOptions) + ]; + } + + private static string BuildPythonCommand( + PyannoteDiarizationOptions diarization, + SpeechRecognitionPipelineOptions pipelineOptions) + { + var model = diarization.Model; + var annotationProperty = diarization.AnnotationSource == PyannoteAnnotationSource.ExclusiveSpeakerDiarization + ? "exclusive_speaker_diarization" + : "speaker_diarization"; + return + "cat > /tmp/meeting_assistant_pyannote.py <<'PY'\n" + + "import json\n" + + "import os\n" + + "import torch\n" + + "from pyannote.audio import Pipeline\n" + + $"pipeline = Pipeline.from_pretrained({JsonSerializer.Serialize(model)}, token=os.environ.get('HF_TOKEN'))\n" + + "pipeline.to(torch.device('cpu'))\n" + + BuildPipelineInvocation(pipelineOptions) + + $"diarization = getattr(result, {JsonSerializer.Serialize(annotationProperty)}, result)\n" + + "turns = []\n" + + "for turn, _, speaker in diarization.itertracks(yield_label=True):\n" + + " turns.append({'start': float(turn.start), 'end': float(turn.end), 'speaker': str(speaker)})\n" + + $"print('{JsonStart}')\n" + + "print(json.dumps(turns, ensure_ascii=False))\n" + + $"print('{JsonEnd}')\n" + + "PY\n" + + "python /tmp/meeting_assistant_pyannote.py"; + } + + private static string BuildPipelineInvocation(SpeechRecognitionPipelineOptions pipelineOptions) + { + return pipelineOptions.NumSpeakers is > 0 + ? $"result = pipeline('/workspace/input.wav', num_speakers={pipelineOptions.NumSpeakers.Value})\n" + : "result = pipeline('/workspace/input.wav')\n"; + } + + private static IReadOnlyList BuildTurnSegments( + IReadOnlyList liveSegments, + IReadOnlyList turns) + { + if (turns.Count == 0) + { + return []; + } + + var segments = new List(); + foreach (var turn in turns) + { + var text = BuildTurnText(liveSegments, turn); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + segments.Add(new TranscriptionSegment( + TimeSpan.FromSeconds(turn.Start), + TimeSpan.FromSeconds(turn.End), + turn.Speaker, + text)); + } + + return segments.Count == 0 + ? AssignSpeakers(liveSegments, turns) + : segments; + } + + private static string BuildTurnText(IReadOnlyList liveSegments, SpeakerTurn turn) + { + var parts = new List(); + foreach (var segment in liveSegments) + { + var segmentStart = segment.Start.TotalSeconds; + var segmentEnd = segment.End > segment.Start + ? segment.End.TotalSeconds + : segmentStart; + var overlapStart = Math.Max(segmentStart, turn.Start); + var overlapEnd = Math.Min(segmentEnd, turn.End); + if (overlapEnd <= overlapStart) + { + continue; + } + + var part = SliceTextByTime(segment.Text, segmentStart, segmentEnd, overlapStart, overlapEnd); + if (!string.IsNullOrWhiteSpace(part)) + { + parts.Add(part); + } + } + + return string.Join(' ', parts); + } + + private static string SliceTextByTime( + string text, + double segmentStart, + double segmentEnd, + double overlapStart, + double overlapEnd) + { + var words = text + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (words.Length == 0) + { + return ""; + } + + var duration = Math.Max(0.001, segmentEnd - segmentStart); + var startFraction = Math.Clamp((overlapStart - segmentStart) / duration, 0, 1); + var endFraction = Math.Clamp((overlapEnd - segmentStart) / duration, 0, 1); + var startIndex = Math.Clamp((int)Math.Floor(startFraction * words.Length), 0, words.Length - 1); + var endIndex = Math.Clamp((int)Math.Ceiling(endFraction * words.Length), startIndex + 1, words.Length); + return string.Join(' ', words[startIndex..endIndex]); + } + + private static IReadOnlyList AssignSpeakers( + IReadOnlyList liveSegments, + IReadOnlyList turns) + { + if (turns.Count == 0) + { + return []; + } + + return liveSegments + .Select(segment => + { + var speaker = FindBestSpeaker(segment, turns) ?? segment.Speaker; + return segment with { Speaker = speaker }; + }) + .ToList(); + } + + private static string? FindBestSpeaker(TranscriptionSegment segment, IReadOnlyList turns) + { + var segmentStart = segment.Start.TotalSeconds; + var segmentEnd = segment.End > segment.Start + ? segment.End.TotalSeconds + : segmentStart; + var bestOverlap = 0d; + string? bestSpeaker = null; + + foreach (var turn in turns) + { + var overlap = Math.Min(segmentEnd, turn.End) - Math.Max(segmentStart, turn.Start); + if (overlap > bestOverlap) + { + bestOverlap = overlap; + bestSpeaker = turn.Speaker; + } + } + + return bestSpeaker; + } + + private static string ExtractJson(string output) + { + var start = output.IndexOf(JsonStart, StringComparison.Ordinal); + if (start < 0) + { + throw new InvalidOperationException("pyannote output did not contain the JSON start marker."); + } + + start += JsonStart.Length; + var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal); + if (end < 0) + { + throw new InvalidOperationException("pyannote output did not contain the JSON end marker."); + } + + return output[start..end].Trim(); + } + + private static IReadOnlyList ParseTurns(string json) + { + using var document = JsonDocument.Parse(json); + var turns = new List(); + foreach (var item in document.RootElement.EnumerateArray()) + { + var speaker = ReadString(item, "speaker"); + if (string.IsNullOrWhiteSpace(speaker)) + { + continue; + } + + turns.Add(new SpeakerTurn( + ReadDouble(item, "start"), + ReadDouble(item, "end"), + speaker)); + } + + return turns; + } + + private static string ResolveToken(PyannoteDiarizationOptions options) + { + if (!string.IsNullOrWhiteSpace(options.Token)) + { + return options.Token; + } + + return string.IsNullOrWhiteSpace(options.TokenEnv) + ? "" + : Environment.GetEnvironmentVariable(options.TokenEnv) ?? ""; + } + + private static string ReadString(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.String + ? property.GetString() ?? "" + : ""; + } + + private static double ReadDouble(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.Number + && property.TryGetDouble(out var value) + ? value + : 0; + } + + private static string SanitizeFileName(string value) + { + var invalid = Path.GetInvalidFileNameChars(); + return string.Concat(value.Select(character => invalid.Contains(character) ? '_' : character)); + } + + private sealed record SpeakerTurn(double Start, double End, string Speaker); +} diff --git a/MeetingAssistant/Transcription/SpeechRecognitionPipelineHostedService.cs b/MeetingAssistant/Transcription/SpeechRecognitionPipelineHostedService.cs new file mode 100644 index 0000000..09657fd --- /dev/null +++ b/MeetingAssistant/Transcription/SpeechRecognitionPipelineHostedService.cs @@ -0,0 +1,77 @@ +namespace MeetingAssistant.Transcription; + +public sealed class SpeechRecognitionPipelineHostedService : IHostedService +{ + private readonly ISpeechRecognitionPipelineFactory pipelineFactory; + private readonly ILogger logger; + private CancellationTokenSource? startupCancellation; + private Task? startupTask; + private ISpeechRecognitionPipeline? startupPipeline; + + public SpeechRecognitionPipelineHostedService( + ISpeechRecognitionPipelineFactory pipelineFactory, + ILogger logger) + { + this.pipelineFactory = pipelineFactory; + this.logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + startupCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + startupPipeline = pipelineFactory.Create(); + startupTask = Task.Run( + () => WarmUpPipelineAsync(startupPipeline, startupCancellation.Token), + CancellationToken.None); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + var cancellation = startupCancellation; + var task = startupTask; + var pipeline = startupPipeline; + if (cancellation is null || task is null || pipeline is null) + { + return; + } + + startupCancellation = null; + startupTask = null; + startupPipeline = null; + await cancellation.CancelAsync(); + try + { + await task.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + } + finally + { + await pipeline.DisposeAsync(); + cancellation.Dispose(); + } + } + + private async Task WarmUpPipelineAsync( + ISpeechRecognitionPipeline pipeline, + CancellationToken cancellationToken) + { + try + { + await pipeline.InitializeAsync(cancellationToken); + await pipeline.WaitUntilReadyAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + logger.LogInformation("Speech recognition pipeline warm-up was cancelled during application shutdown"); + } + catch (Exception exception) + { + logger.LogError( + exception, + "Speech recognition pipeline warm-up failed; recording can still start and recognition will retry when audio is processed"); + } + } +} diff --git a/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs new file mode 100644 index 0000000..da8e12f --- /dev/null +++ b/MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs @@ -0,0 +1,114 @@ +using System.Threading.Channels; +using MeetingAssistant.Recording; + +namespace MeetingAssistant.Transcription; + +public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPipeline +{ + private readonly IStreamingTranscriptionProvider transcriptionProvider; + private readonly Channel audio = Channel.CreateUnbounded(); + private readonly Channel liveTranscript = Channel.CreateUnbounded(); + private readonly List liveSegments = []; + private Task? transcriptionTask; + private bool initialized; + + protected StreamingSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider) + { + this.transcriptionProvider = transcriptionProvider; + } + + public virtual Task InitializeAsync(CancellationToken cancellationToken) + { + if (initialized) + { + return Task.CompletedTask; + } + + initialized = true; + transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None); + return Task.CompletedTask; + } + + public virtual Task WaitUntilReadyAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken) + { + EnsureInitialized(); + return audio.Writer.WriteAsync(chunk, cancellationToken); + } + + public async Task CompleteAsync(CancellationToken cancellationToken) + { + EnsureInitialized(); + audio.Writer.TryComplete(); + if (transcriptionTask is not null) + { + await transcriptionTask.WaitAsync(cancellationToken); + } + } + + public async IAsyncEnumerable ReadLiveTranscriptAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + EnsureInitialized(); + await foreach (var segment in liveTranscript.Reader.ReadAllAsync(cancellationToken)) + { + yield return segment; + } + } + + public async Task> ReadFinishedTranscriptAsync( + string audioPath, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + EnsureInitialized(); + await CompleteAsync(cancellationToken); + var finalSegments = await BuildFinishedTranscriptAsync(audioPath, liveSegments, options, cancellationToken); + return finalSegments.Count == 0 ? liveSegments : finalSegments; + } + + public virtual ValueTask DisposeAsync() + { + audio.Writer.TryComplete(); + liveTranscript.Writer.TryComplete(); + return ValueTask.CompletedTask; + } + + protected abstract Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken); + + private async Task TranscribeAsync(CancellationToken cancellationToken) + { + try + { + await foreach (var segment in transcriptionProvider.TranscribeAsync( + audio.Reader.ReadAllAsync(cancellationToken), + cancellationToken)) + { + liveSegments.Add(segment); + await liveTranscript.Writer.WriteAsync(segment, cancellationToken); + } + + liveTranscript.Writer.TryComplete(); + } + catch (Exception exception) + { + liveTranscript.Writer.TryComplete(exception); + } + } + + private void EnsureInitialized() + { + if (!initialized) + { + throw new InvalidOperationException("Speech recognition pipeline must be initialized before use."); + } + } +} diff --git a/MeetingAssistant/Transcription/TranscriptionSegment.cs b/MeetingAssistant/Transcription/TranscriptionSegment.cs new file mode 100644 index 0000000..73ce93d --- /dev/null +++ b/MeetingAssistant/Transcription/TranscriptionSegment.cs @@ -0,0 +1,3 @@ +namespace MeetingAssistant.Transcription; + +public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text); diff --git a/MeetingAssistant/Transcription/WhisperLocalSpeechRecognitionPipeline.cs b/MeetingAssistant/Transcription/WhisperLocalSpeechRecognitionPipeline.cs new file mode 100644 index 0000000..5538861 --- /dev/null +++ b/MeetingAssistant/Transcription/WhisperLocalSpeechRecognitionPipeline.cs @@ -0,0 +1,23 @@ +namespace MeetingAssistant.Transcription; + +public sealed class WhisperLocalSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline +{ + private readonly PyannoteTranscriptFinalizer transcriptFinalizer; + + public WhisperLocalSpeechRecognitionPipeline( + WhisperLocalStreamingTranscriptionProvider transcriptionProvider, + PyannoteTranscriptFinalizer transcriptFinalizer) + : base(transcriptionProvider) + { + this.transcriptFinalizer = transcriptFinalizer; + } + + protected override Task> BuildFinishedTranscriptAsync( + string audioPath, + IReadOnlyList liveSegments, + SpeechRecognitionPipelineOptions options, + CancellationToken cancellationToken) + { + return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, options, cancellationToken); + } +} diff --git a/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs new file mode 100644 index 0000000..be0e46f --- /dev/null +++ b/MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs @@ -0,0 +1,168 @@ +using MeetingAssistant.Recording; +using Microsoft.Extensions.Options; +using NAudio.Wave; +using Whisper.net; + +namespace MeetingAssistant.Transcription; + +public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTranscriptionProvider +{ + private readonly MeetingAssistantOptions options; + private readonly ILogger logger; + + public WhisperLocalStreamingTranscriptionProvider( + IOptions options, + ILogger logger) + { + this.options = options.Value; + this.logger = logger; + } + + public async IAsyncEnumerable TranscribeAsync( + IAsyncEnumerable audio, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath)); + if (!File.Exists(modelPath)) + { + throw new FileNotFoundException($"Whisper model was not found at '{modelPath}'.", modelPath); + } + + using var factory = WhisperFactory.FromPath(modelPath); + using var processor = factory.CreateBuilder() + .WithLanguage(options.WhisperLocal.Language) + .Build(); + + var window = new List(); + var windowDuration = TimeSpan.Zero; + var windowTarget = TimeSpan.FromSeconds(Math.Max(1, options.WhisperLocal.WindowSeconds)); + var offset = TimeSpan.Zero; + + logger.LogInformation( + "Starting local Whisper streaming transcription with model {ModelPath} and {WindowSeconds}s windows", + modelPath, + windowTarget.TotalSeconds); + + await foreach (var chunk in audio.WithCancellation(cancellationToken)) + { + window.Add(chunk); + windowDuration += chunk.Duration; + + if (windowDuration >= windowTarget) + { + var segments = 0; + await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken)) + { + segments++; + logger.LogInformation( + "Whisper emitted segment at {Start}: {Text}", + segment.Start, + segment.Text); + yield return segment; + } + + logger.LogInformation( + "Processed Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)", + offset, + windowDuration, + segments); + offset += windowDuration; + window.Clear(); + windowDuration = TimeSpan.Zero; + } + } + + if (window.Count > 0) + { + var segments = 0; + await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken)) + { + segments++; + logger.LogInformation( + "Whisper emitted segment at {Start}: {Text}", + segment.Start, + segment.Text); + yield return segment; + } + + logger.LogInformation( + "Processed final Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)", + offset, + windowDuration, + segments); + } + } + + private async IAsyncEnumerable ProcessWindow( + WhisperProcessor processor, + IReadOnlyList chunks, + TimeSpan offset, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var stream = CreateWaveStream(chunks); + + await foreach (var result in processor.ProcessAsync(stream, cancellationToken)) + { + var text = result.Text?.Trim(); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + if (IsControlMarker(text)) + { + logger.LogInformation("Whisper emitted control marker {Marker}; skipping transcript output", text); + continue; + } + + yield return new TranscriptionSegment(offset + result.Start, offset + result.End, "Unknown", text); + } + } + + private static bool IsControlMarker(string text) + { + return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]"; + } + + private MemoryStream CreateWaveStream(IReadOnlyList chunks) + { + var first = chunks[0]; + var stream = new MemoryStream(); + var pcm = new MemoryStream(); + + foreach (var chunk in chunks) + { + if (chunk.SampleRate != first.SampleRate || chunk.Channels != first.Channels) + { + logger.LogWarning("Dropping audio chunk with mismatched format while creating Whisper window"); + continue; + } + + pcm.Write(chunk.Pcm, 0, chunk.Pcm.Length); + } + + WriteWaveHeader(stream, first.SampleRate, first.Channels, (int)pcm.Length); + pcm.Position = 0; + pcm.CopyTo(stream); + stream.Position = 0; + return stream; + } + + private static void WriteWaveHeader(Stream stream, int sampleRate, int channels, int dataLength) + { + using var writer = new BinaryWriter(stream, System.Text.Encoding.ASCII, leaveOpen: true); + writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF")); + writer.Write(36 + dataLength); + writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE")); + writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt ")); + writer.Write(16); + writer.Write((short)1); + writer.Write((short)channels); + writer.Write(sampleRate); + writer.Write(sampleRate * channels * sizeof(short)); + writer.Write((short)(channels * sizeof(short))); + writer.Write((short)16); + writer.Write(System.Text.Encoding.ASCII.GetBytes("data")); + writer.Write(dataLength); + } +} diff --git a/MeetingAssistant/VaultPath.cs b/MeetingAssistant/VaultPath.cs new file mode 100644 index 0000000..818f06f --- /dev/null +++ b/MeetingAssistant/VaultPath.cs @@ -0,0 +1,17 @@ +namespace MeetingAssistant; + +public static class VaultPath +{ + public static string Resolve(string path) + { + var expanded = Environment.ExpandEnvironmentVariables(path); + if (expanded.StartsWith("~", StringComparison.Ordinal)) + { + expanded = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + expanded[1..].TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + } + + return Path.GetFullPath(expanded); + } +} diff --git a/MeetingAssistant/appsettings.Development.json b/MeetingAssistant/appsettings.Development.json index 0c208ae..c1d902d 100644 --- a/MeetingAssistant/appsettings.Development.json +++ b/MeetingAssistant/appsettings.Development.json @@ -1,4 +1,9 @@ { + "MeetingAssistant": { + "WhisperLocal": { + "WindowSeconds": 3 + } + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/MeetingAssistant/appsettings.json b/MeetingAssistant/appsettings.json index 10f68b8..ebad478 100644 --- a/MeetingAssistant/appsettings.json +++ b/MeetingAssistant/appsettings.json @@ -1,4 +1,102 @@ { + "MeetingAssistant": { + "Hotkey": { + "Toggle": "Ctrl+Alt+M" + }, + "Vault": { + "TranscriptsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Transcripts", + "MeetingNotesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Notes", + "AssistantContextFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Assistant Context", + "SummariesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Summaries", + "ProjectsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Projects", + "DictationWordsPath": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\dictation-words.md" + }, + "Recording": { + "TranscriptionProvider": "funasr", + "SampleRate": 16000, + "Channels": 1, + "StopProcessingTimeout": "00:10:00", + "TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings" + }, + "FunAsr": { + "Endpoint": "ws://127.0.0.1:10095", + "Mode": "2pass", + "ChunkSize": [5, 10, 5], + "ChunkInterval": 10, + "EncoderChunkLookBack": 4, + "DecoderChunkLookBack": 1, + "Itn": true, + "EmitOnlinePartials": false, + "WavName": "meeting", + "FinalResultTimeout": "00:00:10", + "Backend": { + "Enabled": true, + "DockerCommand": "docker", + "Image": "registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.13", + "ContainerName": "meeting-assistant-funasr", + "ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\FunASR\\models", + "ContainerPort": 10095, + "Privileged": true, + "CommandTimeout": "00:15:00", + "StartupTimeout": "00:10:00", + "StopOnDispose": true, + "ServerCommand": "cd /workspace/FunASR/runtime && bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --itn-dir thuduj12/fst_itn_zh --certfile 0 --hotword /workspace/models/hotwords.txt && tail -f /dev/null" + }, + "Diarization": { + "Enabled": true, + "AsrModel": "paraformer-zh", + "VadModel": "fsmn-vad", + "PunctuationModel": "ct-punc", + "SpeakerModel": "cam++", + "BatchSizeSeconds": 300, + "BatchSizeThresholdSeconds": 60, + "CommandTimeout": "00:30:00" + } + }, + "WhisperLocal": { + "ModelPath": "models\\ggml-base.bin", + "Language": "auto", + "WindowSeconds": 15, + "Diarization": { + "Enabled": true, + "DockerCommand": "docker", + "BaseImage": "python:3.11-slim", + "Image": "meeting-assistant-pyannote:local", + "BuildImage": true, + "ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Pyannote\\models", + "Model": "pyannote/speaker-diarization-3.1", + "AnnotationSource": "SpeakerDiarization", + "AlignmentMode": "PyannoteTurns", + "TokenEnv": "HF_TOKEN", + "CommandTimeout": "00:30:00" + } + }, + "AzureSpeech": { + "Endpoint": "https://germanywestcentral.api.cognitive.microsoft.com/", + "Region": "germanywestcentral", + "Language": "en-US", + "KeyEnv": "AZURE_SPEECH_KEY", + "RecognitionStopTimeout": "00:00:10", + "DiarizeIntermediateResults": true + }, + "Agent": { + "Endpoint": "https://litellm.schweigert.cloud", + "KeyEnv": "LITELLM_API_KEY", + "Model": "copilot-gpt-5.5", + "EnableThinking": true, + "ReasoningEffort": "Medium", + "ReconnectionAttempts": 2, + "ReconnectionDelay": "00:00:02", + "ContextWindowTokens": 128000, + "MaxOutputTokens": 8192, + "EnableCompaction": true, + "CompactionRemainingRatio": 0.10, + "ResponsesCompactPath": "responses/compact" + }, + "Api": { + "PublicBaseUrl": "http://localhost:5090" + } + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/README.md b/README.md index 6d64238..579e2f7 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,17 @@ Its core purpose is to work with any kind of meeting, including in-person meetin The application is intended to: - create an Obsidian markdown note before transcription starts -- keep meeting metadata, user notes, detected context, and generated output in that note +- keep meeting metadata, user notes, detected context, and links to generated output in that note +- toggle recording/transcription mode through a configurable global hotkey +- capture microphone input and computer output into one transcription stream - transcribe meetings with speaker attribution +- use a configurable speech recognition pipeline, with local Whisper as the first version 1 fallback +- use FunASR or local Whisper plus pyannote as configurable speaker-attribution paths +- write transcript markdown files into the configured vault folder - generate summaries, decisions, and next steps - build and maintain a project knowledge base from meetings and future context sources - use Microsoft Agent Framework-based agents to reason over meeting and project context -- give agents tools for project lookup, keyword lookup, retrieval-augmented generation, and controlled project file updates when the user or project context allows it +- give agents tools for project lookup, keyword lookup, retrieval-augmented generation, and project file updates in existing projects ## Repository Status @@ -34,12 +39,184 @@ Run the server: dotnet run --project MeetingAssistant ``` -The initial health endpoint is: +The initial endpoints are: ```text GET /health +GET /recording/status +POST /recording/toggle +POST /recording/start +POST /recording/stop +POST /asr/transcribe-file +POST /asr/diarize-file +POST /meetings/current/summary/run +POST /meetings/summary/retry +GET /meetings/summary/retry?summaryPath=... ``` +## Configuration + +Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` section: + +```json +{ + "MeetingAssistant": { + "Hotkey": { + "Toggle": "Ctrl+Alt+M" + }, + "Vault": { + "TranscriptsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Transcripts", + "MeetingNotesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Notes", + "AssistantContextFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Assistant Context", + "SummariesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Summaries", + "ProjectsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Projects", + "DictationWordsPath": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\dictation-words.md" + }, + "Recording": { + "TranscriptionProvider": "funasr", + "SampleRate": 16000, + "Channels": 1, + "StopProcessingTimeout": "00:10:00", + "TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings" + }, + "FunAsr": { + "Endpoint": "ws://127.0.0.1:10095", + "Mode": "2pass", + "ChunkSize": [5, 10, 5], + "ChunkInterval": 10, + "EncoderChunkLookBack": 4, + "DecoderChunkLookBack": 1, + "Itn": true, + "EmitOnlinePartials": false, + "WavName": "meeting", + "FinalResultTimeout": "00:00:10", + "Backend": { + "Enabled": true, + "DockerCommand": "docker", + "Image": "registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.13", + "ContainerName": "meeting-assistant-funasr", + "ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\FunASR\\models", + "ContainerPort": 10095, + "Privileged": true, + "CommandTimeout": "00:15:00", + "StartupTimeout": "00:10:00", + "StopOnDispose": true, + "ServerCommand": "cd /workspace/FunASR/runtime && bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --itn-dir thuduj12/fst_itn_zh --certfile 0 --hotword /workspace/models/hotwords.txt && tail -f /dev/null" + }, + "Diarization": { + "Enabled": true, + "AsrModel": "paraformer-zh", + "VadModel": "fsmn-vad", + "PunctuationModel": "ct-punc", + "SpeakerModel": "cam++", + "BatchSizeSeconds": 300, + "BatchSizeThresholdSeconds": 60, + "CommandTimeout": "00:30:00" + } + }, + "WhisperLocal": { + "ModelPath": "models\\ggml-base.bin", + "Language": "auto", + "WindowSeconds": 15, + "Diarization": { + "Enabled": true, + "DockerCommand": "docker", + "BaseImage": "python:3.11-slim", + "Image": "meeting-assistant-pyannote:local", + "BuildImage": true, + "ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Pyannote\\models", + "Model": "pyannote/speaker-diarization-3.1", + "AnnotationSource": "SpeakerDiarization", + "AlignmentMode": "PyannoteTurns", + "TokenEnv": "HF_TOKEN", + "CommandTimeout": "00:30:00" + } + }, + "AzureSpeech": { + "Endpoint": "https://germanywestcentral.api.cognitive.microsoft.com/", + "Region": "germanywestcentral", + "Language": "en-US", + "KeyEnv": "AZURE_SPEECH_KEY", + "RecognitionStopTimeout": "00:00:10", + "DiarizeIntermediateResults": true + }, + "Agent": { + "Endpoint": "https://litellm.schweigert.cloud", + "KeyEnv": "LITELLM_API_KEY", + "Model": "copilot-gpt-5.5", + "EnableThinking": true, + "ReasoningEffort": "Medium", + "ReconnectionAttempts": 2, + "ReconnectionDelay": "00:00:02", + "ContextWindowTokens": 128000, + "MaxOutputTokens": 8192, + "EnableCompaction": true, + "CompactionRemainingRatio": 0.10, + "ResponsesCompactPath": "responses/compact" + }, + "Api": { + "PublicBaseUrl": "http://localhost:5090" + } + } +} +``` + +`funasr` streams 16 kHz PCM audio to the configured FunASR WebSocket endpoint. When `FunAsr:Backend:Enabled` is true, Meeting Assistant manages a local Docker-backed FunASR runtime: on application startup it begins a non-blocking warm-up that pulls the configured online streaming image, prepares the mounted model and hotword folder, replaces any stale container with the configured name, starts the two-pass WebSocket server mapped to the endpoint port, keeps the container alive after the FunASR startup script backgrounds the server process, and stops the container when the app shuts down. Recording can start while the backend is still warming up; captured audio is queued and then streamed once the WebSocket backend is healthy. Docker commands are bounded by `CommandTimeout`, and backend WebSocket readiness is bounded by `StartupTimeout`. FunASR response fields such as `spk_name`, `spk`, and sentence-level speaker metadata are preserved in transcript segments; missing speaker fields are written as `Unknown`. + +During recording, Meeting Assistant writes the mixed PCM stream to a temporary WAV under `TemporaryRecordingsFolder` only so the final diarization pass has audio to process. After capture stops and the streaming ASR provider drains the already-captured audio, FunASR Python `AutoModel` can run with VAD, punctuation, and CAMPPlus (`spk_model="cam++"`) over that temporary WAV. If sentence-level speaker labels are returned, the transcript markdown is rewritten with the final speaker-attributed segments. If final diarization is disabled or returns no segments, the live streaming transcript is kept. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. + +`whisper-local` uses Whisper.NET with a local ggml model and remains available by setting `Recording:TranscriptionProvider` to `whisper-local`. The model file is not committed to this repository; place it at the configured `ModelPath` before using live transcription. When `WhisperLocal:Diarization:Enabled` is true, the final post-processing pass runs pyannote in Docker over the temporary WAV and reads the Hugging Face token from `WhisperLocal:Diarization:Token` or `TokenEnv` (`HF_TOKEN` by default). `AnnotationSource` selects pyannote's `speaker_diarization` or `exclusive_speaker_diarization` output. `AlignmentMode` selects whether Meeting Assistant assigns the best-overlap pyannote speaker label to each Whisper segment or rebuilds the finished transcript as pyannote speaker-turn segments with matching Whisper text. When `BuildImage` is true, Meeting Assistant builds the configured local pyannote Docker image if Docker does not already have it, so Python packages are installed once instead of during every diarization call. The configured `ModelsFolder` is mounted as the persistent pyannote model cache and stores Hugging Face, torch, and pip cache artifacts so model downloads are reused across runs. If pyannote is disabled, has no token, or returns no turns, the live Whisper transcript is kept. + +`azure-speech` streams captured PCM audio to Azure AI Speech using the Speech SDK `ConversationTranscriber`. Configure `AzureSpeech:Endpoint`, `Region`, `Language`, and either `Key` or `KeyEnv`; this repository uses `AZURE_SPEECH_KEY` by default and does not store the key in appsettings. `DiarizeIntermediateResults` enables diarization for intermediate and final conversation transcription results. Azure returns generic speaker IDs such as `Guest-1` and `Guest-2`, which Meeting Assistant writes directly into live transcript segments. `RecognitionStopTimeout` bounds SDK shutdown after the captured audio stream has been closed, which keeps short diagnostic runs from waiting indefinitely for final service events. Since Azure Speech emits diarized live transcript segments, the Azure pipeline keeps those segments as the finished transcript instead of running pyannote. + +ASR backends are exposed downstream as speech recognition pipelines. A pipeline can initialize, wait for readiness, accept audio chunks, emit live transcript segments, and produce a finished transcript after capture completes. The configured implementation is selected from `Recording:TranscriptionProvider` and registered through DI so recording and diagnostic endpoints do not need to know whether the backend uses FunASR, Whisper.NET, Azure Speech, pyannote, or a later provider. + +`POST /asr/transcribe-file` is a local diagnostic endpoint for the configured ASR backend. Send `{ "path": "C:\\path\\to\\sample.wav" }` with a 16-bit PCM WAV file to stream it through the configured speech recognition pipeline and return the emitted live transcript segments. + +For real meeting recordings, Meeting Assistant derives the optional finished-transcript `numSpeakers` hint from the meeting note `attendees` frontmatter. The hint is set to the attendee count only when at least two attendees are listed; empty and single-person attendee lists are ignored. For pyannote-backed finalization, that hint is passed to pyannote as `num_speakers`. + +`POST /asr/diarize-file` streams a local 16-bit PCM WAV through the configured speech recognition pipeline and returns the finished transcript. This is useful for checking the FunASR CAMPPlus path or the local Whisper plus pyannote path without starting a live recording. The request can include an optional speaker-count hint: `{ "path": "C:\\path\\to\\sample.wav", "numSpeakers": 5 }`. + +`DictationWordsPath` is reserved for providers that can accept vocabulary hints. It points to a markdown word list containing unusual project, person, product, or domain terms that should be biased during transcription when the provider supports it. + +Meeting notes link to separate transcript, assistant context, and summary markdown artifacts using the configured vault folders. + +Meeting note, transcript, assistant context, and summary artifacts use frontmatter links so they backlink to each other. Meeting note frontmatter includes `start_time` when recording starts and `end_time` when transcription processing finishes. Transcript and summary frontmatter copy the meeting title, start time, and end time from the meeting note; if the meeting has no title, the summary agent can provide one when writing the summary. + +Assistant context notes keep their artifact links in frontmatter and expose a lifecycle `state`: `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`. Meeting Assistant updates this state as recording, final speaker recognition, and summary generation progress. + +`ProjectsFolder` contains project knowledge. Each direct subfolder is treated as one project, and a meeting binds projects by listing those subfolder names in the meeting note `projects` frontmatter. + +`Agent` configures the Microsoft Agent Framework summary pipeline. `Key` may be set directly for local testing, but `KeyEnv` is preferred; by default the API key is read from `LITELLM_API_KEY`. After transcription has fully finished, Meeting Assistant automatically runs the summary pipeline for the meeting. `ReconnectionAttempts` and `ReconnectionDelay` control retries for transient model endpoint failures such as LiteLLM token refresh or 5xx responses. If summary generation still fails, Meeting Assistant overwrites the configured summary note with a markdown failure report containing the exception details and linked meeting artifacts. + +`ContextWindowTokens`, `MaxOutputTokens`, `EnableCompaction`, and `CompactionRemainingRatio` configure summary-agent context monitoring. Meeting Assistant estimates the outgoing Responses payload size, logs the estimated token count and remaining context, and compacts the conversation when only the configured remaining ratio is left. It first attempts `POST /v1/{ResponsesCompactPath}` on the configured OpenAI-compatible endpoint. If that endpoint is unavailable or returns invalid data, it falls back to Microsoft Agent Framework compaction: collapse old tool results, summarize older message groups, preserve only the last four user turns if needed, and finally drop oldest groups until the request is back under budget. + +The summary endpoints are: + +```text +POST /meetings/current/summary/run +POST /meetings/summary/retry +GET /meetings/summary/retry?summaryPath=... +``` + +`POST /meetings/summary/retry` accepts `{ "summaryPath": "20260519-124110-summary.md" }` or an absolute path under the configured summaries folder. It resolves the linked meeting note from frontmatter and reruns the summary pipeline, overwriting the existing summary file with either the new summary or a new failure report. + +Failure summary notes include a clickable retry link using `Api:PublicBaseUrl`, for example `http://localhost:5090/meetings/summary/retry?summaryPath=20260519-124110-summary.md`. The GET endpoint exists for Obsidian/browser clicks and performs the same overwrite behavior as the POST endpoint. Meeting note bodies stay reserved for user-authored notes. + +The summary agent currently has meeting-scoped tools to read the transcript, full meeting note including frontmatter, assistant context, user notes, glossary, and project files. Every read tool that returns file-like content can read the whole content or a clamped inclusive 1-based line range with `from` and `to`, so the agent can inspect large transcripts incrementally. The assistant context note is the agent's notebook and user-visible context window; the agent can write internal notes, discovered context, missing tool requests, suggested improvements, or follow-up task ideas there using line-based overwrite, replace, and insert modes. Project lookup and search tools are constrained to the projects listed in the meeting note frontmatter: + +- `read_meetingnote` +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` +- `write_context` + +`search` accepts ripgrep syntax and returns matches as `filename:line text`. If one project is searched, paths are relative to that project; if multiple projects are searched, paths include the project folder name. + +`write_projectfile` writes inside an existing project folder. With no line arguments it overwrites or creates the file. With `from` and `to` it replaces that inclusive 1-based line range. With `insert` it inserts content at that 1-based line position. + ## Spec Workflow This repository is OpenSpec-driven. Before changing behavior, read: diff --git a/openspec/changes/add-cross-platform-build-targets/proposal.md b/openspec/changes/add-cross-platform-build-targets/proposal.md new file mode 100644 index 0000000..64fa60a --- /dev/null +++ b/openspec/changes/add-cross-platform-build-targets/proposal.md @@ -0,0 +1,15 @@ +## Why + +Meeting Assistant currently focuses V1 runtime support on Windows capture and Outlook enrichment. We need a separate future change to define macOS and Linux build targets without expanding the V1 scope. + +## What Changes + +- Define supported macOS and Linux build targets for Meeting Assistant. +- Keep platform-specific capture, hotkey, calendar, and shell integrations isolated per target. +- Ensure non-Windows builds compile without Windows COM, WASAPI, or user32 dependencies. +- Define platform-specific test selection so Windows-only tests do not run on macOS/Linux and future macOS/Linux tests do not run on incompatible targets. + +## Impact + +- Affected specs: build-targets +- Affected code: project target frameworks, platform service registration, platform integration tests diff --git a/openspec/changes/add-cross-platform-build-targets/specs/build-targets/spec.md b/openspec/changes/add-cross-platform-build-targets/specs/build-targets/spec.md new file mode 100644 index 0000000..38bd0bf --- /dev/null +++ b/openspec/changes/add-cross-platform-build-targets/specs/build-targets/spec.md @@ -0,0 +1,19 @@ +## ADDED Requirements + +### Requirement: Meeting Assistant defines macOS and Linux build targets +Meeting Assistant SHALL define macOS and Linux build targets separately from the Windows build target. + +Windows-only integrations such as WASAPI capture, global hotkeys through user32, and Outlook Classic COM SHALL NOT be compiled into macOS or Linux targets. + +#### Scenario: Non-Windows target compiles +- **WHEN** Meeting Assistant is built for macOS or Linux +- **THEN** Windows-only source files and package dependencies are excluded +- **AND** the application compiles with portable service registrations or explicit unsupported implementations + +#### Scenario: Windows target keeps Windows integrations +- **WHEN** Meeting Assistant is built for Windows +- **THEN** WASAPI audio capture, Windows global hotkeys, and Outlook Classic COM enrichment remain available + +#### Scenario: Platform-specific tests are gated +- **WHEN** tests run for a platform target +- **THEN** tests for incompatible platform APIs are excluded or compiled only for their matching target diff --git a/openspec/changes/add-cross-platform-build-targets/tasks.md b/openspec/changes/add-cross-platform-build-targets/tasks.md new file mode 100644 index 0000000..9b7f3d8 --- /dev/null +++ b/openspec/changes/add-cross-platform-build-targets/tasks.md @@ -0,0 +1,11 @@ +## 1. Build Targets + +- [ ] 1.1 Add explicit macOS and Linux target framework/runtime build coverage. +- [ ] 1.2 Keep Windows-only implementations excluded from macOS and Linux builds. +- [ ] 1.3 Add platform service registration for unsupported or future macOS/Linux capture and hotkey implementations. + +## 2. Tests + +- [ ] 2.1 Split platform-specific tests by compilation target or runtime guard. +- [ ] 2.2 Add CI/build commands that verify Windows, macOS, and Linux compile targets. +- [ ] 2.3 Add at least one portable compile test that proves Windows-only packages are not required by non-Windows builds. diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/design.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/design.md new file mode 100644 index 0000000..26ab67d --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/design.md @@ -0,0 +1,77 @@ +## Context + +Meeting Assistant is a local/server-side .NET 10 application that helps capture and process meetings. It should work for any meeting format, including in-person conversations, and should not require direct APIs from meeting products such as Teams or Zoom. + +The central artifact is an Obsidian markdown note created before transcription begins. The user can add attendees, project context, and live notes while the meeting is happening. Meeting Assistant can add discovered context to the linked assistant-context note, then later append or link summaries, next steps, and knowledge base updates. + +## Goals / Non-Goals + +**Goals:** + +- Establish a .NET 10 application skeleton and OpenSpec workflow. +- Model a platform-independent meeting session flow. +- Toggle recording/transcription mode through a configurable hotkey. +- Capture microphone input and computer output for transcription. +- Stream audio into a provider-independent speech recognition pipeline. +- Use FunASR for speaker-attributed transcription, while keeping local Whisper as a fallback provider. +- Write transcript text into the configured vault folder. +- Create a meeting note before transcription starts. +- Preserve user-authored notes as first-class input to summarization. +- Transcribe meeting audio with speaker attribution. +- Generate summaries, decisions, and next steps. +- Maintain project knowledge from meeting output and additional future sources. +- Let agents retrieve project context and keyword context. +- Allow agents to update files in existing project folders through explicit overwrite, replace, and insert tool modes. + +**Non-Goals:** + +- Require a Teams, Zoom, or calendar-specific integration in version 1. +- Require cloud transcription APIs. +- Define the final Obsidian vault path or note template completely in this first change. +- Implement unrestricted autonomous file edits. + +## Decisions + +1. Build the source application as ASP.NET Core on .NET 10. + - Rationale: It matches the requested runtime and keeps the application service-oriented. + +2. Make the Obsidian markdown note the session anchor. + - Rationale: The user can enrich the meeting context before and during transcription, and agents can work from a durable human-readable artifact. + +3. Treat meeting-platform APIs as optional augmentation. + - Rationale: The assistant must be usable for in-person meetings and for platforms where no integration is available or desired. + +4. Keep project knowledge as a generic capability. + - Rationale: Meeting-derived knowledge should later combine with other sources without coupling the design to transcripts only. + +5. Keep project file mutation simple in version 1. + - Rationale: Agents may update files in existing project folders, but project creation and scaffolded project templates are future work. + +6. Use the normal .NET options configuration model for hotkey, vault, recording, and provider settings. + - Rationale: These settings will need machine-specific values and should work through `appsettings.json`, environment variables, and future deployment configuration. + +7. Model ASR backends as speech recognition pipelines. + - Rationale: The application must not assume file-only transcription or split live transcription from final diarization at the coordinator boundary. A pipeline can initialize, wait for readiness, accept captured audio chunks, emit live transcript segments, and produce a finished transcript. FunASR consumes PCM over WebSocket, Whisper.NET remains available as a local windowed fallback with pyannote post-processing, and the app can replace either backend later without changing recording control or persistence. + +8. Capture computer output through WASAPI loopback and microphone input through WASAPI capture, then mix PCM chunks before transcription. + - Rationale: WASAPI loopback is the Windows-supported way to capture the system mix, and tests can cover the mixer without depending on live devices. + +9. Gate Windows-only capture, hotkey, and Outlook Classic COM enrichment behind the Windows target. + - Rationale: The portable build should continue to compile without Windows APIs, while the Windows build can augment meeting notes from the current Teams appointment when Outlook Classic is available. + +## Risks / Trade-offs + +- [Risk] Speaker attribution quality depends on the selected transcription pipeline. -> Mitigation: keep attribution requirements observable and provider-independent. +- [Risk] FunASR speaker metadata depends on the runtime/model configuration and may be absent. -> Mitigation: preserve speaker fields when returned and fall back to `Unknown` instead of dropping transcript text. +- [Risk] Whisper.NET is windowed rather than a dedicated real-time recognizer. -> Mitigation: keep it as a configurable fallback and make the window size configurable. +- [Risk] WASAPI loopback may not emit chunks when no system audio is playing. -> Mitigation: isolate the system-audio source so silence insertion can be added without changing the coordinator. +- [Risk] The Obsidian note may become too dense. -> Mitigation: separate metadata, user notes, assistant context, transcript references, and generated outputs into clear sections. +- [Risk] Agent file updates could affect the wrong project. -> Mitigation: restrict writes to existing project folders and prevent target paths from escaping the selected project. +- [Risk] Retrieval can surface stale or irrelevant project context. -> Mitigation: include source references and update timestamps in retrieved context. + +## Open Questions + +- Should transcript text files eventually be merged into the meeting note, linked from it, or remain separate artifacts? +- What metadata fields are required before transcription can start? +- Which storage backend should hold embeddings and project knowledge? +- What approval model should govern agent project file edits? diff --git a/openspec/changes/define-meeting-assistant-v1/proposal.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/proposal.md similarity index 63% rename from openspec/changes/define-meeting-assistant-v1/proposal.md rename to openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/proposal.md index 3894fd9..ac60804 100644 --- a/openspec/changes/define-meeting-assistant-v1/proposal.md +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/proposal.md @@ -8,22 +8,30 @@ Meeting Assistant should provide a meeting capture workflow that is not tied to - Establish OpenSpec as the source of truth for behavior changes. - Define the version 1 meeting flow around Obsidian markdown notes created before transcription starts. - Require the core flow to support in-person meetings and not depend on Teams, Zoom, or similar APIs. +- Add a configurable global hotkey that toggles recording/transcription mode. +- Capture microphone input and computer output into one transcription audio stream. +- Require speech recognition pipelines to accept streamable audio, with FunASR configured for speaker-attributed transcription and local Whisper plus pyannote kept as a fallback pipeline. +- Store the live transcript in a text file under the configured vault folder. - Add requirements for speaker-attributed transcription, summaries, decisions, and next steps. - Define project context capture and retrieval as first-class capabilities. -- Define a guarded agent tool model for project lookup and controlled project file changes. +- Define agent tools for project lookup and project file changes in existing projects. ## Capabilities ### New Capabilities - `meeting-session`: creation and lifecycle of meeting notes and metadata. +- `meeting-recording`: hotkey-controlled recording mode, mixed audio capture, and transcript persistence. - `meeting-transcription`: platform-independent transcription with speaker attribution. - `meeting-summary`: generated summaries, decisions, and next steps. - `project-knowledge`: project context, retrieval, and knowledge base updates from meetings. -- `agent-project-tools`: agent tools for lookup, retrieval, and controlled project file updates. +- `agent-project-tools`: agent tools for lookup, retrieval, and project file updates. ## Impact - New .NET 10 solution, service project, tests, README, and OpenSpec structure. +- New recording/transcription services using configurable .NET options. +- NAudio is used for Windows microphone and WASAPI loopback capture. +- FunASR is used as the configured speaker-attribution pipeline, and Whisper.NET plus pyannote remains available as a local transcription fallback. - Version 1 behavior is documented before feature implementation. - Future platform integrations remain optional augmentations instead of hard dependencies. diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/agent-project-tools/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/agent-project-tools/spec.md new file mode 100644 index 0000000..ffec73c --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/agent-project-tools/spec.md @@ -0,0 +1,57 @@ +## ADDED Requirements + +### Requirement: Agents can use project context tools +Meeting Assistant SHALL expose tools that allow agents to look up project information, retrieve keyword-relevant context, and inspect meeting-derived knowledge. + +Project tools SHALL treat each direct subfolder of the configured projects folder as one project. A meeting note binds projects by listing those subfolder names in the `projects` frontmatter field. + +The summary agent SHALL expose these project tools: + +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` + +#### Scenario: Agent looks up meeting project context +- **WHEN** a meeting note identifies a project +- **THEN** the agent can retrieve relevant project context through Meeting Assistant tools + +#### Scenario: Agent lists projects bound to the meeting +- **WHEN** the meeting note frontmatter lists project folder names +- **THEN** `list_projects` returns the matching configured project subfolders + +#### Scenario: Agent reads a clamped project file range +- **WHEN** the agent reads a project file with optional line bounds outside the file length +- **THEN** `read_projectfile` clamps the requested range to the available file lines without failing + +#### Scenario: Agent searches project knowledge +- **WHEN** the agent calls `search` with ripgrep syntax keywords +- **THEN** Meeting Assistant runs the search against the requested projects or the meeting-bound projects and returns matches as `filename:line text` + +### Requirement: Agents can write files in existing projects +Meeting Assistant SHALL expose a `write_projectfile` tool that can create or update files inside an existing project folder. + +The target project SHALL be an existing direct subfolder of the configured projects folder. The target file path SHALL stay inside that project folder. + +When no line edit arguments are supplied, `write_projectfile` SHALL overwrite the whole file and SHALL create the file when it does not exist. + +When both `from` and `to` are supplied, `write_projectfile` SHALL replace the inclusive 1-based line range with the supplied content. + +When `insert` is supplied, `write_projectfile` SHALL insert the supplied content at that 1-based line position. + +#### Scenario: Project file is overwritten or created +- **WHEN** the agent writes a project file without line edit arguments +- **THEN** Meeting Assistant writes the supplied content as the complete file content + +#### Scenario: Project file line range is replaced +- **WHEN** the agent writes a project file with `from` and `to` line numbers +- **THEN** Meeting Assistant replaces the inclusive line range with the supplied content + +#### Scenario: Project file content is inserted +- **WHEN** the agent writes a project file with an `insert` line number +- **THEN** Meeting Assistant inserts the supplied content at that line position + +#### Scenario: Project file write target is invalid +- **WHEN** the target project is missing or the target path escapes the project folder +- **THEN** Meeting Assistant refuses the project file write diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-recording/spec.md new file mode 100644 index 0000000..2df0794 --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-recording/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Recording mode is controlled by a configurable hotkey +Meeting Assistant SHALL use normal .NET configuration to define the global hotkey that toggles recording/transcription mode. + +#### Scenario: Hotkey starts recording +- **WHEN** the configured hotkey is pressed while recording mode is inactive +- **THEN** Meeting Assistant starts recording/transcription mode + +#### Scenario: Hotkey stops recording +- **WHEN** the configured hotkey is pressed while recording mode is active +- **THEN** Meeting Assistant stops recording/transcription mode + +### Requirement: Recording mode captures microphone and computer output +Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription. + +#### Scenario: Both sources produce audio +- **WHEN** microphone and computer output audio chunks are available +- **THEN** Meeting Assistant mixes them into one PCM stream before transcription + +#### Scenario: Device-level capture cannot be verified in tests +- **WHEN** automated tests run without live audio devices +- **THEN** Meeting Assistant verifies the audio mixer through deterministic source abstractions rather than depending on physical microphone or speaker devices + +### Requirement: Stopping recording drains captured audio through transcription +Meeting Assistant SHALL stop capturing new audio when recording mode is stopped, but it SHALL allow already captured audio to finish running through the configured speech recognition pipeline before the recording session completes. + +#### Scenario: Hotkey stops recording with buffered audio +- **WHEN** the user presses the hotkey to stop recording while captured audio is still buffered for transcription +- **THEN** Meeting Assistant stops capturing new audio and appends transcription output for the already captured audio before reporting the recording stopped + +### Requirement: Transcripts are written to the configured vault folder +Meeting Assistant SHALL write live transcript text to a markdown file in the configured vault folder. + +#### Scenario: Recording starts +- **WHEN** recording mode starts +- **THEN** Meeting Assistant creates the configured vault folder if necessary and creates a transcript markdown file + +#### Scenario: Transcription segment arrives +- **WHEN** the speech recognition pipeline emits a live transcript segment +- **THEN** Meeting Assistant appends that segment to the active transcript text file diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-session/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-session/spec.md new file mode 100644 index 0000000..6f7925d --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-session/spec.md @@ -0,0 +1,105 @@ +## ADDED Requirements + +### Requirement: Meeting Assistant creates a note before transcription +Meeting Assistant SHALL create an Obsidian markdown note for a meeting session before transcription starts. + +The note SHALL be the durable anchor for meeting metadata, user notes, assistant-discovered context, transcript references, summaries, decisions, and next steps. + +#### Scenario: Session starts +- **WHEN** the user starts a new meeting session +- **THEN** Meeting Assistant creates a markdown note in the configured meetings folder before starting transcription + +#### Scenario: Recording starts from hotkey +- **WHEN** the user presses the recording hotkey while recording is inactive +- **THEN** Meeting Assistant creates the transcript file, creates the meeting note with links to the transcript, assistant context, and summary notes, and opens the meeting note through an `obsidian://` system command + +#### Scenario: User adds context before transcription +- **WHEN** the note exists before transcription starts +- **THEN** the user can add attendees, project context, and other notes to that file + +### Requirement: Vault locations are configurable +Meeting Assistant SHALL expose separate configuration settings for transcript files, meeting notes, assistant context, generated summaries, project knowledge, and dictation words. + +The dictation words location SHALL point to a word-list artifact that speech recognition pipelines MAY use as vocabulary hints when supported. + +#### Scenario: Transcript and note locations are configured separately +- **WHEN** Meeting Assistant starts with configured vault locations +- **THEN** transcripts, meeting notes, assistant context, generated summaries, project knowledge, and dictation words each have an independently configured path + +#### Scenario: Provider supports dictation words +- **WHEN** the configured speech recognition pipeline supports vocabulary hints +- **THEN** Meeting Assistant can provide words from the configured dictation words path without changing the other vault locations + +### Requirement: Meeting note template is coded and round-trippable +Meeting Assistant SHALL own a coded meeting note template and a store that can save and read meeting notes. + +The meeting note frontmatter SHALL include: + +- `start_time` as the meeting recording start timestamp +- `end_time` as the timestamp when transcript processing stops, empty while the meeting is active +- `attendees` as an array of display strings, where email is optional and may use a format such as `Mike ` +- `projects` as an array because one meeting can cover multiple projects +- `transcript` as text containing a markdown link to the transcript note +- `assistant_context` as text containing a markdown link to the assistant context note +- `summary` as text containing a markdown link to the generated summary note once Meeting Assistant creates it + +The user notes SHALL be the markdown body. Decisions and next steps SHALL NOT be required as frontmatter fields. + +#### Scenario: Meeting note is saved +- **WHEN** Meeting Assistant saves a meeting note +- **THEN** it writes the coded frontmatter template and the user notes body to a markdown file in the configured meeting notes folder + +#### Scenario: Meeting note links generated artifacts +- **WHEN** Meeting Assistant creates a meeting note for a new recording +- **THEN** the note frontmatter links to the configured transcript, assistant context, and summary note locations + +#### Scenario: Generated artifacts link back to each other +- **WHEN** Meeting Assistant creates transcript, assistant context, and summary artifacts +- **THEN** those artifacts use frontmatter links for the meeting note, transcript note, assistant context note, and summary note where applicable + +#### Scenario: Meeting note records session times +- **WHEN** Meeting Assistant creates a meeting note for a new recording +- **THEN** the note frontmatter contains `start_time` +- **AND** when transcript processing stops, Meeting Assistant writes `end_time` to the same note frontmatter +- **AND** transcript and summary frontmatter copy the meeting note title, `start_time`, and `end_time` + +#### Scenario: Meeting note is read +- **WHEN** Meeting Assistant reads a meeting note created from the coded template +- **THEN** it returns the frontmatter fields and user notes body without losing attendees, projects, transcript link, assistant context link, or summary + +#### Scenario: Meeting note frontmatter is updated +- **WHEN** Meeting Assistant reads a meeting note, changes frontmatter, and writes the note back +- **THEN** the existing user-authored markdown body remains unchanged + +### Requirement: Meeting notes preserve user-authored content +Meeting Assistant SHALL preserve user-authored meeting notes and include them as input when generating summaries, decisions, and next steps. + +#### Scenario: User writes notes during the meeting +- **WHEN** the user updates the meeting note while the meeting is active +- **THEN** Meeting Assistant keeps that content and incorporates it into generated meeting outputs + +### Requirement: Meeting sessions are platform independent +Meeting Assistant SHALL support meeting sessions that do not originate from Teams, Zoom, or any other meeting software API. + +Meeting software integrations MAY augment the meeting experience, but they SHALL NOT be required for the primary meeting capture, transcription, or summary flow. + +#### Scenario: In-person meeting is captured +- **WHEN** the user starts transcription for an in-person meeting +- **THEN** Meeting Assistant captures and processes the meeting without requiring a meeting platform integration + +### Requirement: Windows Outlook enrichment is optional +Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target. + +When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title and attendees to the meeting note and copy the appointment agenda to the assistant-context frontmatter. + +The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text. + +#### Scenario: Current Teams appointment enriches meeting artifacts +- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment +- **THEN** Meeting Assistant uses the appointment subject as the meeting title +- **AND** writes the appointment attendees into meeting note frontmatter +- **AND** writes the appointment agenda into assistant-context frontmatter + +#### Scenario: Outlook is unavailable or ambiguous +- **WHEN** Outlook Classic is unavailable or more than one current Teams appointment is found +- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-summary/spec.md new file mode 100644 index 0000000..85c11a1 --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-summary/spec.md @@ -0,0 +1,98 @@ +## ADDED Requirements + +### Requirement: Meeting Assistant generates meeting outputs +Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context. + +Meeting Assistant SHALL use a Microsoft Agent Framework pipeline for the first summary implementation. The pipeline SHALL use a configurable OpenAI-compatible endpoint, model, and API key source, including support for a direct key or an environment variable name. + +The summary pipeline SHALL retry transient model endpoint failures according to configurable reconnection attempts and delay settings. + +The summary pipeline SHALL track context-window usage for the configured model using response usage when available and request-size estimates otherwise. The context-window limit, maximum output reserve, compaction enablement, compaction threshold, and Responses compact endpoint path SHALL be configurable. + +When only the configured remaining context ratio is available, the summary pipeline SHALL try to compact the conversation through the configured OpenAI-compatible `POST /v1/responses/compact` endpoint. If that endpoint is unavailable or returns invalid data, the pipeline SHALL fall back to Microsoft Agent Framework compaction. + +Fallback compaction SHALL become increasingly aggressive: collapse old tool results, summarize older conversation spans, keep only the last four user turns, and drop oldest groups if still over budget. + +When summary generation fails after retries, Meeting Assistant SHALL write a markdown failure report to the configured summary note path. The failure report SHALL include a clickable retry link, error details, and the meeting artifact paths needed to diagnose or retry the run. + +After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for that meeting. + +Meeting Assistant SHALL expose an API operation to retry summary generation for a given summary note path. The retry operation SHALL resolve the linked meeting artifacts from the meeting note frontmatter and SHALL overwrite the existing summary note with either the new summary or a new failure report. + +The summary pipeline SHALL expose tools scoped to the current meeting: + +- `read_meetingnote` +- `read_transcript` +- `read_context` +- `read_usernotes` +- `read_glossary` +- `write_summary` +- `write_context` +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` + +All summary-agent read tools that return file-like content SHALL support either reading the whole file or reading a clamped inclusive 1-based line range when `from` and `to` are supplied. + +After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced project-relevant knowledge. + +The assistant context note SHALL keep `meeting`, `transcript`, `summary`, `state`, and `agenda` in frontmatter. The `state` value SHALL be one of `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`. + +The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, and `end_time` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary. + +The summary agent SHALL be able to read and write the assistant context body as its own notebook using overwrite, replace, and insert line modes. + +#### Scenario: Meeting processing completes +- **WHEN** transcription and context collection have completed +- **THEN** Meeting Assistant generates a summary, decisions, and next steps for the meeting note + +#### Scenario: Assistant context note is initialized +- **WHEN** Meeting Assistant starts a meeting session +- **THEN** it creates the assistant context note with frontmatter links to the meeting note, transcript note, and summary note +- **AND** the assistant context frontmatter state is `transcribing` +- **AND** the assistant context frontmatter includes `agenda`, empty when no agenda is known + +#### Scenario: Assistant context state follows meeting processing +- **WHEN** transcription, final speaker recognition, and summary generation progress +- **THEN** Meeting Assistant updates assistant context frontmatter state to `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error` as appropriate + +#### Scenario: Summary pipeline is invoked +- **WHEN** transcript processing finishes for the current meeting +- **THEN** the agent can read the transcript, assistant context, user notes, glossary, and bound project knowledge, can write the finished markdown summary to the configured summary note, and can update existing project files + +#### Scenario: Summary agent reads meeting frontmatter +- **WHEN** the summary pipeline runs +- **THEN** the agent can read the full meeting note including frontmatter such as title, attendees, projects, start time, and end time +- **AND** the agent can read the assistant-context frontmatter including agenda + +#### Scenario: Summary agent reads large inputs by line range +- **WHEN** the summary agent reads the meeting note, transcript, assistant context, user notes, glossary, or a project file with `from` and `to` line numbers +- **THEN** Meeting Assistant returns only the clamped inclusive line range +- **AND** omitting the line arguments returns the whole readable content + +#### Scenario: Summary agent writes assistant context notes +- **WHEN** the summary pipeline runs +- **THEN** the agent can overwrite, replace lines in, or insert lines into the assistant context body while preserving assistant context frontmatter + +#### Scenario: Transient summary model endpoint failure is retried +- **WHEN** the model endpoint returns a retryable failure while the summary pipeline is running +- **THEN** Meeting Assistant retries the model request until it succeeds or the configured reconnection attempts are exhausted + +#### Scenario: Summary context window is compacted +- **WHEN** the summary pipeline estimates that only the configured remaining context ratio is available +- **THEN** Meeting Assistant attempts to compact the outgoing conversation with the configured Responses compact endpoint +- **AND** if remote compaction fails, Meeting Assistant uses Microsoft Agent Framework compaction with increasingly aggressive fallback strategies + +#### Scenario: Summary failure is written to the summary note +- **WHEN** summary generation fails after configured retries +- **THEN** Meeting Assistant writes a markdown failure report with a retry link to the configured summary note path + +#### Scenario: Summary retry overwrites existing summary note +- **WHEN** the user triggers summary retry for a configured summary note path +- **THEN** Meeting Assistant resolves the linked meeting artifacts and overwrites the existing summary note with the retry result + +#### Scenario: User notes conflict with transcript interpretation +- **WHEN** user-authored notes provide context that changes the interpretation of the transcript +- **THEN** Meeting Assistant considers those notes during summary and next-step generation diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-transcription/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-transcription/spec.md new file mode 100644 index 0000000..4f7cd0c --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/meeting-transcription/spec.md @@ -0,0 +1,186 @@ +## ADDED Requirements + +### Requirement: Meeting Assistant transcribes meetings with speaker attribution +Meeting Assistant SHALL transcribe meeting audio and associate transcript segments with speakers when speaker attribution is available. + +The transcription contract SHALL remain independent of the meeting source so that local audio, in-person meetings, and future meeting-platform integrations can use the same downstream processing. + +#### Scenario: Speaker-attributed transcript is produced +- **WHEN** meeting audio is transcribed and speaker attribution is available +- **THEN** the transcript contains ordered speaker-attributed segments + +#### Scenario: Speaker attribution is unavailable +- **WHEN** meeting audio is transcribed but speaker attribution is unavailable +- **THEN** Meeting Assistant keeps the transcript and marks speaker identity as unknown rather than discarding the meeting + +### Requirement: Speech recognition pipelines are streamable and configurable +Meeting Assistant SHALL route audio through a provider-independent speech recognition pipeline contract. + +The configured pipeline SHALL be selected through DI from the ASR backend configuration and SHALL expose operations to initialize, wait for readiness, write captured audio chunks, read live transcript segments, and read the finished transcript after capture completes. + +Reading the finished transcript SHALL accept optional pipeline options, including a requested speaker count hint. + +For real meeting recordings, Meeting Assistant SHALL derive the requested speaker count hint from the meeting note `attendees` frontmatter when at least two attendees are listed. + +The configured pipeline SHALL receive audio chunks as they are captured rather than requiring the recording coordinator to finish a complete meeting audio file first. + +#### Scenario: Streaming provider receives live audio +- **WHEN** recording mode is active and the combined audio source emits a chunk +- **THEN** Meeting Assistant can pass that chunk to the configured speech recognition pipeline before the audio source completes + +#### Scenario: Pipeline readiness is separated from recording capture +- **WHEN** recording mode starts before the configured speech recognition backend is ready +- **THEN** Meeting Assistant starts audio capture immediately and lets the pipeline finish readiness before it emits live transcript output + +#### Scenario: Pipeline is changed by configuration +- **WHEN** the configured ASR backend setting is changed +- **THEN** Meeting Assistant selects the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: ASR backend is tested with a WAV file +- **WHEN** the user submits a local PCM WAV file to the ASR diagnostic endpoint +- **THEN** Meeting Assistant streams that file through the configured speech recognition pipeline and returns the emitted live transcript segments + +#### Scenario: ASR diagnostic backend is unavailable +- **WHEN** the configured speech recognition pipeline fails while processing the diagnostic WAV file +- **THEN** Meeting Assistant returns a diagnostic backend failure response instead of hiding the pipeline error + +#### Scenario: Final diarization backend is tested with a WAV file +- **WHEN** the user submits a local PCM WAV file to the final diarization diagnostic endpoint +- **THEN** Meeting Assistant streams the file through the configured speech recognition pipeline and returns the finished transcript segments + +#### Scenario: Final diarization receives a speaker count hint +- **WHEN** the user submits a local PCM WAV file and `numSpeakers` to the final diarization diagnostic endpoint +- **THEN** Meeting Assistant passes the requested speaker count to the configured speech recognition pipeline as a finished-transcript option + +#### Scenario: Meeting attendee count provides final speaker hint +- **WHEN** a meeting note has two or more entries in `attendees` +- **THEN** Meeting Assistant passes that attendee count to the configured speech recognition pipeline as the finished-transcript speaker count hint + +#### Scenario: Empty or single-attendee meeting note has no final speaker hint +- **WHEN** a meeting note has zero or one entry in `attendees` +- **THEN** Meeting Assistant leaves the finished-transcript speaker count hint unset + +### Requirement: Version 1 keeps local Whisper transcription available +Meeting Assistant SHALL provide a local Whisper speech recognition pipeline as a fallback ASR backend for version 1. + +The local Whisper adapter SHALL use configurable model path, language, and streaming window settings. + +When local Whisper is the configured speech recognition pipeline, Meeting Assistant SHALL be able to run a final pyannote diarization pass over the temporary mixed audio and either assign pyannote speaker turns back to live Whisper transcript segments by timestamp overlap or build finished transcript segments from pyannote speaker turns. + +The pyannote finalization backend SHALL allow configuration to use either pyannote's normal `speaker_diarization` annotation or pyannote's `exclusive_speaker_diarization` annotation. + +When a speaker count hint is provided through the speech recognition pipeline options, the pyannote finalization backend SHALL pass it to pyannote as `num_speakers`. + +The pyannote finalization backend SHALL use a reusable runtime image for Python dependencies and a persistent configured cache folder for Hugging Face models, torch artifacts, and pip artifacts so repeated finalization runs do not redownload the diarization model or reinstall the full Python dependency stack. + +#### Scenario: Local Whisper model is configured +- **WHEN** the configured provider is `whisper-local` +- **THEN** Meeting Assistant uses the configured local Whisper model path to transcribe incoming audio windows + +#### Scenario: Local Whisper transcript is finalized with pyannote speaker turns +- **WHEN** the configured provider is `whisper-local`, live Whisper transcript segments exist, and pyannote returns speaker turns for the temporary recording +- **THEN** Meeting Assistant rewrites the transcript with the original Whisper text and the best-overlap pyannote speaker label for each segment + +#### Scenario: Local Whisper transcript is segmented by pyannote speaker turns +- **WHEN** the configured provider is `whisper-local`, pyannote turn alignment is configured, live Whisper transcript segments exist, and pyannote returns speaker turns for the temporary recording +- **THEN** Meeting Assistant rewrites the transcript as ordered pyannote speaker-turn segments with matching Whisper text assigned to each turn + +#### Scenario: Local Whisper uses exclusive pyannote diarization +- **WHEN** the configured provider is `whisper-local` and exclusive pyannote annotation is configured +- **THEN** Meeting Assistant reads speaker turns from pyannote's `exclusive_speaker_diarization` output + +#### Scenario: Local Whisper pyannote runtime cache is reused +- **WHEN** local Whisper finalization runs pyannote more than once +- **THEN** Meeting Assistant uses the configured reusable pyannote runtime image and persistent pyannote model cache instead of treating each run as a clean download environment + +#### Scenario: Local Whisper pyannote diarization is unavailable +- **WHEN** the configured provider is `whisper-local` but pyannote is disabled, has no configured token, or returns no speaker turns +- **THEN** Meeting Assistant keeps the live Whisper transcript instead of deleting it + +### Requirement: Azure Speech can provide streaming transcription +Meeting Assistant SHALL provide an Azure Speech speech recognition pipeline that streams captured PCM audio to Azure AI Speech through the Azure Speech SDK conversation transcription API. + +The Azure Speech adapter SHALL use configurable endpoint, region, language, key, and key environment variable settings. + +The Azure Speech adapter SHALL support configurable diarization of intermediate conversation transcription results. + +The Azure Speech adapter SHALL bound recognizer shutdown after the input audio stream is closed. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL emit live transcript segments from Azure conversation transcription events with Azure speaker IDs when available. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL use the live Azure conversation transcription segments as the finished transcript without running pyannote finalization. + +#### Scenario: Azure Speech pipeline is configured +- **WHEN** the configured provider is `azure-speech` +- **THEN** Meeting Assistant streams captured PCM chunks to Azure AI Speech conversation transcription through the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: Azure Speech returns speaker IDs +- **WHEN** Azure Speech conversation transcription returns transcript text with speaker identity +- **THEN** Meeting Assistant emits ordered transcript segments with those Azure speaker identities + +#### Scenario: Azure Speech key is resolved from environment +- **WHEN** Azure Speech is configured with `KeyEnv` +- **THEN** Meeting Assistant reads the Azure Speech key from that environment variable + +#### Scenario: Azure Speech stream shutdown is bounded +- **WHEN** the captured audio stream has ended +- **THEN** Meeting Assistant stops Azure Speech continuous recognition within the configured stop timeout instead of waiting indefinitely + +#### Scenario: Azure Speech transcript is finalized from live conversation segments +- **WHEN** the configured provider is `azure-speech` and live Azure transcript segments exist +- **THEN** Meeting Assistant uses the live Azure conversation transcript segments as the finished transcript + +### Requirement: FunASR can provide speaker-attributed streaming transcription +Meeting Assistant SHALL provide a FunASR speech recognition pipeline that streams PCM audio to a configured FunASR WebSocket endpoint. + +The FunASR provider SHALL preserve speaker attribution from FunASR response fields when they are present and SHALL fall back to `Unknown` when no speaker field is returned. + +Meeting Assistant SHALL write the mixed recording audio only as a temporary processing artifact while streaming transcription is active. After capture stops and the streaming provider has drained already-captured audio, Meeting Assistant SHALL be able to run a final FunASR Python `AutoModel` pass with VAD, punctuation, and CAMPPlus speaker modeling over the temporary audio. When that final pass returns sentence-level speaker labels, Meeting Assistant SHALL rewrite the transcript markdown with the final speaker-attributed sentence segments. + +Meeting Assistant SHALL delete temporary recording audio after the run completes, whether final diarization succeeds, fails, or produces no speaker output. On application startup, Meeting Assistant SHALL delete stale temporary recording audio left behind by previous interrupted runs. + +When configured, Meeting Assistant SHALL manage a local FunASR backend lifecycle for the FunASR provider by starting a non-blocking backend warm-up when the application starts, installing the configured streaming runtime image, preparing the mounted model and hotword folder, starting the backend, keeping the container alive for the backgrounded FunASR server process, and stopping it when the application stops. Recording SHALL be able to start while the backend is still warming up; captured audio SHALL be queued and streamed once the backend is healthy. Backend installation, startup commands, and WebSocket readiness SHALL be bounded by configurable timeouts so startup failures are diagnosable. + +#### Scenario: FunASR pipeline is configured +- **WHEN** the configured provider is `funasr` +- **THEN** Meeting Assistant streams captured PCM chunks to the configured FunASR WebSocket endpoint through the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: FunASR returns speaker fields +- **WHEN** FunASR returns transcript text with speaker identity fields +- **THEN** Meeting Assistant emits ordered transcript segments with those speaker identities + +#### Scenario: Final FunASR diarization rewrites transcript +- **WHEN** recording stops after mixed audio was temporarily stored and the final FunASR diarization pass returns sentence-level speaker labels +- **THEN** Meeting Assistant rewrites the transcript markdown with the final speaker-attributed sentence segments + +#### Scenario: Final FunASR diarization has no speaker output +- **WHEN** recording stops but the final diarization pass is disabled or returns no sentence-level speaker labels +- **THEN** Meeting Assistant keeps the live streaming transcript instead of deleting it + +#### Scenario: Temporary recording audio is deleted after completion +- **WHEN** a recording run completes after using temporary audio for final processing +- **THEN** Meeting Assistant deletes the temporary audio file from disk + +#### Scenario: Stale temporary recordings are deleted on startup +- **WHEN** Meeting Assistant starts and stale temporary recording files exist from a previous interrupted run +- **THEN** Meeting Assistant deletes those stale temporary recording files before accepting new recordings + +#### Scenario: Managed FunASR backend starts with the application +- **WHEN** the configured provider is `funasr` and managed backend startup is enabled +- **THEN** Meeting Assistant begins installing the configured FunASR backend image and starting the two-pass streaming backend without blocking recording startup + +#### Scenario: Recording starts while managed FunASR backend is warming up +- **WHEN** recording starts before the managed FunASR backend is ready +- **THEN** Meeting Assistant starts audio capture immediately and queues captured audio until the backend WebSocket is healthy + +#### Scenario: Managed FunASR backend installation stalls +- **WHEN** a managed FunASR backend command exceeds its configured timeout +- **THEN** Meeting Assistant reports a backend startup timeout instead of waiting indefinitely + +#### Scenario: Managed FunASR backend WebSocket is not healthy +- **WHEN** the managed backend container has started but the FunASR WebSocket endpoint is not accepting sessions +- **THEN** Meeting Assistant continues waiting for backend readiness until the configured startup timeout is reached + +#### Scenario: Managed FunASR backend stops with the application +- **WHEN** Meeting Assistant stops after starting a managed FunASR backend +- **THEN** Meeting Assistant stops the managed backend process or container diff --git a/openspec/changes/define-meeting-assistant-v1/specs/project-knowledge/spec.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/project-knowledge/spec.md similarity index 64% rename from openspec/changes/define-meeting-assistant-v1/specs/project-knowledge/spec.md rename to openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/project-knowledge/spec.md index 1658479..b513aeb 100644 --- a/openspec/changes/define-meeting-assistant-v1/specs/project-knowledge/spec.md +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/specs/project-knowledge/spec.md @@ -5,10 +5,16 @@ Meeting Assistant SHALL maintain project knowledge based on meeting content and Project knowledge SHALL support retrieval by project context and relevant keywords. +Project knowledge SHALL be stored under the configured projects folder, where each direct subfolder is a project identified by its folder name. + #### Scenario: Meeting has project context - **WHEN** a meeting note identifies a project - **THEN** Meeting Assistant can associate transcript-derived and summary-derived knowledge with that project +#### Scenario: Meeting binds project folders +- **WHEN** the meeting note `projects` frontmatter lists configured project subfolder names +- **THEN** Meeting Assistant treats those subfolders as the projects available to meeting agents + #### Scenario: Agent retrieves project context - **WHEN** an agent needs context for a meeting or follow-up task - **THEN** Meeting Assistant can retrieve relevant project knowledge and source references diff --git a/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/tasks.md b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/tasks.md new file mode 100644 index 0000000..3418387 --- /dev/null +++ b/openspec/changes/archive/2026-05-19-define-meeting-assistant-v1/tasks.md @@ -0,0 +1,58 @@ +## 1. Repository Setup + +- [x] 1.1 Create the Gitea repository and clone it to `C:\Manuel\meeting-assistant`. +- [x] 1.2 Add .NET 10 solution, service project, and test project. +- [x] 1.3 Add repository README and agent instructions. +- [x] 1.4 Add OpenSpec config and the version 1 proposal/design/specs. + +## 2. Meeting Session Notes + +- [x] 2.1 Define configurable vault locations for transcripts, meeting notes, assistant context, summaries, project knowledge, and dictation words. +- [x] 2.2 Add a coded meeting note template and store with attendees, projects, transcript link, assistant context link, summary link, and body user notes. +- [x] 2.3 Implement meeting session creation that writes the initial note before transcription starts and opens it through `obsidian://`. +- [x] 2.4 Add tests proving user-authored notes remain preserved and included in later processing. +- [x] 2.5 Add a configurable summary note location and store summary as a markdown link in meeting note frontmatter. + +## 2A. Recording Control and Audio Capture + +- [x] 2A.1 Add configurable hotkey settings using the normal .NET options model. +- [x] 2A.2 Implement hotkey-controlled recording mode toggle. +- [x] 2A.3 Capture microphone input and computer output through replaceable audio source abstractions. +- [x] 2A.4 Mix microphone and system audio chunks into one PCM stream. +- [x] 2A.5 Write transcript text into the configured vault folder. +- [x] 2A.6 Add deterministic tests for streaming coordinator behavior, vault transcript persistence, hotkey parsing, and audio mixing. +- [x] 2A.7 Stop capture while draining already captured audio through the speech recognition pipeline. + +## 3. Transcription + +- [x] 3.1 Select the first ASR backend. +- [x] 3.2 Add provider-independent speech recognition pipeline contracts. +- [x] 3.3 Implement transcription ingestion for platform-independent audio input. +- [x] 3.4 Add FunASR speaker-attribution provider and tests. +- [x] 3.5 Add local Whisper.NET provider adapter with configurable model path, language, and window size. +- [x] 3.6 Add a small WAV fixture for transcription-provider tests. +- [x] 3.7 Add an ASR diagnostic endpoint that streams a local WAV file through the configured provider. +- [x] 3.8 Add managed FunASR backend install/start/stop lifecycle for the FunASR provider. +- [x] 3.9 Temporarily write mixed recording audio and run a final FunASR CAMPPlus diarization pass that can rewrite the transcript. +- [x] 3.10 Delete temporary recording audio after each run and clean up stale temporary recordings at startup. +- [x] 3.11 Add a local Whisper final pyannote diarization pass that maps speaker turns onto live Whisper transcript segments. +- [x] 3.12 Refactor ASR backends behind a speech recognition pipeline interface for initialization, readiness, chunk writes, live transcript reads, and finished transcript reads. + +## 4. Summary and Next Steps + +- [x] 4.1 Define the summary, decisions, open questions, and next steps output structure for the first summary note. +- [x] 4.2 Create the assistant context note at session start and link it to the meeting, transcript, and summary notes. +- [x] 4.3 Add a Microsoft Agent Framework summary pipeline with OpenAI-compatible configuration and meeting-scoped tools. +- [x] 4.4 Add assistant-context agenda frontmatter and Windows-only Outlook Classic Teams appointment enrichment. + +## 5. Project Knowledge and Agent Tools + +- [x] 5.1 Define project binding metadata for meetings. +- [x] 5.2 Define project knowledge storage and retrieval requirements. +- [x] 5.3 Add agent tools for project lookup and keyword lookup. +- [x] 5.4 Define and implement project file write modes for overwrite, replace, and insert. + +## 6. Verification + +- [x] 6.1 Run `dotnet test MeetingAssistant.slnx`. +- [x] 6.2 Run `openspec validate define-meeting-assistant-v1 --strict`. diff --git a/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/proposal.md b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/proposal.md new file mode 100644 index 0000000..46ec0b1 --- /dev/null +++ b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/proposal.md @@ -0,0 +1,14 @@ +## Why + +When Outlook Classic detects the current Teams appointment at transcription start, Meeting Assistant already copies title, attendees, and agenda into meeting artifacts. The assistant context should also record the appointment's scheduled end time so later agent steps can understand meeting timing without querying Outlook again. + +## What Changes + +- Add `scheduled_end` frontmatter to assistant context notes when a Teams appointment is detected at recording start. +- Preserve `scheduled_end` when assistant context state changes. +- Keep the field absent when no Teams appointment is detected. + +## Impact + +- Affected specs: meeting-summary, meeting-session +- Affected code: Outlook meeting metadata, recording coordinator, assistant context artifact store diff --git a/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-session/spec.md b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-session/spec.md new file mode 100644 index 0000000..09df502 --- /dev/null +++ b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-session/spec.md @@ -0,0 +1,20 @@ +## MODIFIED Requirements + +### Requirement: Windows Outlook enrichment is optional +Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target. + +When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title and attendees to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter. + +The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text. + +#### Scenario: Current Teams appointment enriches meeting artifacts +- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment +- **THEN** Meeting Assistant uses the appointment subject as the meeting title +- **AND** writes the appointment attendees into meeting note frontmatter +- **AND** writes the appointment agenda into assistant-context frontmatter +- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter + +#### Scenario: Outlook is unavailable or ambiguous +- **WHEN** Outlook Classic is unavailable or more than one current Teams appointment is found +- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda +- **AND** omits `scheduled_end` from assistant-context frontmatter diff --git a/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-summary/spec.md b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-summary/spec.md new file mode 100644 index 0000000..8b69395 --- /dev/null +++ b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/specs/meeting-summary/spec.md @@ -0,0 +1,57 @@ +## MODIFIED Requirements + +### Requirement: Meeting Assistant generates meeting outputs +Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context. + +Meeting Assistant SHALL use a Microsoft Agent Framework pipeline for the first summary implementation. The pipeline SHALL use a configurable OpenAI-compatible endpoint, model, and API key source, including support for a direct key or an environment variable name. + +The summary pipeline SHALL retry transient model endpoint failures according to configurable reconnection attempts and delay settings. + +The summary pipeline SHALL track context-window usage for the configured model using response usage when available and request-size estimates otherwise. The context-window limit, maximum output reserve, compaction enablement, compaction threshold, and Responses compact endpoint path SHALL be configurable. + +When only the configured remaining context ratio is available, the summary pipeline SHALL try to compact the conversation through the configured OpenAI-compatible `POST /v1/responses/compact` endpoint. If that endpoint is unavailable or returns invalid data, the pipeline SHALL fall back to Microsoft Agent Framework compaction. + +Fallback compaction SHALL become increasingly aggressive: collapse old tool results, summarize older conversation spans, keep only the last four user turns, and drop oldest groups if still over budget. + +When summary generation fails after retries, Meeting Assistant SHALL write a markdown failure report to the configured summary note path. The failure report SHALL include a clickable retry link, error details, and the meeting artifact paths needed to diagnose or retry the run. + +After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for that meeting. + +Meeting Assistant SHALL expose an API operation to retry summary generation for a given summary note path. The retry operation SHALL resolve the linked meeting artifacts from the meeting note frontmatter and SHALL overwrite the existing summary note with either the new summary or a new failure report. + +The summary pipeline SHALL expose tools scoped to the current meeting: + +- `read_meetingnote` +- `read_transcript` +- `read_context` +- `read_usernotes` +- `read_glossary` +- `write_summary` +- `write_context` +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` + +All summary-agent read tools that return file-like content SHALL support either reading the whole file or reading a clamped inclusive 1-based line range when `from` and `to` are supplied. + +After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced project-relevant knowledge. + +The assistant context note SHALL keep `meeting`, `transcript`, `summary`, `state`, and `agenda` in frontmatter. When a Teams appointment is detected at recording start, the assistant context note SHALL also keep `scheduled_end` in frontmatter. The `state` value SHALL be one of `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`. + +The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, and `end_time` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary. + +The summary agent SHALL be able to read and write the assistant context body as its own notebook using overwrite, replace, and insert line modes. + +#### Scenario: Assistant context note is initialized +- **WHEN** Meeting Assistant starts a meeting session +- **THEN** it creates the assistant context note with frontmatter links to the meeting note, transcript note, and summary note +- **AND** the assistant context frontmatter state is `transcribing` +- **AND** the assistant context frontmatter includes `agenda`, empty when no agenda is known +- **AND** the assistant context frontmatter includes `scheduled_end` when Teams metadata supplied a scheduled end time + +#### Scenario: Assistant context state follows meeting processing +- **WHEN** transcription, final speaker recognition, and summary generation progress +- **THEN** Meeting Assistant updates assistant context frontmatter state to `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error` as appropriate +- **AND** preserves existing `agenda` and `scheduled_end` frontmatter values diff --git a/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/tasks.md b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/tasks.md new file mode 100644 index 0000000..2880746 --- /dev/null +++ b/openspec/changes/archive/2026-05-20-add-teams-scheduled-end-context/tasks.md @@ -0,0 +1,6 @@ +## 1. Scheduled End Metadata + +- [x] 1.1 Extend Teams meeting metadata with scheduled end time. +- [x] 1.2 Write `scheduled_end` to assistant context frontmatter only when Teams metadata is detected. +- [x] 1.3 Preserve `scheduled_end` across assistant context state updates. +- [x] 1.4 Add behavior tests and validate the OpenSpec change. diff --git a/openspec/changes/define-meeting-assistant-v1/design.md b/openspec/changes/define-meeting-assistant-v1/design.md deleted file mode 100644 index 722ea89..0000000 --- a/openspec/changes/define-meeting-assistant-v1/design.md +++ /dev/null @@ -1,58 +0,0 @@ -## Context - -Meeting Assistant is a local/server-side .NET 10 application that helps capture and process meetings. It should work for any meeting format, including in-person conversations, and should not require direct APIs from meeting products such as Teams or Zoom. - -The central artifact is an Obsidian markdown note created before transcription begins. The user can add attendees, project context, meeting purpose, and live notes while the meeting is happening. Meeting Assistant can add discovered context to the same note, then later append or link summaries, next steps, and knowledge base updates. - -## Goals / Non-Goals - -**Goals:** - -- Establish a .NET 10 application skeleton and OpenSpec workflow. -- Model a platform-independent meeting session flow. -- Create a meeting note before transcription starts. -- Preserve user-authored notes as first-class input to summarization. -- Transcribe meeting audio with speaker attribution. -- Generate summaries, decisions, and next steps. -- Maintain project knowledge from meeting output and additional future sources. -- Let agents retrieve project context and keyword context. -- Allow guarded project file updates when the project context or user explicitly permits them. - -**Non-Goals:** - -- Build a Teams, Zoom, or calendar-specific integration in version 1. -- Require cloud transcription APIs. -- Define the final Obsidian vault path or note template completely in this first change. -- Implement unrestricted autonomous file edits. - -## Decisions - -1. Build the source application as ASP.NET Core on .NET 10. - - Rationale: It matches the requested runtime and keeps the application service-oriented. - -2. Make the Obsidian markdown note the session anchor. - - Rationale: The user can enrich the meeting context before and during transcription, and agents can work from a durable human-readable artifact. - -3. Treat meeting-platform APIs as optional augmentation. - - Rationale: The assistant must be usable for in-person meetings and for platforms where no integration is available or desired. - -4. Keep project knowledge as a generic capability. - - Rationale: Meeting-derived knowledge should later combine with other sources without coupling the design to transcripts only. - -5. Require explicit guardrails for project file mutation. - - Rationale: Agents may update project files, but only when the project is known and the user or project context permits edits. - -## Risks / Trade-offs - -- [Risk] Speaker attribution quality depends on the selected transcription pipeline. -> Mitigation: keep attribution requirements observable and provider-independent. -- [Risk] The Obsidian note may become too dense. -> Mitigation: separate metadata, user notes, assistant context, transcript references, and generated outputs into clear sections. -- [Risk] Agent file updates could affect the wrong project. -> Mitigation: require explicit project binding before mutation tools are available. -- [Risk] Retrieval can surface stale or irrelevant project context. -> Mitigation: include source references and update timestamps in retrieved context. - -## Open Questions - -- What exact Obsidian vault folder should hold meeting notes? -- What metadata fields are required before transcription can start? -- Which transcription provider should be used first? -- Which storage backend should hold embeddings and project knowledge? -- What approval model should govern agent project file edits? diff --git a/openspec/changes/define-meeting-assistant-v1/specs/agent-project-tools/spec.md b/openspec/changes/define-meeting-assistant-v1/specs/agent-project-tools/spec.md deleted file mode 100644 index be14331..0000000 --- a/openspec/changes/define-meeting-assistant-v1/specs/agent-project-tools/spec.md +++ /dev/null @@ -1,19 +0,0 @@ -## ADDED Requirements - -### Requirement: Agents can use project context tools -Meeting Assistant SHALL expose tools that allow agents to look up project information, retrieve keyword-relevant context, and inspect meeting-derived knowledge. - -#### Scenario: Agent looks up meeting project context -- **WHEN** a meeting note identifies a project -- **THEN** the agent can retrieve relevant project context through Meeting Assistant tools - -### Requirement: Agents can only mutate project files with explicit project binding -Meeting Assistant SHALL only expose project file mutation tools when the target project is known and mutation is permitted by user instruction or project context. - -#### Scenario: Project mutation is permitted -- **WHEN** the user or project context permits file updates for a known project -- **THEN** the agent can use controlled tools to change files in that project - -#### Scenario: Project mutation is not permitted -- **WHEN** the target project is missing or mutation permission is absent -- **THEN** Meeting Assistant does not expose project file mutation tools for that request diff --git a/openspec/changes/define-meeting-assistant-v1/specs/meeting-session/spec.md b/openspec/changes/define-meeting-assistant-v1/specs/meeting-session/spec.md deleted file mode 100644 index e297da1..0000000 --- a/openspec/changes/define-meeting-assistant-v1/specs/meeting-session/spec.md +++ /dev/null @@ -1,30 +0,0 @@ -## ADDED Requirements - -### Requirement: Meeting Assistant creates a note before transcription -Meeting Assistant SHALL create an Obsidian markdown note for a meeting session before transcription starts. - -The note SHALL be the durable anchor for meeting metadata, user notes, assistant-discovered context, transcript references, summaries, decisions, and next steps. - -#### Scenario: Session starts -- **WHEN** the user starts a new meeting session -- **THEN** Meeting Assistant creates a markdown note in the configured meetings folder before starting transcription - -#### Scenario: User adds context before transcription -- **WHEN** the note exists before transcription starts -- **THEN** the user can add attendees, project context, meeting purpose, and other notes to that file - -### Requirement: Meeting notes preserve user-authored content -Meeting Assistant SHALL preserve user-authored meeting notes and include them as input when generating summaries, decisions, and next steps. - -#### Scenario: User writes notes during the meeting -- **WHEN** the user updates the meeting note while the meeting is active -- **THEN** Meeting Assistant keeps that content and incorporates it into generated meeting outputs - -### Requirement: Meeting sessions are platform independent -Meeting Assistant SHALL support meeting sessions that do not originate from Teams, Zoom, or any other meeting software API. - -Meeting software integrations MAY augment the meeting experience, but they SHALL NOT be required for the primary meeting capture, transcription, or summary flow. - -#### Scenario: In-person meeting is captured -- **WHEN** the user starts transcription for an in-person meeting -- **THEN** Meeting Assistant captures and processes the meeting without requiring a meeting platform integration diff --git a/openspec/changes/define-meeting-assistant-v1/specs/meeting-summary/spec.md b/openspec/changes/define-meeting-assistant-v1/specs/meeting-summary/spec.md deleted file mode 100644 index eef9174..0000000 --- a/openspec/changes/define-meeting-assistant-v1/specs/meeting-summary/spec.md +++ /dev/null @@ -1,12 +0,0 @@ -## ADDED Requirements - -### Requirement: Meeting Assistant generates meeting outputs -Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context. - -#### Scenario: Meeting processing completes -- **WHEN** transcription and context collection have completed -- **THEN** Meeting Assistant generates a summary, decisions, and next steps for the meeting note - -#### Scenario: User notes conflict with transcript interpretation -- **WHEN** user-authored notes provide context that changes the interpretation of the transcript -- **THEN** Meeting Assistant considers those notes during summary and next-step generation diff --git a/openspec/changes/define-meeting-assistant-v1/specs/meeting-transcription/spec.md b/openspec/changes/define-meeting-assistant-v1/specs/meeting-transcription/spec.md deleted file mode 100644 index 115f157..0000000 --- a/openspec/changes/define-meeting-assistant-v1/specs/meeting-transcription/spec.md +++ /dev/null @@ -1,14 +0,0 @@ -## ADDED Requirements - -### Requirement: Meeting Assistant transcribes meetings with speaker attribution -Meeting Assistant SHALL transcribe meeting audio and associate transcript segments with speakers when speaker attribution is available. - -The transcription contract SHALL remain independent of the meeting source so that local audio, in-person meetings, and future meeting-platform integrations can use the same downstream processing. - -#### Scenario: Speaker-attributed transcript is produced -- **WHEN** meeting audio is transcribed and speaker attribution is available -- **THEN** the transcript contains ordered speaker-attributed segments - -#### Scenario: Speaker attribution is unavailable -- **WHEN** meeting audio is transcribed but speaker attribution is unavailable -- **THEN** Meeting Assistant keeps the transcript and marks speaker identity as unknown rather than discarding the meeting diff --git a/openspec/changes/define-meeting-assistant-v1/tasks.md b/openspec/changes/define-meeting-assistant-v1/tasks.md deleted file mode 100644 index 67e71b1..0000000 --- a/openspec/changes/define-meeting-assistant-v1/tasks.md +++ /dev/null @@ -1,38 +0,0 @@ -## 1. Repository Setup - -- [x] 1.1 Create the Gitea repository and clone it to `C:\Manuel\meeting-assistant`. -- [x] 1.2 Add .NET 10 solution, service project, and test project. -- [x] 1.3 Add repository README and agent instructions. -- [x] 1.4 Add OpenSpec config and the version 1 proposal/design/specs. - -## 2. Meeting Session Notes - -- [ ] 2.1 Define the Obsidian meeting note location and naming convention. -- [ ] 2.2 Add a meeting note template with metadata, attendees, purpose, project context, user notes, assistant context, transcript, summary, decisions, and next steps. -- [ ] 2.3 Implement meeting session creation that writes the initial note before transcription starts. -- [ ] 2.4 Add tests proving user-authored notes remain preserved and included in later processing. - -## 3. Transcription - -- [ ] 3.1 Select the first transcription provider. -- [ ] 3.2 Add provider-independent transcription contracts. -- [ ] 3.3 Implement transcription ingestion for platform-independent audio input. -- [ ] 3.4 Add speaker attribution handling and tests. - -## 4. Summary and Next Steps - -- [ ] 4.1 Define the summary, decisions, and next steps output structure. -- [ ] 4.2 Implement summary generation from transcript plus user notes. -- [ ] 4.3 Append or link generated outputs from the meeting note. - -## 5. Project Knowledge and Agent Tools - -- [ ] 5.1 Define project binding metadata for meetings. -- [ ] 5.2 Define project knowledge storage and retrieval requirements. -- [ ] 5.3 Add agent tools for project lookup and keyword lookup. -- [ ] 5.4 Define and implement guardrails for project file mutation tools. - -## 6. Verification - -- [x] 6.1 Run `dotnet test MeetingAssistant.slnx`. -- [x] 6.2 Run `openspec validate define-meeting-assistant-v1 --strict`. diff --git a/openspec/specs/agent-project-tools/spec.md b/openspec/specs/agent-project-tools/spec.md new file mode 100644 index 0000000..6f29733 --- /dev/null +++ b/openspec/specs/agent-project-tools/spec.md @@ -0,0 +1,61 @@ +# agent-project-tools Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Agents can use project context tools +Meeting Assistant SHALL expose tools that allow agents to look up project information, retrieve keyword-relevant context, and inspect meeting-derived knowledge. + +Project tools SHALL treat each direct subfolder of the configured projects folder as one project. A meeting note binds projects by listing those subfolder names in the `projects` frontmatter field. + +The summary agent SHALL expose these project tools: + +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` + +#### Scenario: Agent looks up meeting project context +- **WHEN** a meeting note identifies a project +- **THEN** the agent can retrieve relevant project context through Meeting Assistant tools + +#### Scenario: Agent lists projects bound to the meeting +- **WHEN** the meeting note frontmatter lists project folder names +- **THEN** `list_projects` returns the matching configured project subfolders + +#### Scenario: Agent reads a clamped project file range +- **WHEN** the agent reads a project file with optional line bounds outside the file length +- **THEN** `read_projectfile` clamps the requested range to the available file lines without failing + +#### Scenario: Agent searches project knowledge +- **WHEN** the agent calls `search` with ripgrep syntax keywords +- **THEN** Meeting Assistant runs the search against the requested projects or the meeting-bound projects and returns matches as `filename:line text` + +### Requirement: Agents can write files in existing projects +Meeting Assistant SHALL expose a `write_projectfile` tool that can create or update files inside an existing project folder. + +The target project SHALL be an existing direct subfolder of the configured projects folder. The target file path SHALL stay inside that project folder. + +When no line edit arguments are supplied, `write_projectfile` SHALL overwrite the whole file and SHALL create the file when it does not exist. + +When both `from` and `to` are supplied, `write_projectfile` SHALL replace the inclusive 1-based line range with the supplied content. + +When `insert` is supplied, `write_projectfile` SHALL insert the supplied content at that 1-based line position. + +#### Scenario: Project file is overwritten or created +- **WHEN** the agent writes a project file without line edit arguments +- **THEN** Meeting Assistant writes the supplied content as the complete file content + +#### Scenario: Project file line range is replaced +- **WHEN** the agent writes a project file with `from` and `to` line numbers +- **THEN** Meeting Assistant replaces the inclusive line range with the supplied content + +#### Scenario: Project file content is inserted +- **WHEN** the agent writes a project file with an `insert` line number +- **THEN** Meeting Assistant inserts the supplied content at that line position + +#### Scenario: Project file write target is invalid +- **WHEN** the target project is missing or the target path escapes the project folder +- **THEN** Meeting Assistant refuses the project file write + diff --git a/openspec/specs/meeting-recording/spec.md b/openspec/specs/meeting-recording/spec.md new file mode 100644 index 0000000..5a745ca --- /dev/null +++ b/openspec/specs/meeting-recording/spec.md @@ -0,0 +1,45 @@ +# meeting-recording Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Recording mode is controlled by a configurable hotkey +Meeting Assistant SHALL use normal .NET configuration to define the global hotkey that toggles recording/transcription mode. + +#### Scenario: Hotkey starts recording +- **WHEN** the configured hotkey is pressed while recording mode is inactive +- **THEN** Meeting Assistant starts recording/transcription mode + +#### Scenario: Hotkey stops recording +- **WHEN** the configured hotkey is pressed while recording mode is active +- **THEN** Meeting Assistant stops recording/transcription mode + +### Requirement: Recording mode captures microphone and computer output +Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription. + +#### Scenario: Both sources produce audio +- **WHEN** microphone and computer output audio chunks are available +- **THEN** Meeting Assistant mixes them into one PCM stream before transcription + +#### Scenario: Device-level capture cannot be verified in tests +- **WHEN** automated tests run without live audio devices +- **THEN** Meeting Assistant verifies the audio mixer through deterministic source abstractions rather than depending on physical microphone or speaker devices + +### Requirement: Stopping recording drains captured audio through transcription +Meeting Assistant SHALL stop capturing new audio when recording mode is stopped, but it SHALL allow already captured audio to finish running through the configured speech recognition pipeline before the recording session completes. + +#### Scenario: Hotkey stops recording with buffered audio +- **WHEN** the user presses the hotkey to stop recording while captured audio is still buffered for transcription +- **THEN** Meeting Assistant stops capturing new audio and appends transcription output for the already captured audio before reporting the recording stopped + +### Requirement: Transcripts are written to the configured vault folder +Meeting Assistant SHALL write live transcript text to a markdown file in the configured vault folder. + +#### Scenario: Recording starts +- **WHEN** recording mode starts +- **THEN** Meeting Assistant creates the configured vault folder if necessary and creates a transcript markdown file + +#### Scenario: Transcription segment arrives +- **WHEN** the speech recognition pipeline emits a live transcript segment +- **THEN** Meeting Assistant appends that segment to the active transcript text file + diff --git a/openspec/specs/meeting-session/spec.md b/openspec/specs/meeting-session/spec.md new file mode 100644 index 0000000..7015025 --- /dev/null +++ b/openspec/specs/meeting-session/spec.md @@ -0,0 +1,111 @@ +# meeting-session Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Meeting Assistant creates a note before transcription +Meeting Assistant SHALL create an Obsidian markdown note for a meeting session before transcription starts. + +The note SHALL be the durable anchor for meeting metadata, user notes, assistant-discovered context, transcript references, summaries, decisions, and next steps. + +#### Scenario: Session starts +- **WHEN** the user starts a new meeting session +- **THEN** Meeting Assistant creates a markdown note in the configured meetings folder before starting transcription + +#### Scenario: Recording starts from hotkey +- **WHEN** the user presses the recording hotkey while recording is inactive +- **THEN** Meeting Assistant creates the transcript file, creates the meeting note with links to the transcript, assistant context, and summary notes, and opens the meeting note through an `obsidian://` system command + +#### Scenario: User adds context before transcription +- **WHEN** the note exists before transcription starts +- **THEN** the user can add attendees, project context, and other notes to that file + +### Requirement: Vault locations are configurable +Meeting Assistant SHALL expose separate configuration settings for transcript files, meeting notes, assistant context, generated summaries, project knowledge, and dictation words. + +The dictation words location SHALL point to a word-list artifact that speech recognition pipelines MAY use as vocabulary hints when supported. + +#### Scenario: Transcript and note locations are configured separately +- **WHEN** Meeting Assistant starts with configured vault locations +- **THEN** transcripts, meeting notes, assistant context, generated summaries, project knowledge, and dictation words each have an independently configured path + +#### Scenario: Provider supports dictation words +- **WHEN** the configured speech recognition pipeline supports vocabulary hints +- **THEN** Meeting Assistant can provide words from the configured dictation words path without changing the other vault locations + +### Requirement: Meeting note template is coded and round-trippable +Meeting Assistant SHALL own a coded meeting note template and a store that can save and read meeting notes. + +The meeting note frontmatter SHALL include: + +- `start_time` as the meeting recording start timestamp +- `end_time` as the timestamp when transcript processing stops, empty while the meeting is active +- `attendees` as an array of display strings, where email is optional and may use a format such as `Mike ` +- `projects` as an array because one meeting can cover multiple projects +- `transcript` as text containing a markdown link to the transcript note +- `assistant_context` as text containing a markdown link to the assistant context note +- `summary` as text containing a markdown link to the generated summary note once Meeting Assistant creates it + +The user notes SHALL be the markdown body. Decisions and next steps SHALL NOT be required as frontmatter fields. + +#### Scenario: Meeting note is saved +- **WHEN** Meeting Assistant saves a meeting note +- **THEN** it writes the coded frontmatter template and the user notes body to a markdown file in the configured meeting notes folder + +#### Scenario: Meeting note links generated artifacts +- **WHEN** Meeting Assistant creates a meeting note for a new recording +- **THEN** the note frontmatter links to the configured transcript, assistant context, and summary note locations + +#### Scenario: Generated artifacts link back to each other +- **WHEN** Meeting Assistant creates transcript, assistant context, and summary artifacts +- **THEN** those artifacts use frontmatter links for the meeting note, transcript note, assistant context note, and summary note where applicable + +#### Scenario: Meeting note records session times +- **WHEN** Meeting Assistant creates a meeting note for a new recording +- **THEN** the note frontmatter contains `start_time` +- **AND** when transcript processing stops, Meeting Assistant writes `end_time` to the same note frontmatter +- **AND** transcript and summary frontmatter copy the meeting note title, `start_time`, and `end_time` + +#### Scenario: Meeting note is read +- **WHEN** Meeting Assistant reads a meeting note created from the coded template +- **THEN** it returns the frontmatter fields and user notes body without losing attendees, projects, transcript link, assistant context link, or summary + +#### Scenario: Meeting note frontmatter is updated +- **WHEN** Meeting Assistant reads a meeting note, changes frontmatter, and writes the note back +- **THEN** the existing user-authored markdown body remains unchanged + +### Requirement: Meeting notes preserve user-authored content +Meeting Assistant SHALL preserve user-authored meeting notes and include them as input when generating summaries, decisions, and next steps. + +#### Scenario: User writes notes during the meeting +- **WHEN** the user updates the meeting note while the meeting is active +- **THEN** Meeting Assistant keeps that content and incorporates it into generated meeting outputs + +### Requirement: Meeting sessions are platform independent +Meeting Assistant SHALL support meeting sessions that do not originate from Teams, Zoom, or any other meeting software API. + +Meeting software integrations MAY augment the meeting experience, but they SHALL NOT be required for the primary meeting capture, transcription, or summary flow. + +#### Scenario: In-person meeting is captured +- **WHEN** the user starts transcription for an in-person meeting +- **THEN** Meeting Assistant captures and processes the meeting without requiring a meeting platform integration + +### Requirement: Windows Outlook enrichment is optional +Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target. + +When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title and attendees to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter. + +The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text. + +#### Scenario: Current Teams appointment enriches meeting artifacts +- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment +- **THEN** Meeting Assistant uses the appointment subject as the meeting title +- **AND** writes the appointment attendees into meeting note frontmatter +- **AND** writes the appointment agenda into assistant-context frontmatter +- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter + +#### Scenario: Outlook is unavailable or ambiguous +- **WHEN** Outlook Classic is unavailable or more than one current Teams appointment is found +- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda +- **AND** omits `scheduled_end` from assistant-context frontmatter + diff --git a/openspec/specs/meeting-summary/spec.md b/openspec/specs/meeting-summary/spec.md new file mode 100644 index 0000000..8666536 --- /dev/null +++ b/openspec/specs/meeting-summary/spec.md @@ -0,0 +1,61 @@ +# meeting-summary Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Meeting Assistant generates meeting outputs +Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context. + +Meeting Assistant SHALL use a Microsoft Agent Framework pipeline for the first summary implementation. The pipeline SHALL use a configurable OpenAI-compatible endpoint, model, and API key source, including support for a direct key or an environment variable name. + +The summary pipeline SHALL retry transient model endpoint failures according to configurable reconnection attempts and delay settings. + +The summary pipeline SHALL track context-window usage for the configured model using response usage when available and request-size estimates otherwise. The context-window limit, maximum output reserve, compaction enablement, compaction threshold, and Responses compact endpoint path SHALL be configurable. + +When only the configured remaining context ratio is available, the summary pipeline SHALL try to compact the conversation through the configured OpenAI-compatible `POST /v1/responses/compact` endpoint. If that endpoint is unavailable or returns invalid data, the pipeline SHALL fall back to Microsoft Agent Framework compaction. + +Fallback compaction SHALL become increasingly aggressive: collapse old tool results, summarize older conversation spans, keep only the last four user turns, and drop oldest groups if still over budget. + +When summary generation fails after retries, Meeting Assistant SHALL write a markdown failure report to the configured summary note path. The failure report SHALL include a clickable retry link, error details, and the meeting artifact paths needed to diagnose or retry the run. + +After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for that meeting. + +Meeting Assistant SHALL expose an API operation to retry summary generation for a given summary note path. The retry operation SHALL resolve the linked meeting artifacts from the meeting note frontmatter and SHALL overwrite the existing summary note with either the new summary or a new failure report. + +The summary pipeline SHALL expose tools scoped to the current meeting: + +- `read_meetingnote` +- `read_transcript` +- `read_context` +- `read_usernotes` +- `read_glossary` +- `write_summary` +- `write_context` +- `list_projects` +- `list_projectfiles` +- `read_projectfile` +- `write_projectfile` +- `search` + +All summary-agent read tools that return file-like content SHALL support either reading the whole file or reading a clamped inclusive 1-based line range when `from` and `to` are supplied. + +After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced project-relevant knowledge. + +The assistant context note SHALL keep `meeting`, `transcript`, `summary`, `state`, and `agenda` in frontmatter. When a Teams appointment is detected at recording start, the assistant context note SHALL also keep `scheduled_end` in frontmatter. The `state` value SHALL be one of `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`. + +The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, and `end_time` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary. + +The summary agent SHALL be able to read and write the assistant context body as its own notebook using overwrite, replace, and insert line modes. + +#### Scenario: Assistant context note is initialized +- **WHEN** Meeting Assistant starts a meeting session +- **THEN** it creates the assistant context note with frontmatter links to the meeting note, transcript note, and summary note +- **AND** the assistant context frontmatter state is `transcribing` +- **AND** the assistant context frontmatter includes `agenda`, empty when no agenda is known +- **AND** the assistant context frontmatter includes `scheduled_end` when Teams metadata supplied a scheduled end time + +#### Scenario: Assistant context state follows meeting processing +- **WHEN** transcription, final speaker recognition, and summary generation progress +- **THEN** Meeting Assistant updates assistant context frontmatter state to `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error` as appropriate +- **AND** preserves existing `agenda` and `scheduled_end` frontmatter values + diff --git a/openspec/specs/meeting-transcription/spec.md b/openspec/specs/meeting-transcription/spec.md new file mode 100644 index 0000000..1f4bad6 --- /dev/null +++ b/openspec/specs/meeting-transcription/spec.md @@ -0,0 +1,190 @@ +# meeting-transcription Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Meeting Assistant transcribes meetings with speaker attribution +Meeting Assistant SHALL transcribe meeting audio and associate transcript segments with speakers when speaker attribution is available. + +The transcription contract SHALL remain independent of the meeting source so that local audio, in-person meetings, and future meeting-platform integrations can use the same downstream processing. + +#### Scenario: Speaker-attributed transcript is produced +- **WHEN** meeting audio is transcribed and speaker attribution is available +- **THEN** the transcript contains ordered speaker-attributed segments + +#### Scenario: Speaker attribution is unavailable +- **WHEN** meeting audio is transcribed but speaker attribution is unavailable +- **THEN** Meeting Assistant keeps the transcript and marks speaker identity as unknown rather than discarding the meeting + +### Requirement: Speech recognition pipelines are streamable and configurable +Meeting Assistant SHALL route audio through a provider-independent speech recognition pipeline contract. + +The configured pipeline SHALL be selected through DI from the ASR backend configuration and SHALL expose operations to initialize, wait for readiness, write captured audio chunks, read live transcript segments, and read the finished transcript after capture completes. + +Reading the finished transcript SHALL accept optional pipeline options, including a requested speaker count hint. + +For real meeting recordings, Meeting Assistant SHALL derive the requested speaker count hint from the meeting note `attendees` frontmatter when at least two attendees are listed. + +The configured pipeline SHALL receive audio chunks as they are captured rather than requiring the recording coordinator to finish a complete meeting audio file first. + +#### Scenario: Streaming provider receives live audio +- **WHEN** recording mode is active and the combined audio source emits a chunk +- **THEN** Meeting Assistant can pass that chunk to the configured speech recognition pipeline before the audio source completes + +#### Scenario: Pipeline readiness is separated from recording capture +- **WHEN** recording mode starts before the configured speech recognition backend is ready +- **THEN** Meeting Assistant starts audio capture immediately and lets the pipeline finish readiness before it emits live transcript output + +#### Scenario: Pipeline is changed by configuration +- **WHEN** the configured ASR backend setting is changed +- **THEN** Meeting Assistant selects the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: ASR backend is tested with a WAV file +- **WHEN** the user submits a local PCM WAV file to the ASR diagnostic endpoint +- **THEN** Meeting Assistant streams that file through the configured speech recognition pipeline and returns the emitted live transcript segments + +#### Scenario: ASR diagnostic backend is unavailable +- **WHEN** the configured speech recognition pipeline fails while processing the diagnostic WAV file +- **THEN** Meeting Assistant returns a diagnostic backend failure response instead of hiding the pipeline error + +#### Scenario: Final diarization backend is tested with a WAV file +- **WHEN** the user submits a local PCM WAV file to the final diarization diagnostic endpoint +- **THEN** Meeting Assistant streams the file through the configured speech recognition pipeline and returns the finished transcript segments + +#### Scenario: Final diarization receives a speaker count hint +- **WHEN** the user submits a local PCM WAV file and `numSpeakers` to the final diarization diagnostic endpoint +- **THEN** Meeting Assistant passes the requested speaker count to the configured speech recognition pipeline as a finished-transcript option + +#### Scenario: Meeting attendee count provides final speaker hint +- **WHEN** a meeting note has two or more entries in `attendees` +- **THEN** Meeting Assistant passes that attendee count to the configured speech recognition pipeline as the finished-transcript speaker count hint + +#### Scenario: Empty or single-attendee meeting note has no final speaker hint +- **WHEN** a meeting note has zero or one entry in `attendees` +- **THEN** Meeting Assistant leaves the finished-transcript speaker count hint unset + +### Requirement: Version 1 keeps local Whisper transcription available +Meeting Assistant SHALL provide a local Whisper speech recognition pipeline as a fallback ASR backend for version 1. + +The local Whisper adapter SHALL use configurable model path, language, and streaming window settings. + +When local Whisper is the configured speech recognition pipeline, Meeting Assistant SHALL be able to run a final pyannote diarization pass over the temporary mixed audio and either assign pyannote speaker turns back to live Whisper transcript segments by timestamp overlap or build finished transcript segments from pyannote speaker turns. + +The pyannote finalization backend SHALL allow configuration to use either pyannote's normal `speaker_diarization` annotation or pyannote's `exclusive_speaker_diarization` annotation. + +When a speaker count hint is provided through the speech recognition pipeline options, the pyannote finalization backend SHALL pass it to pyannote as `num_speakers`. + +The pyannote finalization backend SHALL use a reusable runtime image for Python dependencies and a persistent configured cache folder for Hugging Face models, torch artifacts, and pip artifacts so repeated finalization runs do not redownload the diarization model or reinstall the full Python dependency stack. + +#### Scenario: Local Whisper model is configured +- **WHEN** the configured provider is `whisper-local` +- **THEN** Meeting Assistant uses the configured local Whisper model path to transcribe incoming audio windows + +#### Scenario: Local Whisper transcript is finalized with pyannote speaker turns +- **WHEN** the configured provider is `whisper-local`, live Whisper transcript segments exist, and pyannote returns speaker turns for the temporary recording +- **THEN** Meeting Assistant rewrites the transcript with the original Whisper text and the best-overlap pyannote speaker label for each segment + +#### Scenario: Local Whisper transcript is segmented by pyannote speaker turns +- **WHEN** the configured provider is `whisper-local`, pyannote turn alignment is configured, live Whisper transcript segments exist, and pyannote returns speaker turns for the temporary recording +- **THEN** Meeting Assistant rewrites the transcript as ordered pyannote speaker-turn segments with matching Whisper text assigned to each turn + +#### Scenario: Local Whisper uses exclusive pyannote diarization +- **WHEN** the configured provider is `whisper-local` and exclusive pyannote annotation is configured +- **THEN** Meeting Assistant reads speaker turns from pyannote's `exclusive_speaker_diarization` output + +#### Scenario: Local Whisper pyannote runtime cache is reused +- **WHEN** local Whisper finalization runs pyannote more than once +- **THEN** Meeting Assistant uses the configured reusable pyannote runtime image and persistent pyannote model cache instead of treating each run as a clean download environment + +#### Scenario: Local Whisper pyannote diarization is unavailable +- **WHEN** the configured provider is `whisper-local` but pyannote is disabled, has no configured token, or returns no speaker turns +- **THEN** Meeting Assistant keeps the live Whisper transcript instead of deleting it + +### Requirement: Azure Speech can provide streaming transcription +Meeting Assistant SHALL provide an Azure Speech speech recognition pipeline that streams captured PCM audio to Azure AI Speech through the Azure Speech SDK conversation transcription API. + +The Azure Speech adapter SHALL use configurable endpoint, region, language, key, and key environment variable settings. + +The Azure Speech adapter SHALL support configurable diarization of intermediate conversation transcription results. + +The Azure Speech adapter SHALL bound recognizer shutdown after the input audio stream is closed. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL emit live transcript segments from Azure conversation transcription events with Azure speaker IDs when available. + +When Azure Speech is the configured speech recognition pipeline, Meeting Assistant SHALL use the live Azure conversation transcription segments as the finished transcript without running pyannote finalization. + +#### Scenario: Azure Speech pipeline is configured +- **WHEN** the configured provider is `azure-speech` +- **THEN** Meeting Assistant streams captured PCM chunks to Azure AI Speech conversation transcription through the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: Azure Speech returns speaker IDs +- **WHEN** Azure Speech conversation transcription returns transcript text with speaker identity +- **THEN** Meeting Assistant emits ordered transcript segments with those Azure speaker identities + +#### Scenario: Azure Speech key is resolved from environment +- **WHEN** Azure Speech is configured with `KeyEnv` +- **THEN** Meeting Assistant reads the Azure Speech key from that environment variable + +#### Scenario: Azure Speech stream shutdown is bounded +- **WHEN** the captured audio stream has ended +- **THEN** Meeting Assistant stops Azure Speech continuous recognition within the configured stop timeout instead of waiting indefinitely + +#### Scenario: Azure Speech transcript is finalized from live conversation segments +- **WHEN** the configured provider is `azure-speech` and live Azure transcript segments exist +- **THEN** Meeting Assistant uses the live Azure conversation transcript segments as the finished transcript + +### Requirement: FunASR can provide speaker-attributed streaming transcription +Meeting Assistant SHALL provide a FunASR speech recognition pipeline that streams PCM audio to a configured FunASR WebSocket endpoint. + +The FunASR provider SHALL preserve speaker attribution from FunASR response fields when they are present and SHALL fall back to `Unknown` when no speaker field is returned. + +Meeting Assistant SHALL write the mixed recording audio only as a temporary processing artifact while streaming transcription is active. After capture stops and the streaming provider has drained already-captured audio, Meeting Assistant SHALL be able to run a final FunASR Python `AutoModel` pass with VAD, punctuation, and CAMPPlus speaker modeling over the temporary audio. When that final pass returns sentence-level speaker labels, Meeting Assistant SHALL rewrite the transcript markdown with the final speaker-attributed sentence segments. + +Meeting Assistant SHALL delete temporary recording audio after the run completes, whether final diarization succeeds, fails, or produces no speaker output. On application startup, Meeting Assistant SHALL delete stale temporary recording audio left behind by previous interrupted runs. + +When configured, Meeting Assistant SHALL manage a local FunASR backend lifecycle for the FunASR provider by starting a non-blocking backend warm-up when the application starts, installing the configured streaming runtime image, preparing the mounted model and hotword folder, starting the backend, keeping the container alive for the backgrounded FunASR server process, and stopping it when the application stops. Recording SHALL be able to start while the backend is still warming up; captured audio SHALL be queued and streamed once the backend is healthy. Backend installation, startup commands, and WebSocket readiness SHALL be bounded by configurable timeouts so startup failures are diagnosable. + +#### Scenario: FunASR pipeline is configured +- **WHEN** the configured provider is `funasr` +- **THEN** Meeting Assistant streams captured PCM chunks to the configured FunASR WebSocket endpoint through the configured speech recognition pipeline without changing recording control or transcript persistence code + +#### Scenario: FunASR returns speaker fields +- **WHEN** FunASR returns transcript text with speaker identity fields +- **THEN** Meeting Assistant emits ordered transcript segments with those speaker identities + +#### Scenario: Final FunASR diarization rewrites transcript +- **WHEN** recording stops after mixed audio was temporarily stored and the final FunASR diarization pass returns sentence-level speaker labels +- **THEN** Meeting Assistant rewrites the transcript markdown with the final speaker-attributed sentence segments + +#### Scenario: Final FunASR diarization has no speaker output +- **WHEN** recording stops but the final diarization pass is disabled or returns no sentence-level speaker labels +- **THEN** Meeting Assistant keeps the live streaming transcript instead of deleting it + +#### Scenario: Temporary recording audio is deleted after completion +- **WHEN** a recording run completes after using temporary audio for final processing +- **THEN** Meeting Assistant deletes the temporary audio file from disk + +#### Scenario: Stale temporary recordings are deleted on startup +- **WHEN** Meeting Assistant starts and stale temporary recording files exist from a previous interrupted run +- **THEN** Meeting Assistant deletes those stale temporary recording files before accepting new recordings + +#### Scenario: Managed FunASR backend starts with the application +- **WHEN** the configured provider is `funasr` and managed backend startup is enabled +- **THEN** Meeting Assistant begins installing the configured FunASR backend image and starting the two-pass streaming backend without blocking recording startup + +#### Scenario: Recording starts while managed FunASR backend is warming up +- **WHEN** recording starts before the managed FunASR backend is ready +- **THEN** Meeting Assistant starts audio capture immediately and queues captured audio until the backend WebSocket is healthy + +#### Scenario: Managed FunASR backend installation stalls +- **WHEN** a managed FunASR backend command exceeds its configured timeout +- **THEN** Meeting Assistant reports a backend startup timeout instead of waiting indefinitely + +#### Scenario: Managed FunASR backend WebSocket is not healthy +- **WHEN** the managed backend container has started but the FunASR WebSocket endpoint is not accepting sessions +- **THEN** Meeting Assistant continues waiting for backend readiness until the configured startup timeout is reached + +#### Scenario: Managed FunASR backend stops with the application +- **WHEN** Meeting Assistant stops after starting a managed FunASR backend +- **THEN** Meeting Assistant stops the managed backend process or container + diff --git a/openspec/specs/project-knowledge/spec.md b/openspec/specs/project-knowledge/spec.md new file mode 100644 index 0000000..120c030 --- /dev/null +++ b/openspec/specs/project-knowledge/spec.md @@ -0,0 +1,24 @@ +# project-knowledge Specification + +## Purpose +TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive. +## Requirements +### Requirement: Meeting Assistant maintains project knowledge +Meeting Assistant SHALL maintain project knowledge based on meeting content and future generic information sources. + +Project knowledge SHALL support retrieval by project context and relevant keywords. + +Project knowledge SHALL be stored under the configured projects folder, where each direct subfolder is a project identified by its folder name. + +#### Scenario: Meeting has project context +- **WHEN** a meeting note identifies a project +- **THEN** Meeting Assistant can associate transcript-derived and summary-derived knowledge with that project + +#### Scenario: Meeting binds project folders +- **WHEN** the meeting note `projects` frontmatter lists configured project subfolder names +- **THEN** Meeting Assistant treats those subfolders as the projects available to meeting agents + +#### Scenario: Agent retrieves project context +- **WHEN** an agent needs context for a meeting or follow-up task +- **THEN** Meeting Assistant can retrieve relevant project knowledge and source references +