diff --git a/MeetingAssistant.Tests/AudioMixingTests.cs b/MeetingAssistant.Tests/AudioMixingTests.cs index 711f7b8..64aa82c 100644 --- a/MeetingAssistant.Tests/AudioMixingTests.cs +++ b/MeetingAssistant.Tests/AudioMixingTests.cs @@ -6,40 +6,129 @@ namespace MeetingAssistant.Tests; public sealed class AudioMixingTests { [Fact] - public async Task CompositeAudioSourceMixesMicrophoneAndSystemAudioIntoOnePcmStream() + public async Task CompositeAudioSourceCleansMicrophoneBeforeAddingSystemAudio() { - var microphone = new FixedAudioSource(Pcm16(10_000)); - var system = new FixedAudioSource(Pcm16(20_000)); - var source = new CompositeMeetingAudioSource( - microphone, - system, - NullLogger.Instance); + var microphone = new FixedAudioSource(Pcm16(12_000)); + var system = new FixedAudioSource(Pcm16(10_000)); + var source = CreateSource(microphone, system, new FixedEchoCanceller(Pcm16(2_000))); var chunks = await ReadChunks(source); Assert.Single(chunks); - Assert.Equal(30_000, BitConverter.ToInt16(chunks[0].Pcm)); + Assert.Equal(12_000, BitConverter.ToInt16(chunks[0].Pcm)); } [Fact] - public async Task CompositeAudioSourceClampsMixedSamples() + public async Task CompositeAudioSourceClampsAfterConfiguredGain() { var microphone = new FixedAudioSource(Pcm16(30_000)); var system = new FixedAudioSource(Pcm16(10_000)); - var source = new CompositeMeetingAudioSource( - microphone, - system, - NullLogger.Instance); + var source = CreateSource(microphone, system); var chunks = await ReadChunks(source); Assert.Equal(short.MaxValue, BitConverter.ToInt16(chunks[0].Pcm)); } - private static async Task> ReadChunks(IMeetingAudioSource source) + [Fact] + public async Task CompositeAudioSourceUsesRunOptionsForChildSourcesAndGains() + { + var microphone = new OptionsCapturingAudioSource(Pcm16(10_000)); + var system = new OptionsCapturingAudioSource(Pcm16(10_000)); + var source = CreateSource(microphone, system); + var options = new MeetingAssistantOptions + { + Recording = + { + SampleRate = 24000, + Channels = 1, + MicrophoneMixGain = 0.5, + SystemAudioMixGain = 0.25 + } + }; + + var chunks = await ReadChunks(source, options); + + var chunk = Assert.Single(chunks); + Assert.Equal(7_500, BitConverter.ToInt16(chunk.Pcm)); + Assert.Same(options, microphone.CapturedOptions); + Assert.Same(options, system.CapturedOptions); + } + + [Fact] + public async Task CompositeAudioSourceWaitsForMatchingStreamsBeforeMixing() + { + var microphone = new DelayedAudioSource(TimeSpan.Zero, Pcm16(2_000)); + var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(100), Pcm16(10_000)); + var source = CreateSource(microphone, system); + + var chunks = await ReadChunks(source); + + var chunk = Assert.Single(chunks); + Assert.Equal(12_000, BitConverter.ToInt16(chunk.Pcm)); + } + + [Fact] + public async Task CompositeAudioSourceEmitsMicrophoneWithSilentSystemWhenSystemAudioIsQuiet() + { + var microphone = new DelayedAudioSource(TimeSpan.Zero, Pcm16(2_000)); + var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(500), Pcm16(10_000)); + var source = CreateSource(microphone, system); + + var chunks = await ReadChunks(source); + + Assert.Collection( + chunks, + first => + { + Assert.Equal(2_000, BitConverter.ToInt16(first.Pcm)); + }, + second => + { + Assert.Equal(10_000, BitConverter.ToInt16(second.Pcm)); + }); + } + + [Fact] + public async Task CompositeAudioSourceFlushesContinuouslyArrivingMicrophoneBeforeLateSystemAudio() + { + var microphone = new RepeatedAudioSource(50, TimeSpan.FromMilliseconds(10), Pcm16(2_000)); + var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(750), Pcm16(10_000)); + var source = CreateSource(microphone, system); + + var chunks = await ReadChunks(source); + + Assert.NotEmpty(chunks); + Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm)); + } + + [Fact] + public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal() + { + var canceller = new AdaptiveFilterAcousticEchoCanceller(); + var system = Enumerable.Repeat(Pcm16(8_000), 80).SelectMany(bytes => bytes).ToArray(); + var microphone = Enumerable.Repeat(Pcm16(8_000), 80).SelectMany(bytes => bytes).ToArray(); + + AudioChunk cleaned = new(microphone, 16000, 1); + for (var i = 0; i < 40; i++) + { + cleaned = canceller.CleanMicrophone( + new AudioChunk(microphone, 16000, 1), + new AudioChunk(system, 16000, 1)); + } + + var average = Enumerable.Range(0, cleaned.Pcm.Length / sizeof(short)) + .Select(index => (double)Math.Abs(BitConverter.ToInt16(cleaned.Pcm, index * sizeof(short)))) + .Average(); + Assert.True(average < 1_000); + } + + private static async Task> ReadChunks( + IMeetingAudioSource source, + MeetingAssistantOptions? options = null) { var chunks = new List(); - await foreach (var chunk in source.CaptureAsync(CancellationToken.None)) + await foreach (var chunk in source.CaptureAsync(options ?? new MeetingAssistantOptions(), CancellationToken.None)) { chunks.Add(chunk); } @@ -52,6 +141,18 @@ public sealed class AudioMixingTests return BitConverter.GetBytes(sample); } + private static CompositeMeetingAudioSource CreateSource( + IMeetingAudioSource microphone, + IMeetingAudioSource system, + IAcousticEchoCanceller? echoCanceller = null) + { + return new CompositeMeetingAudioSource( + microphone, + system, + new FixedEchoCancellerFactory(echoCanceller ?? new NoopAcousticEchoCanceller()), + NullLogger.Instance); + } + private sealed class FixedAudioSource : IMeetingAudioSource { private readonly AudioChunk chunk; @@ -68,4 +169,115 @@ public sealed class AudioMixingTests yield return chunk; } } + + private sealed class DelayedAudioSource : IMeetingAudioSource + { + private readonly TimeSpan delay; + private readonly AudioChunk chunk; + + public DelayedAudioSource(TimeSpan delay, byte[] pcm) + { + this.delay = delay; + chunk = new AudioChunk(pcm, 16000, 1); + } + + public async IAsyncEnumerable CaptureAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, cancellationToken); + } + + yield return chunk; + } + } + + private sealed class RepeatedAudioSource : IMeetingAudioSource + { + private readonly int count; + private readonly TimeSpan interval; + private readonly AudioChunk chunk; + + public RepeatedAudioSource(int count, TimeSpan interval, byte[] pcm) + { + this.count = count; + this.interval = interval; + chunk = new AudioChunk(pcm, 16000, 1); + } + + public async IAsyncEnumerable CaptureAsync( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + for (var index = 0; index < count; index++) + { + if (interval > TimeSpan.Zero) + { + await Task.Delay(interval, cancellationToken); + } + + yield return chunk; + } + } + } + + private sealed class OptionsCapturingAudioSource : IMeetingAudioSource + { + private readonly AudioChunk chunk; + + public OptionsCapturingAudioSource(byte[] pcm) + { + chunk = new AudioChunk(pcm, 16000, 1); + } + + public MeetingAssistantOptions? CapturedOptions { get; private set; } + + public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + return CaptureAsync(new MeetingAssistantOptions(), cancellationToken); + } + + public async IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + CapturedOptions = options; + yield return chunk; + } + } + + private sealed class FixedEchoCanceller : IAcousticEchoCanceller + { + private readonly byte[] cleanedPcm; + + public FixedEchoCanceller(byte[] cleanedPcm) + { + this.cleanedPcm = cleanedPcm; + } + + public void Reset() + { + } + + public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio) + { + return microphone with { Pcm = cleanedPcm }; + } + } + + private sealed class FixedEchoCancellerFactory : IAcousticEchoCancellerFactory + { + private readonly IAcousticEchoCanceller echoCanceller; + + public FixedEchoCancellerFactory(IAcousticEchoCanceller echoCanceller) + { + this.echoCanceller = echoCanceller; + } + + public IAcousticEchoCanceller Create() + { + return echoCanceller; + } + } } diff --git a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs index 013388c..a8a19e4 100644 --- a/MeetingAssistant.Tests/RecordingCoordinatorTests.cs +++ b/MeetingAssistant.Tests/RecordingCoordinatorTests.cs @@ -1613,6 +1613,37 @@ public sealed class RecordingCoordinatorTests Assert.True(new FileInfo(session.AudioPath).Length > 44); } + [Fact] + public async Task TemporaryRecordedAudioStoreDoesNotWriteDiagnosticSidecars() + { + var recordingFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); + var store = new TemporaryRecordedAudioStore( + Options.Create(new MeetingAssistantOptions + { + Recording = new RecordingOptions + { + SampleRate = 16000, + Channels = 1, + TemporaryRecordingsFolder = recordingFolder + } + }), + NullLogger.Instance); + + await using var session = await store.CreateSessionAsync(CancellationToken.None); + await session.AppendAsync(new AudioChunk([3, 0], 16000, 1), CancellationToken.None); + await session.CompleteAsync(CancellationToken.None); + + var microphonePath = Path.Combine( + recordingFolder, + $"{Path.GetFileNameWithoutExtension(session.AudioPath)}-microphone.wav"); + var systemPath = Path.Combine( + recordingFolder, + $"{Path.GetFileNameWithoutExtension(session.AudioPath)}-system.wav"); + Assert.True(new FileInfo(session.AudioPath).Length > 44); + Assert.False(File.Exists(microphonePath)); + Assert.False(File.Exists(systemPath)); + } + [Fact] public async Task TemporaryRecordedAudioStoreDeletesStaleRecordingsOnStartup() { diff --git a/MeetingAssistant/MeetingAssistant.csproj b/MeetingAssistant/MeetingAssistant.csproj index b0dd592..6fec471 100644 --- a/MeetingAssistant/MeetingAssistant.csproj +++ b/MeetingAssistant/MeetingAssistant.csproj @@ -5,11 +5,13 @@ enable enable true + 1.50.0 WinExe Assets\meeting-assistant.ico + x64 @@ -19,7 +21,8 @@ - + + @@ -41,6 +44,11 @@ + + + + + diff --git a/MeetingAssistant/MeetingAssistantOptions.cs b/MeetingAssistant/MeetingAssistantOptions.cs index 188ddc2..97cb04b 100644 --- a/MeetingAssistant/MeetingAssistantOptions.cs +++ b/MeetingAssistant/MeetingAssistantOptions.cs @@ -114,6 +114,10 @@ public sealed class RecordingOptions public int Channels { get; set; } = 1; + public double MicrophoneMixGain { get; set; } = 1; + + public double SystemAudioMixGain { get; set; } = 1; + public TimeSpan StopProcessingTimeout { get; set; } = TimeSpan.FromMinutes(10); public TimeSpan MetadataLookupTimeout { get; set; } = TimeSpan.FromSeconds(10); diff --git a/MeetingAssistant/Program.cs b/MeetingAssistant/Program.cs index cb1cd66..45d41af 100644 --- a/MeetingAssistant/Program.cs +++ b/MeetingAssistant/Program.cs @@ -18,9 +18,11 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(services => new CompositeMeetingAudioSource( services.GetRequiredService(), services.GetRequiredService(), + services.GetRequiredService(), services.GetRequiredService>())); builder.Services.AddSingleton(); #else diff --git a/MeetingAssistant/Recording/AcousticEchoCanceller.cs b/MeetingAssistant/Recording/AcousticEchoCanceller.cs new file mode 100644 index 0000000..32bc767 --- /dev/null +++ b/MeetingAssistant/Recording/AcousticEchoCanceller.cs @@ -0,0 +1,130 @@ +namespace MeetingAssistant.Recording; + +public interface IAcousticEchoCanceller +{ + void Reset(); + + AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio); +} + +public interface IAcousticEchoCancellerFactory +{ + IAcousticEchoCanceller Create(); +} + +public sealed class AdaptiveFilterAcousticEchoCancellerFactory : IAcousticEchoCancellerFactory +{ + public IAcousticEchoCanceller Create() + { + return new AdaptiveFilterAcousticEchoCanceller(); + } +} + +public sealed class NoopAcousticEchoCanceller : IAcousticEchoCanceller +{ + public void Reset() + { + } + + public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio) + { + return microphone; + } +} + +public sealed class AdaptiveFilterAcousticEchoCanceller : IAcousticEchoCanceller +{ + private const int DefaultFilterLength = 256; + private const double DefaultAdaptationStep = 0.25; + private const double Epsilon = 1.0; + private readonly double[] coefficients; + private readonly double[] renderHistory; + private int historyPosition; + + public AdaptiveFilterAcousticEchoCanceller() + : this(DefaultFilterLength) + { + } + + public AdaptiveFilterAcousticEchoCanceller(int filterLength) + { + var effectiveFilterLength = Math.Max(16, filterLength); + coefficients = new double[effectiveFilterLength]; + renderHistory = new double[effectiveFilterLength]; + } + + public void Reset() + { + Array.Clear(coefficients); + Array.Clear(renderHistory); + historyPosition = 0; + } + + public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio) + { + if (microphone.Pcm.Length == 0 || + systemAudio.Pcm.Length == 0 || + microphone.SampleRate != systemAudio.SampleRate || + microphone.Channels != 1 || + systemAudio.Channels != 1) + { + return microphone; + } + + var byteCount = Math.Min(microphone.Pcm.Length, systemAudio.Pcm.Length); + byteCount -= byteCount % sizeof(short); + if (byteCount <= 0) + { + return microphone; + } + + var cleaned = new byte[microphone.Pcm.Length]; + for (var byteIndex = 0; byteIndex < byteCount; byteIndex += sizeof(short)) + { + var render = BitConverter.ToInt16(systemAudio.Pcm, byteIndex) / 32768.0; + var capture = BitConverter.ToInt16(microphone.Pcm, byteIndex) / 32768.0; + renderHistory[historyPosition] = render; + + var estimatedEcho = 0.0; + var energy = Epsilon; + for (var tap = 0; tap < coefficients.Length; tap++) + { + var sample = renderHistory[HistoryIndex(tap)]; + estimatedEcho += coefficients[tap] * sample; + energy += sample * sample; + } + + var error = capture - estimatedEcho; + var updateScale = DefaultAdaptationStep * error / energy; + for (var tap = 0; tap < coefficients.Length; tap++) + { + coefficients[tap] += updateScale * renderHistory[HistoryIndex(tap)]; + } + + WriteSample(cleaned, byteIndex, error); + historyPosition = historyPosition == 0 ? renderHistory.Length - 1 : historyPosition - 1; + } + + if (byteCount < microphone.Pcm.Length) + { + Buffer.BlockCopy(microphone.Pcm, byteCount, cleaned, byteCount, microphone.Pcm.Length - byteCount); + } + + return microphone with { Pcm = cleaned }; + } + + private int HistoryIndex(int offset) + { + var index = historyPosition + offset; + return index >= renderHistory.Length ? index - renderHistory.Length : index; + } + + private static void WriteSample(byte[] target, int byteIndex, double sample) + { + var scaled = Math.Clamp( + (int)Math.Round(sample * 32768.0), + short.MinValue, + short.MaxValue); + BitConverter.GetBytes((short)scaled).CopyTo(target, byteIndex); + } +} diff --git a/MeetingAssistant/Recording/AudioChunk.cs b/MeetingAssistant/Recording/AudioChunk.cs index 04ea2ec..6eb6cdb 100644 --- a/MeetingAssistant/Recording/AudioChunk.cs +++ b/MeetingAssistant/Recording/AudioChunk.cs @@ -1,6 +1,9 @@ namespace MeetingAssistant.Recording; -public sealed record AudioChunk(byte[] Pcm, int SampleRate, int Channels) +public sealed record AudioChunk( + byte[] Pcm, + int SampleRate, + int Channels) { public TimeSpan Duration { diff --git a/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs index baf9478..78ed8ba 100644 --- a/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs +++ b/MeetingAssistant/Recording/CompositeMeetingAudioSource.cs @@ -5,26 +5,44 @@ namespace MeetingAssistant.Recording; public sealed class CompositeMeetingAudioSource : IMeetingAudioSource { + private static readonly TimeSpan SourceSkewTimeout = TimeSpan.FromMilliseconds(250); private readonly IMeetingAudioSource microphone; private readonly IMeetingAudioSource systemAudio; + private readonly IAcousticEchoCancellerFactory echoCancellerFactory; private readonly ILogger logger; public CompositeMeetingAudioSource( IMeetingAudioSource microphone, IMeetingAudioSource systemAudio, + IAcousticEchoCancellerFactory echoCancellerFactory, ILogger logger) { this.microphone = microphone; this.systemAudio = systemAudio; + this.echoCancellerFactory = echoCancellerFactory; this.logger = logger; } public async IAsyncEnumerable CaptureAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { + await foreach (var chunk in CaptureAsync(new MeetingAssistantOptions(), cancellationToken)) + { + yield return chunk; + } + } + + public async IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var echoCanceller = echoCancellerFactory.Create(); + echoCanceller.Reset(); + var microphoneGain = NormalizeGain(options.Recording.MicrophoneMixGain); + var systemAudioGain = NormalizeGain(options.Recording.SystemAudioMixGain); var channel = Channel.CreateUnbounded(); - var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, channel.Writer, cancellationToken); - var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, channel.Writer, cancellationToken); + var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, options, channel.Writer, cancellationToken); + var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, options, channel.Writer, cancellationToken); _ = Task.WhenAll(microphoneTask, systemTask) .ContinueWith( task => @@ -41,8 +59,8 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); - AudioChunk? pendingMicrophone = null; - AudioChunk? pendingSystem = null; + var pendingMicrophone = new PendingAudioStream(); + var pendingSystem = new PendingAudioStream(); await foreach (var sourceChunk in channel.Reader.ReadAllAsync(cancellationToken)) { @@ -52,22 +70,32 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource sourceChunk.Kind); if (sourceChunk.Kind == AudioSourceKind.Microphone) { - pendingMicrophone = sourceChunk.Chunk; + pendingMicrophone.Enqueue(sourceChunk.Chunk); } else { - pendingSystem = sourceChunk.Chunk; + pendingSystem.Enqueue(sourceChunk.Chunk); } - if (pendingMicrophone is not null && pendingSystem is not null) + while (TryReadAlignedPair(pendingMicrophone, pendingSystem, out var microphoneChunk, out var systemChunk)) { - yield return Mix(pendingMicrophone, pendingSystem); - pendingMicrophone = null; - pendingSystem = null; - continue; + yield return CleanAndMix(microphoneChunk, systemChunk, echoCanceller, microphoneGain, systemAudioGain); } - using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + if (ShouldFlushTimedOutRemainder(pendingMicrophone, pendingSystem)) + { + while (TryReadRemainder(pendingMicrophone, pendingSystem, out var bufferedMicrophoneChunk, out var bufferedSystemChunk)) + { + yield return CleanAndMix( + bufferedMicrophoneChunk, + bufferedSystemChunk, + echoCanceller, + microphoneGain, + systemAudioGain); + } + } + + using var timeout = new CancellationTokenSource(SourceSkewTimeout); using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token); try { @@ -80,39 +108,33 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource { } - if (pendingMicrophone is not null) + while (TryReadRemainder(pendingMicrophone, pendingSystem, out var timeoutMicrophoneChunk, out var timeoutSystemChunk)) { - yield return pendingMicrophone; - pendingMicrophone = null; - } - - if (pendingSystem is not null) - { - yield return pendingSystem; - pendingSystem = null; + yield return CleanAndMix( + timeoutMicrophoneChunk, + timeoutSystemChunk, + echoCanceller, + microphoneGain, + systemAudioGain); } } - if (pendingMicrophone is not null) + while (TryReadRemainder(pendingMicrophone, pendingSystem, out var microphoneChunk, out var systemChunk)) { - yield return pendingMicrophone; - } - - if (pendingSystem is not null) - { - yield return pendingSystem; + yield return CleanAndMix(microphoneChunk, systemChunk, echoCanceller, microphoneGain, systemAudioGain); } } private static async Task PumpAsync( AudioSourceKind kind, IMeetingAudioSource source, + MeetingAssistantOptions options, ChannelWriter writer, CancellationToken cancellationToken) { try { - await foreach (var chunk in source.CaptureAsync(cancellationToken)) + await foreach (var chunk in source.CaptureAsync(options, cancellationToken)) { await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken); } @@ -128,32 +150,49 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource } } - private AudioChunk Mix(AudioChunk left, AudioChunk right) + private AudioChunk CleanAndMix( + AudioChunk microphoneChunk, + AudioChunk systemChunk, + IAcousticEchoCanceller echoCanceller, + double microphoneGain, + double systemAudioGain) { - if (left.SampleRate != right.SampleRate || left.Channels != right.Channels) + if (microphoneChunk.SampleRate != systemChunk.SampleRate || microphoneChunk.Channels != systemChunk.Channels) { logger.LogWarning( "Cannot mix audio chunks with different formats. Using microphone format {MicrophoneSampleRate}/{MicrophoneChannels} and dropping system chunk format {SystemSampleRate}/{SystemChannels}.", - left.SampleRate, - left.Channels, - right.SampleRate, - right.Channels); - return left; + microphoneChunk.SampleRate, + microphoneChunk.Channels, + systemChunk.SampleRate, + systemChunk.Channels); + return microphoneChunk; } - var mixed = new byte[Math.Max(left.Pcm.Length, right.Pcm.Length)]; + var cleanedMicrophone = echoCanceller.CleanMicrophone(microphoneChunk, systemChunk); + var mixed = new byte[Math.Max(cleanedMicrophone.Pcm.Length, systemChunk.Pcm.Length)]; var sampleCount = mixed.Length / sizeof(short); for (var sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) { var byteIndex = sampleIndex * sizeof(short); - var leftSample = ReadSample(left.Pcm, byteIndex); - var rightSample = ReadSample(right.Pcm, byteIndex); - var sample = Math.Clamp(leftSample + rightSample, short.MinValue, short.MaxValue); + var leftSample = ReadSample(cleanedMicrophone.Pcm, byteIndex); + var rightSample = ReadSample(systemChunk.Pcm, byteIndex); + var sample = Math.Clamp( + (int)Math.Round(leftSample * microphoneGain + rightSample * systemAudioGain), + short.MinValue, + short.MaxValue); BitConverter.GetBytes((short)sample).CopyTo(mixed, byteIndex); } - return new AudioChunk(mixed, left.SampleRate, left.Channels); + return new AudioChunk( + mixed, + cleanedMicrophone.SampleRate, + cleanedMicrophone.Channels); + } + + private static double NormalizeGain(double gain) + { + return double.IsFinite(gain) && gain >= 0 ? gain : 0; } private static int ReadSample(byte[] pcm, int byteIndex) @@ -161,6 +200,144 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource return byteIndex + 1 < pcm.Length ? BitConverter.ToInt16(pcm, byteIndex) : 0; } + private static bool TryReadAlignedPair( + PendingAudioStream microphone, + PendingAudioStream systemAudio, + out AudioChunk microphoneChunk, + out AudioChunk systemChunk) + { + microphoneChunk = null!; + systemChunk = null!; + var byteCount = Math.Min(microphone.ByteCount, systemAudio.ByteCount); + byteCount -= byteCount % sizeof(short); + if (byteCount <= 0 || microphone.Format is null || systemAudio.Format is null) + { + return false; + } + + microphoneChunk = microphone.Read(byteCount); + systemChunk = systemAudio.Read(byteCount); + return true; + } + + private static bool TryReadRemainder( + PendingAudioStream microphone, + PendingAudioStream systemAudio, + out AudioChunk microphoneChunk, + out AudioChunk systemChunk) + { + microphoneChunk = null!; + systemChunk = null!; + var byteCount = Math.Max(microphone.ByteCount, systemAudio.ByteCount); + byteCount -= byteCount % sizeof(short); + var format = microphone.Format ?? systemAudio.Format; + if (byteCount <= 0 || format is null) + { + return false; + } + + microphoneChunk = microphone.ByteCount > 0 + ? microphone.Read(Math.Min(byteCount, microphone.ByteCount)) + : new AudioChunk(new byte[byteCount], format.SampleRate, format.Channels); + systemChunk = systemAudio.ByteCount > 0 + ? systemAudio.Read(Math.Min(byteCount, systemAudio.ByteCount)) + : new AudioChunk(new byte[byteCount], format.SampleRate, format.Channels); + + if (microphoneChunk.Pcm.Length < byteCount) + { + microphoneChunk = microphoneChunk with { Pcm = PadSilence(microphoneChunk.Pcm, byteCount) }; + } + + if (systemChunk.Pcm.Length < byteCount) + { + systemChunk = systemChunk with { Pcm = PadSilence(systemChunk.Pcm, byteCount) }; + } + + return true; + } + + private static bool ShouldFlushTimedOutRemainder(PendingAudioStream microphone, PendingAudioStream systemAudio) + { + return microphone.ByteCount > 0 && systemAudio.ByteCount == 0 && microphone.HasBufferedFor(SourceSkewTimeout) || + systemAudio.ByteCount > 0 && microphone.ByteCount == 0 && systemAudio.HasBufferedFor(SourceSkewTimeout); + } + + private static byte[] PadSilence(byte[] pcm, int byteCount) + { + var padded = new byte[byteCount]; + Buffer.BlockCopy(pcm, 0, padded, 0, pcm.Length); + return padded; + } + + private sealed class PendingAudioStream + { + private readonly LinkedList> chunks = new(); + private long? firstBufferedAt; + + public int ByteCount { get; private set; } + + public AudioFormat? Format { get; private set; } + + public void Enqueue(AudioChunk chunk) + { + if (chunk.Pcm.Length == 0) + { + return; + } + + firstBufferedAt ??= Environment.TickCount64; + Format ??= new AudioFormat(chunk.SampleRate, chunk.Channels); + chunks.AddLast(new ArraySegment(chunk.Pcm)); + ByteCount += chunk.Pcm.Length; + } + + public bool HasBufferedFor(TimeSpan duration) + { + return firstBufferedAt is not null && + TimeSpan.FromMilliseconds(Environment.TickCount64 - firstBufferedAt.Value) >= duration; + } + + public AudioChunk Read(int byteCount) + { + if (Format is null) + { + throw new InvalidOperationException("Cannot read audio before a stream format has been captured."); + } + + var target = new byte[byteCount]; + var written = 0; + while (written < byteCount && chunks.First is not null) + { + var chunk = chunks.First.Value; + var copyCount = Math.Min(byteCount - written, chunk.Count); + Buffer.BlockCopy(chunk.Array!, chunk.Offset, target, written, copyCount); + written += copyCount; + ByteCount -= copyCount; + + if (copyCount == chunk.Count) + { + chunks.RemoveFirst(); + } + else + { + chunks.First.Value = new ArraySegment( + chunk.Array!, + chunk.Offset + copyCount, + chunk.Count - copyCount); + } + } + + if (ByteCount == 0) + { + firstBufferedAt = null; + } + + return new AudioChunk(target, Format.SampleRate, Format.Channels); + } + } + + private sealed record AudioFormat(int SampleRate, int Channels); + private sealed record SourceChunk(AudioSourceKind Kind, AudioChunk Chunk); private enum AudioSourceKind diff --git a/MeetingAssistant/Recording/IMeetingAudioSource.cs b/MeetingAssistant/Recording/IMeetingAudioSource.cs index 8519f95..5247f98 100644 --- a/MeetingAssistant/Recording/IMeetingAudioSource.cs +++ b/MeetingAssistant/Recording/IMeetingAudioSource.cs @@ -3,4 +3,11 @@ namespace MeetingAssistant.Recording; public interface IMeetingAudioSource { IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken); + + IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return CaptureAsync(cancellationToken); + } } diff --git a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs index c32b39e..dfc1bac 100644 --- a/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs +++ b/MeetingAssistant/Recording/MeetingRecordingCoordinator.cs @@ -485,7 +485,7 @@ public sealed class MeetingRecordingCoordinator long byteCount = 0; try { - await foreach (var chunk in audioSource.CaptureAsync(run.CaptureCancellation)) + await foreach (var chunk in audioSource.CaptureAsync(run.Options, run.CaptureCancellation)) { chunkCount++; byteCount += chunk.Pcm.Length; diff --git a/MeetingAssistant/Recording/NaudioCaptureSource.cs b/MeetingAssistant/Recording/NaudioCaptureSource.cs index 4ba70a3..aa74053 100644 --- a/MeetingAssistant/Recording/NaudioCaptureSource.cs +++ b/MeetingAssistant/Recording/NaudioCaptureSource.cs @@ -1,5 +1,4 @@ using System.Threading.Channels; -using Microsoft.Extensions.Options; using NAudio.CoreAudioApi; using NAudio.Wave; @@ -7,23 +6,29 @@ namespace MeetingAssistant.Recording; public sealed class MicrophoneAudioSource : IMeetingAudioSource { - private readonly MeetingAssistantOptions options; private readonly ILogger logger; - public MicrophoneAudioSource( - IOptions options, - ILogger logger) + public MicrophoneAudioSource(ILogger logger) { - this.options = options.Value; this.logger = logger; } public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) { - return CaptureAsync(new WasapiCapture(), cancellationToken); + return CaptureAsync(new MeetingAssistantOptions(), cancellationToken); } - private IAsyncEnumerable CaptureAsync(IWaveIn capture, CancellationToken cancellationToken) + public IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + CancellationToken cancellationToken) + { + return CaptureAsync(new WasapiCapture(), options, cancellationToken); + } + + private IAsyncEnumerable CaptureAsync( + IWaveIn capture, + MeetingAssistantOptions options, + CancellationToken cancellationToken) { capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels); return CaptureWith(capture, "microphone", logger, cancellationToken); @@ -94,18 +99,21 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource public sealed class SystemAudioSource : IMeetingAudioSource { - private readonly MeetingAssistantOptions options; private readonly ILogger logger; - public SystemAudioSource( - IOptions options, - ILogger logger) + public SystemAudioSource(ILogger logger) { - this.options = options.Value; this.logger = logger; } public IAsyncEnumerable CaptureAsync(CancellationToken cancellationToken) + { + return CaptureAsync(new MeetingAssistantOptions(), cancellationToken); + } + + public IAsyncEnumerable CaptureAsync( + MeetingAssistantOptions options, + CancellationToken cancellationToken) { var capture = new WasapiLoopbackCapture { diff --git a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs index 9f9acb8..c8b8694 100644 --- a/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs +++ b/MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs @@ -28,74 +28,74 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc var enumerator = audio.GetAsyncEnumerator(cancellationToken); try { - if (!await enumerator.MoveNextAsync()) - { - yield break; - } - - var firstChunk = enumerator.Current; - using var pushStream = AudioInputStream.CreatePushStream( - AudioStreamFormat.GetWaveFormatPCM( - (uint)firstChunk.SampleRate, - 16, - (byte)firstChunk.Channels)); - using var audioConfig = AudioConfig.FromStreamInput(pushStream); - var speechConfig = CreateSpeechConfig(); - using var recognizer = CreateTranscriber(speechConfig, audioConfig); - AddPhraseList(recognizer, pipelineOptions.DictationWords); - var segments = Channel.CreateUnbounded(); - - recognizer.Transcribed += (_, args) => - { - if (args.Result.Reason != ResultReason.RecognizedSpeech - || string.IsNullOrWhiteSpace(args.Result.Text)) + if (!await enumerator.MoveNextAsync()) { - return; + yield break; } - var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); - var segment = new TranscriptionSegment( - start, - start + args.Result.Duration, - NormalizeSpeakerId(args.Result.SpeakerId), - args.Result.Text.Trim()); - logger.LogInformation( - "Azure Speech emitted segment at {Start} from {Speaker}: {Text}", - segment.Start, - segment.Speaker, - segment.Text); - segments.Writer.TryWrite(segment); - }; - recognizer.Canceled += (_, args) => - { - if (args.Reason == CancellationReason.Error) + var firstChunk = enumerator.Current; + using var pushStream = AudioInputStream.CreatePushStream( + AudioStreamFormat.GetWaveFormatPCM( + (uint)firstChunk.SampleRate, + 16, + (byte)firstChunk.Channels)); + using var audioConfig = AudioConfig.FromStreamInput(pushStream); + var speechConfig = CreateSpeechConfig(); + using var recognizer = CreateTranscriber(speechConfig, audioConfig); + AddPhraseList(recognizer, pipelineOptions.DictationWords); + var segments = Channel.CreateUnbounded(); + + recognizer.Transcribed += (_, args) => { - segments.Writer.TryComplete(new InvalidOperationException( - $"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}")); - return; + if (args.Result.Reason != ResultReason.RecognizedSpeech + || string.IsNullOrWhiteSpace(args.Result.Text)) + { + return; + } + + var start = TimeSpan.FromTicks(args.Result.OffsetInTicks); + var segment = new TranscriptionSegment( + start, + start + args.Result.Duration, + NormalizeSpeakerId(args.Result.SpeakerId), + args.Result.Text.Trim()); + logger.LogInformation( + "Azure Speech emitted segment at {Start} from {Speaker}: {Text}", + segment.Start, + segment.Speaker, + segment.Text); + segments.Writer.TryWrite(segment); + }; + recognizer.Canceled += (_, args) => + { + if (args.Reason == CancellationReason.Error) + { + segments.Writer.TryComplete(new InvalidOperationException( + $"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}")); + return; + } + + segments.Writer.TryComplete(); + }; + + await recognizer.StartTranscribingAsync(); + var pumpTask = Task.Run( + () => PumpAudioAsync( + firstChunk, + enumerator, + pushStream, + recognizer, + segments.Writer, + options.AzureSpeech.RecognitionStopTimeout, + cancellationToken), + CancellationToken.None); + + await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) + { + yield return segment; } - segments.Writer.TryComplete(); - }; - - await recognizer.StartTranscribingAsync(); - var pumpTask = Task.Run( - () => PumpAudioAsync( - firstChunk, - enumerator, - pushStream, - recognizer, - segments.Writer, - options.AzureSpeech.RecognitionStopTimeout, - cancellationToken), - CancellationToken.None); - - await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken)) - { - yield return segment; - } - - await pumpTask.WaitAsync(cancellationToken); + await pumpTask.WaitAsync(cancellationToken); } finally { diff --git a/MeetingAssistant/appsettings.json b/MeetingAssistant/appsettings.json index 61f8ae8..2f03f4b 100644 --- a/MeetingAssistant/appsettings.json +++ b/MeetingAssistant/appsettings.json @@ -17,6 +17,8 @@ "TranscriptionProvider": "azure-speech", "SampleRate": 16000, "Channels": 1, + "MicrophoneMixGain": 1, + "SystemAudioMixGain": 1, "StopProcessingTimeout": "00:10:00", "MetadataLookupTimeout": "00:00:10", "BackgroundMetadataLookupTimeout": "00:01:00", diff --git a/README.md b/README.md index 0638db1..425b9f5 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` se "TranscriptionProvider": "funasr", "SampleRate": 16000, "Channels": 1, + "MicrophoneMixGain": 1, + "SystemAudioMixGain": 1, "StopProcessingTimeout": "00:10:00", "MaxMetadataAttendeeImportCount": 30, "TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings" @@ -230,11 +232,11 @@ Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` se `funasr` streams 16 kHz PCM audio to the configured FunASR WebSocket endpoint. When `FunAsr:Backend:Enabled` is true, Meeting Assistant manages a local Docker-backed FunASR runtime: on application startup it begins a non-blocking warm-up for the default launch profile and any named launch profile that selects `funasr`. That warm-up pulls the configured online streaming image, prepares the mounted model and hotword folder, replaces any stale container with the configured name, starts the two-pass WebSocket server mapped to the endpoint port, keeps the container alive after the FunASR startup script backgrounds the server process, and stops it when the app shuts down. Recording can start while the backend is still warming up; captured audio is queued and then streamed once the WebSocket backend is healthy. Docker commands are bounded by `CommandTimeout`, and backend WebSocket readiness is bounded by `StartupTimeout`. FunASR response fields such as `spk_name`, `spk`, and sentence-level speaker metadata are preserved in transcript segments; missing speaker fields are written as `Unknown`. -During recording, Meeting Assistant writes the mixed PCM stream to a temporary WAV under `TemporaryRecordingsFolder` only so the final diarization pass has audio to process. After capture stops and the streaming ASR provider drains the already-captured audio, FunASR Python `AutoModel` can run with VAD, punctuation, and CAMPPlus (`spk_model="cam++"`) over that temporary WAV. If sentence-level speaker labels are returned, the transcript markdown is rewritten with the final speaker-attributed segments. If final diarization is disabled or returns no segments, the live streaming transcript is kept. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. +During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription. The active launch profile's `Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during that final mix and default to `1`, because the signals are expected to be distinct after echo cancellation. The recorder writes only the mixed stream to the temporary WAV under `TemporaryRecordingsFolder`. After capture stops and the streaming ASR provider drains the already-captured audio, FunASR Python `AutoModel` can run with VAD, punctuation, and CAMPPlus (`spk_model="cam++"`) over that temporary WAV. If sentence-level speaker labels are returned, the transcript markdown is rewritten with the final speaker-attributed segments. If final diarization is disabled or returns no segments, the live streaming transcript is kept. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. `whisper-local` uses Whisper.NET with a local ggml model and remains available by setting `Recording:TranscriptionProvider` to `whisper-local`. The model file is not committed to this repository; place it at the configured `ModelPath` before using live transcription. When `WhisperLocal:Diarization:Enabled` is true, the final post-processing pass runs pyannote in Docker over the temporary WAV and reads the Hugging Face token from `WhisperLocal:Diarization:Token` or `TokenEnv` (`HF_TOKEN` by default). `AnnotationSource` selects pyannote's `speaker_diarization` or `exclusive_speaker_diarization` output. `AlignmentMode` selects whether Meeting Assistant assigns the best-overlap pyannote speaker label to each Whisper segment or rebuilds the finished transcript as pyannote speaker-turn segments with matching Whisper text. When `BuildImage` is true, Meeting Assistant builds the configured local pyannote Docker image if Docker does not already have it, so Python packages are installed once instead of during every diarization call. The configured `ModelsFolder` is mounted as the persistent pyannote model cache and stores Hugging Face, torch, and pip cache artifacts so model downloads are reused across runs. Active pyannote runtimes are also warmed up on application start so image setup and model download do not wait for the first diarization request. If pyannote is disabled, has no token, or returns no turns, the live Whisper transcript is kept. -`azure-speech` streams captured PCM audio to Azure AI Speech using the Speech SDK `ConversationTranscriber`. Configure `AzureSpeech:Endpoint`, `Region`, `Language`, and either `Key` or `KeyEnv`; this repository uses `AZURE_SPEECH_KEY` by default and does not store the key in appsettings. `DiarizeIntermediateResults` enables diarization for intermediate and final conversation transcription results. Azure returns generic speaker IDs such as `Guest-1` and `Guest-2`, which Meeting Assistant writes directly into live transcript segments. `RecognitionStopTimeout` bounds SDK shutdown after the captured audio stream has been closed; the default is 3 minutes so Azure has time to finalize longer speaker-recognition samples without letting diagnostic runs wait indefinitely. Since Azure Speech emits diarized live transcript segments, the Azure pipeline keeps those segments as the finished transcript instead of running pyannote. +`azure-speech` streams Meeting Assistant's captured PCM chunks to Azure AI Speech using the Speech SDK `ConversationTranscriber`; it does not use an Azure-owned microphone capture or MAS/AEC input path. Configure `AzureSpeech:Endpoint`, `Region`, `Language`, and either `Key` or `KeyEnv`; this repository uses `AZURE_SPEECH_KEY` by default and does not store the key in appsettings. `DiarizeIntermediateResults` enables diarization for intermediate and final conversation transcription results. Azure returns generic speaker IDs such as `Guest-1` and `Guest-2`, which Meeting Assistant writes directly into live transcript segments. `RecognitionStopTimeout` bounds SDK shutdown after the captured audio stream has been closed; the default is 3 minutes so Azure has time to finalize longer speaker-recognition samples without letting diagnostic runs wait indefinitely. Since Azure Speech emits diarized live transcript segments, the Azure pipeline keeps those segments as the finished transcript instead of running pyannote. Speaker identity matching keeps candidate samples only after a diarized speaker has at least `SpeakerIdentification:MinimumSampleSpeechDuration` of continuous speech, defaulting to 30 seconds. Adjacent same-speaker segments may be combined when the gap is no larger than `MaximumSampleSegmentGap`, but a different speaker resets the pending span. `SpeakerIdentification:PyannoteValidation` is an optional secondary confidence layer. When enabled, pyannote rejects multi-speaker samples and must confirm an Azure-confirmed identity match before Meeting Assistant accepts it. It uses the same Docker-based pyannote runtime shape as local Whisper finalization, reads its Hugging Face token from the nested diarization options, warms that runtime on application start, and defaults the validation command timeout to 1 hour because the local model can need substantial setup time. diff --git a/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/proposal.md b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/proposal.md new file mode 100644 index 0000000..0eb29f3 --- /dev/null +++ b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/proposal.md @@ -0,0 +1,18 @@ +# Add Local Echo Cancellation and MAS Assets + +## Why +The current mono meeting mix adds microphone and system samples directly, which can clip when both streams are loud. Speaker identity samples are extracted from this stream, so clipping and loopback bleed reduce sample quality. + +Azure Speech SDK's Microsoft Audio Stack can perform local acoustic echo cancellation when the Azure backend is used, but it does not expose reusable cleaned PCM chunks back to Meeting Assistant. Meeting Assistant should keep the MAS package/assets available for experiments while using an owned local echo cleanup stage that produces normal 16 kHz mono PCM chunks for storage and every ASR provider. + +## What Changes +- Capture microphone and system loopback as separate PCM streams inside the composite audio source. +- Clean the microphone stream with a local adaptive echo canceller that uses system loopback as the far-end reference. +- Mix the cleaned microphone and system streams by addition, with configurable gains still available. +- Write only the mixed temporary WAV used by transcription/finalization; separate channel files are not kept outside diagnostics. +- Keep Microsoft MAS package/assets in the publish output without exposing an Azure AEC runtime feature flag. + +## Impact +- Current mixed WAV output should contain less loopback bleed on the microphone side while preserving remote/system audio. +- Azure, FunASR, Whisper, pyannote, and speaker samples continue consuming 16 kHz mono PCM chunks. +- Sidecar temporary WAV files are diagnostic artifacts and are deleted with the main temporary recording. diff --git a/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/specs/meeting-recording/spec.md b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/specs/meeting-recording/spec.md new file mode 100644 index 0000000..2527bf4 --- /dev/null +++ b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/specs/meeting-recording/spec.md @@ -0,0 +1,60 @@ +## MODIFIED Requirements + +### Requirement: Meeting Assistant captures meeting audio +Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline. + +Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks. + +Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference. + +Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples. + +Meeting Assistant SHALL align microphone and system samples through per-source buffers before mixing and SHALL NOT emit normal live audio chunks that contain only one source while the other source is merely delayed. + +When one source stays quiet beyond the alignment timeout, Meeting Assistant SHALL mix the available source with synthetic silence for the missing source instead of blocking transcription. + +Meeting Assistant SHALL allow the final microphone/system mono mix to apply configurable microphone and system gain before combining samples. + +Meeting Assistant SHALL use the active run or launch profile recording options when configuring capture format and final microphone/system gains. + +Meeting Assistant SHALL clamp mixed samples after gain is applied. + +Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization. + +#### Scenario: Mixed audio uses cleaned microphone and system audio +- **GIVEN** the echo canceller cleans a microphone chunk to sample `2000` +- **AND** the matching system chunk has sample `10000` +- **WHEN** microphone and system chunks are mixed with gains `1` and `1` +- **THEN** the mixed sample is `12000` + +#### Scenario: Temporary recording stores only the mixed stream +- **GIVEN** Meeting Assistant has mixed microphone and system audio into one PCM chunk +- **WHEN** Meeting Assistant appends the chunk to the temporary recording +- **THEN** the main temporary WAV contains the mixed PCM +- **AND** no microphone or system sidecar WAV is written + +#### Scenario: Launch profile recording options configure capture and gains +- **GIVEN** an active launch profile configures sample format and microphone/system mix gains +- **WHEN** Meeting Assistant captures and mixes audio for that run +- **THEN** the microphone and system capture sources receive that launch profile recording configuration +- **AND** the mixed output uses that launch profile's microphone/system gains + +#### Scenario: Delayed sources are buffered before mixing +- **GIVEN** microphone audio arrives before matching system audio +- **WHEN** matching system audio arrives after a short delay +- **THEN** Meeting Assistant emits one mixed chunk for the aligned samples +- **AND** it does not emit separate microphone-only and system-only chunks for that delayed pair + +#### Scenario: Quiet system audio does not block microphone transcription +- **GIVEN** microphone audio arrives +- **AND** system loopback audio does not arrive within the alignment timeout +- **WHEN** Meeting Assistant mixes the available audio +- **THEN** it emits the microphone audio mixed with silent system audio +- **AND** live transcription can continue while system loopback is quiet + +#### Scenario: Continuous microphone audio does not suppress the alignment timeout +- **GIVEN** microphone audio keeps arriving +- **AND** system loopback audio stays unavailable past the alignment timeout +- **WHEN** Meeting Assistant checks the buffered microphone audio +- **THEN** it emits the buffered microphone audio mixed with silent system audio +- **AND** it does not wait indefinitely for a loopback chunk diff --git a/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/tasks.md b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/tasks.md new file mode 100644 index 0000000..384dc4e --- /dev/null +++ b/openspec/changes/archive/2026-05-29-add-audio-headroom-and-azure-aec/tasks.md @@ -0,0 +1,12 @@ +## 1. Implementation +- [x] Keep Microsoft MAS package/assets available without an Azure AEC runtime feature flag. +- [x] Capture and buffer microphone/system input streams separately before mixing. +- [x] Clean microphone chunks with a local echo cancellation stage before mixing. +- [x] Mix cleaned microphone and system audio by addition with configurable gains. +- [x] Keep temporary recording output to the mixed WAV; do not persist diagnostic channel sidecars. +- [x] Add focused tests for local echo cleanup, additive mixing, aligned mixing, and mixed-only temporary recording. + +## 2. Verification +- [x] Run focused tests. +- [x] Run `dotnet test MeetingAssistant.slnx`. +- [x] Run `openspec validate add-audio-headroom-and-azure-aec --strict`. diff --git a/openspec/specs/meeting-recording/spec.md b/openspec/specs/meeting-recording/spec.md index f50859b..6c9ef2a 100644 --- a/openspec/specs/meeting-recording/spec.md +++ b/openspec/specs/meeting-recording/spec.md @@ -35,10 +35,68 @@ When no recording is active, aborting SHALL leave the current recording status u ### Requirement: Recording mode captures microphone and computer output Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription. +Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline. + +Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks. + +Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference. + +Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples. + +Meeting Assistant SHALL align microphone and system samples through per-source buffers before mixing and SHALL NOT emit normal live audio chunks that contain only one source while the other source is merely delayed. + +When one source stays quiet beyond the alignment timeout, Meeting Assistant SHALL mix the available source with synthetic silence for the missing source instead of blocking transcription. + +Meeting Assistant SHALL allow the final microphone/system mono mix to apply configurable microphone and system gain before combining samples. + +Meeting Assistant SHALL use the active run or launch profile recording options when configuring capture format and final microphone/system gains. + +Meeting Assistant SHALL clamp mixed samples after gain is applied. + +Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization. + #### Scenario: Both sources produce audio - **WHEN** microphone and computer output audio chunks are available - **THEN** Meeting Assistant mixes them into one PCM stream before transcription +#### Scenario: Mixed audio uses cleaned microphone and system audio +- **GIVEN** the echo canceller cleans a microphone chunk to sample `2000` +- **AND** the matching system chunk has sample `10000` +- **WHEN** microphone and system chunks are mixed with gains `1` and `1` +- **THEN** the mixed sample is `12000` + +#### Scenario: Temporary recording stores only the mixed stream +- **GIVEN** Meeting Assistant has mixed microphone and system audio into one PCM chunk +- **WHEN** Meeting Assistant appends the chunk to the temporary recording +- **THEN** the main temporary WAV contains the mixed PCM +- **AND** no microphone or system sidecar WAV is written + +#### Scenario: Launch profile recording options configure capture and gains +- **GIVEN** an active launch profile configures sample format and microphone/system mix gains +- **WHEN** Meeting Assistant captures and mixes audio for that run +- **THEN** the microphone and system capture sources receive that launch profile recording configuration +- **AND** the mixed output uses that launch profile's microphone/system gains + +#### Scenario: Delayed sources are buffered before mixing +- **GIVEN** microphone audio arrives before matching system audio +- **WHEN** matching system audio arrives after a short delay +- **THEN** Meeting Assistant emits one mixed chunk for the aligned samples +- **AND** it does not emit separate microphone-only and system-only chunks for that delayed pair + +#### Scenario: Quiet system audio does not block microphone transcription +- **GIVEN** microphone audio arrives +- **AND** system loopback audio does not arrive within the alignment timeout +- **WHEN** Meeting Assistant mixes the available audio +- **THEN** it emits the microphone audio mixed with silent system audio +- **AND** live transcription can continue while system loopback is quiet + +#### Scenario: Continuous microphone audio does not suppress the alignment timeout +- **GIVEN** microphone audio keeps arriving +- **AND** system loopback audio stays unavailable past the alignment timeout +- **WHEN** Meeting Assistant checks the buffered microphone audio +- **THEN** it emits the buffered microphone audio mixed with silent system audio +- **AND** it does not wait indefinitely for a loopback chunk + #### Scenario: Device-level capture cannot be verified in tests - **WHEN** automated tests run without live audio devices - **THEN** Meeting Assistant verifies the audio mixer through deterministic source abstractions rather than depending on physical microphone or speaker devices