Improve meeting audio mixing pipeline

MeetingAssistant/Recording/AcousticEchoCanceller.cs

@@ -0,0 +1,130 @@
+namespace MeetingAssistant.Recording;
+
+public interface IAcousticEchoCanceller
+{
+    void Reset();
+
+    AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio);
+}
+
+public interface IAcousticEchoCancellerFactory
+{
+    IAcousticEchoCanceller Create();
+}
+
+public sealed class AdaptiveFilterAcousticEchoCancellerFactory : IAcousticEchoCancellerFactory
+{
+    public IAcousticEchoCanceller Create()
+    {
+        return new AdaptiveFilterAcousticEchoCanceller();
+    }
+}
+
+public sealed class NoopAcousticEchoCanceller : IAcousticEchoCanceller
+{
+    public void Reset()
+    {
+    }
+
+    public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio)
+    {
+        return microphone;
+    }
+}
+
+public sealed class AdaptiveFilterAcousticEchoCanceller : IAcousticEchoCanceller
+{
+    private const int DefaultFilterLength = 256;
+    private const double DefaultAdaptationStep = 0.25;
+    private const double Epsilon = 1.0;
+    private readonly double[] coefficients;
+    private readonly double[] renderHistory;
+    private int historyPosition;
+
+    public AdaptiveFilterAcousticEchoCanceller()
+        : this(DefaultFilterLength)
+    {
+    }
+
+    public AdaptiveFilterAcousticEchoCanceller(int filterLength)
+    {
+        var effectiveFilterLength = Math.Max(16, filterLength);
+        coefficients = new double[effectiveFilterLength];
+        renderHistory = new double[effectiveFilterLength];
+    }
+
+    public void Reset()
+    {
+        Array.Clear(coefficients);
+        Array.Clear(renderHistory);
+        historyPosition = 0;
+    }
+
+    public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio)
+    {
+        if (microphone.Pcm.Length == 0 ||
+            systemAudio.Pcm.Length == 0 ||
+            microphone.SampleRate != systemAudio.SampleRate ||
+            microphone.Channels != 1 ||
+            systemAudio.Channels != 1)
+        {
+            return microphone;
+        }
+
+        var byteCount = Math.Min(microphone.Pcm.Length, systemAudio.Pcm.Length);
+        byteCount -= byteCount % sizeof(short);
+        if (byteCount <= 0)
+        {
+            return microphone;
+        }
+
+        var cleaned = new byte[microphone.Pcm.Length];
+        for (var byteIndex = 0; byteIndex < byteCount; byteIndex += sizeof(short))
+        {
+            var render = BitConverter.ToInt16(systemAudio.Pcm, byteIndex) / 32768.0;
+            var capture = BitConverter.ToInt16(microphone.Pcm, byteIndex) / 32768.0;
+            renderHistory[historyPosition] = render;
+
+            var estimatedEcho = 0.0;
+            var energy = Epsilon;
+            for (var tap = 0; tap < coefficients.Length; tap++)
+            {
+                var sample = renderHistory[HistoryIndex(tap)];
+                estimatedEcho += coefficients[tap] * sample;
+                energy += sample * sample;
+            }
+
+            var error = capture - estimatedEcho;
+            var updateScale = DefaultAdaptationStep * error / energy;
+            for (var tap = 0; tap < coefficients.Length; tap++)
+            {
+                coefficients[tap] += updateScale * renderHistory[HistoryIndex(tap)];
+            }
+
+            WriteSample(cleaned, byteIndex, error);
+            historyPosition = historyPosition == 0 ? renderHistory.Length - 1 : historyPosition - 1;
+        }
+
+        if (byteCount < microphone.Pcm.Length)
+        {
+            Buffer.BlockCopy(microphone.Pcm, byteCount, cleaned, byteCount, microphone.Pcm.Length - byteCount);
+        }
+
+        return microphone with { Pcm = cleaned };
+    }
+
+    private int HistoryIndex(int offset)
+    {
+        var index = historyPosition + offset;
+        return index >= renderHistory.Length ? index - renderHistory.Length : index;
+    }
+
+    private static void WriteSample(byte[] target, int byteIndex, double sample)
+    {
+        var scaled = Math.Clamp(
+            (int)Math.Round(sample * 32768.0),
+            short.MinValue,
+            short.MaxValue);
+        BitConverter.GetBytes((short)scaled).CopyTo(target, byteIndex);
+    }
+}

MeetingAssistant/Recording/AudioChunk.cs

@@ -1,6 +1,9 @@
 namespace MeetingAssistant.Recording;
 
-public sealed record AudioChunk(byte[] Pcm, int SampleRate, int Channels)
+public sealed record AudioChunk(
+    byte[] Pcm,
+    int SampleRate,
+    int Channels)
 {
     public TimeSpan Duration
     {

MeetingAssistant/Recording/CompositeMeetingAudioSource.cs

@@ -5,26 +5,44 @@ namespace MeetingAssistant.Recording;
 
 public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
 {
+    private static readonly TimeSpan SourceSkewTimeout = TimeSpan.FromMilliseconds(250);
     private readonly IMeetingAudioSource microphone;
     private readonly IMeetingAudioSource systemAudio;
+    private readonly IAcousticEchoCancellerFactory echoCancellerFactory;
     private readonly ILogger<CompositeMeetingAudioSource> logger;
 
     public CompositeMeetingAudioSource(
         IMeetingAudioSource microphone,
         IMeetingAudioSource systemAudio,
+        IAcousticEchoCancellerFactory echoCancellerFactory,
         ILogger<CompositeMeetingAudioSource> logger)
     {
         this.microphone = microphone;
         this.systemAudio = systemAudio;
+        this.echoCancellerFactory = echoCancellerFactory;
         this.logger = logger;
     }
 
     public async IAsyncEnumerable<AudioChunk> CaptureAsync(
         [EnumeratorCancellation] CancellationToken cancellationToken)
     {
+        await foreach (var chunk in CaptureAsync(new MeetingAssistantOptions(), cancellationToken))
+        {
+            yield return chunk;
+        }
+    }
+
+    public async IAsyncEnumerable<AudioChunk> CaptureAsync(
+        MeetingAssistantOptions options,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        var echoCanceller = echoCancellerFactory.Create();
+        echoCanceller.Reset();
+        var microphoneGain = NormalizeGain(options.Recording.MicrophoneMixGain);
+        var systemAudioGain = NormalizeGain(options.Recording.SystemAudioMixGain);
         var channel = Channel.CreateUnbounded<SourceChunk>();
-        var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, channel.Writer, cancellationToken);
-        var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, channel.Writer, cancellationToken);
+        var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, options, channel.Writer, cancellationToken);
+        var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, options, channel.Writer, cancellationToken);
         _ = Task.WhenAll(microphoneTask, systemTask)
             .ContinueWith(
                 task =>
@@ -41,8 +59,8 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
                 TaskContinuationOptions.ExecuteSynchronously,
                 TaskScheduler.Default);
 
-        AudioChunk? pendingMicrophone = null;
-        AudioChunk? pendingSystem = null;
+        var pendingMicrophone = new PendingAudioStream();
+        var pendingSystem = new PendingAudioStream();
 
         await foreach (var sourceChunk in channel.Reader.ReadAllAsync(cancellationToken))
         {
@@ -52,22 +70,32 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
                 sourceChunk.Kind);
             if (sourceChunk.Kind == AudioSourceKind.Microphone)
             {
-                pendingMicrophone = sourceChunk.Chunk;
+                pendingMicrophone.Enqueue(sourceChunk.Chunk);
             }
             else
             {
-                pendingSystem = sourceChunk.Chunk;
+                pendingSystem.Enqueue(sourceChunk.Chunk);
             }
 
-            if (pendingMicrophone is not null && pendingSystem is not null)
+            while (TryReadAlignedPair(pendingMicrophone, pendingSystem, out var microphoneChunk, out var systemChunk))
             {
-                yield return Mix(pendingMicrophone, pendingSystem);
-                pendingMicrophone = null;
-                pendingSystem = null;
-                continue;
+                yield return CleanAndMix(microphoneChunk, systemChunk, echoCanceller, microphoneGain, systemAudioGain);
             }
 
-            using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(50));
+            if (ShouldFlushTimedOutRemainder(pendingMicrophone, pendingSystem))
+            {
+                while (TryReadRemainder(pendingMicrophone, pendingSystem, out var bufferedMicrophoneChunk, out var bufferedSystemChunk))
+                {
+                    yield return CleanAndMix(
+                        bufferedMicrophoneChunk,
+                        bufferedSystemChunk,
+                        echoCanceller,
+                        microphoneGain,
+                        systemAudioGain);
+                }
+            }
+
+            using var timeout = new CancellationTokenSource(SourceSkewTimeout);
             using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token);
             try
             {
@@ -80,39 +108,33 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
             {
             }
 
-            if (pendingMicrophone is not null)
+            while (TryReadRemainder(pendingMicrophone, pendingSystem, out var timeoutMicrophoneChunk, out var timeoutSystemChunk))
             {
-                yield return pendingMicrophone;
-                pendingMicrophone = null;
-            }
-
-            if (pendingSystem is not null)
-            {
-                yield return pendingSystem;
-                pendingSystem = null;
+                yield return CleanAndMix(
+                    timeoutMicrophoneChunk,
+                    timeoutSystemChunk,
+                    echoCanceller,
+                    microphoneGain,
+                    systemAudioGain);
             }
         }
 
-        if (pendingMicrophone is not null)
+        while (TryReadRemainder(pendingMicrophone, pendingSystem, out var microphoneChunk, out var systemChunk))
         {
-            yield return pendingMicrophone;
-        }
-
-        if (pendingSystem is not null)
-        {
-            yield return pendingSystem;
+            yield return CleanAndMix(microphoneChunk, systemChunk, echoCanceller, microphoneGain, systemAudioGain);
         }
     }
 
     private static async Task PumpAsync(
         AudioSourceKind kind,
         IMeetingAudioSource source,
+        MeetingAssistantOptions options,
         ChannelWriter<SourceChunk> writer,
         CancellationToken cancellationToken)
     {
         try
         {
-            await foreach (var chunk in source.CaptureAsync(cancellationToken))
+            await foreach (var chunk in source.CaptureAsync(options, cancellationToken))
             {
                 await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken);
             }
@@ -128,32 +150,49 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
         }
     }
 
-    private AudioChunk Mix(AudioChunk left, AudioChunk right)
+    private AudioChunk CleanAndMix(
+        AudioChunk microphoneChunk,
+        AudioChunk systemChunk,
+        IAcousticEchoCanceller echoCanceller,
+        double microphoneGain,
+        double systemAudioGain)
     {
-        if (left.SampleRate != right.SampleRate || left.Channels != right.Channels)
+        if (microphoneChunk.SampleRate != systemChunk.SampleRate || microphoneChunk.Channels != systemChunk.Channels)
         {
             logger.LogWarning(
                 "Cannot mix audio chunks with different formats. Using microphone format {MicrophoneSampleRate}/{MicrophoneChannels} and dropping system chunk format {SystemSampleRate}/{SystemChannels}.",
-                left.SampleRate,
-                left.Channels,
-                right.SampleRate,
-                right.Channels);
-            return left;
+                microphoneChunk.SampleRate,
+                microphoneChunk.Channels,
+                systemChunk.SampleRate,
+                systemChunk.Channels);
+            return microphoneChunk;
         }
 
-        var mixed = new byte[Math.Max(left.Pcm.Length, right.Pcm.Length)];
+        var cleanedMicrophone = echoCanceller.CleanMicrophone(microphoneChunk, systemChunk);
+        var mixed = new byte[Math.Max(cleanedMicrophone.Pcm.Length, systemChunk.Pcm.Length)];
         var sampleCount = mixed.Length / sizeof(short);
 
         for (var sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++)
         {
             var byteIndex = sampleIndex * sizeof(short);
-            var leftSample = ReadSample(left.Pcm, byteIndex);
-            var rightSample = ReadSample(right.Pcm, byteIndex);
-            var sample = Math.Clamp(leftSample + rightSample, short.MinValue, short.MaxValue);
+            var leftSample = ReadSample(cleanedMicrophone.Pcm, byteIndex);
+            var rightSample = ReadSample(systemChunk.Pcm, byteIndex);
+            var sample = Math.Clamp(
+                (int)Math.Round(leftSample * microphoneGain + rightSample * systemAudioGain),
+                short.MinValue,
+                short.MaxValue);
             BitConverter.GetBytes((short)sample).CopyTo(mixed, byteIndex);
         }
 
-        return new AudioChunk(mixed, left.SampleRate, left.Channels);
+        return new AudioChunk(
+            mixed,
+            cleanedMicrophone.SampleRate,
+            cleanedMicrophone.Channels);
+    }
+
+    private static double NormalizeGain(double gain)
+    {
+        return double.IsFinite(gain) && gain >= 0 ? gain : 0;
     }
 
     private static int ReadSample(byte[] pcm, int byteIndex)
@@ -161,6 +200,144 @@ public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
         return byteIndex + 1 < pcm.Length ? BitConverter.ToInt16(pcm, byteIndex) : 0;
     }
 
+    private static bool TryReadAlignedPair(
+        PendingAudioStream microphone,
+        PendingAudioStream systemAudio,
+        out AudioChunk microphoneChunk,
+        out AudioChunk systemChunk)
+    {
+        microphoneChunk = null!;
+        systemChunk = null!;
+        var byteCount = Math.Min(microphone.ByteCount, systemAudio.ByteCount);
+        byteCount -= byteCount % sizeof(short);
+        if (byteCount <= 0 || microphone.Format is null || systemAudio.Format is null)
+        {
+            return false;
+        }
+
+        microphoneChunk = microphone.Read(byteCount);
+        systemChunk = systemAudio.Read(byteCount);
+        return true;
+    }
+
+    private static bool TryReadRemainder(
+        PendingAudioStream microphone,
+        PendingAudioStream systemAudio,
+        out AudioChunk microphoneChunk,
+        out AudioChunk systemChunk)
+    {
+        microphoneChunk = null!;
+        systemChunk = null!;
+        var byteCount = Math.Max(microphone.ByteCount, systemAudio.ByteCount);
+        byteCount -= byteCount % sizeof(short);
+        var format = microphone.Format ?? systemAudio.Format;
+        if (byteCount <= 0 || format is null)
+        {
+            return false;
+        }
+
+        microphoneChunk = microphone.ByteCount > 0
+            ? microphone.Read(Math.Min(byteCount, microphone.ByteCount))
+            : new AudioChunk(new byte[byteCount], format.SampleRate, format.Channels);
+        systemChunk = systemAudio.ByteCount > 0
+            ? systemAudio.Read(Math.Min(byteCount, systemAudio.ByteCount))
+            : new AudioChunk(new byte[byteCount], format.SampleRate, format.Channels);
+
+        if (microphoneChunk.Pcm.Length < byteCount)
+        {
+            microphoneChunk = microphoneChunk with { Pcm = PadSilence(microphoneChunk.Pcm, byteCount) };
+        }
+
+        if (systemChunk.Pcm.Length < byteCount)
+        {
+            systemChunk = systemChunk with { Pcm = PadSilence(systemChunk.Pcm, byteCount) };
+        }
+
+        return true;
+    }
+
+    private static bool ShouldFlushTimedOutRemainder(PendingAudioStream microphone, PendingAudioStream systemAudio)
+    {
+        return microphone.ByteCount > 0 && systemAudio.ByteCount == 0 && microphone.HasBufferedFor(SourceSkewTimeout) ||
+            systemAudio.ByteCount > 0 && microphone.ByteCount == 0 && systemAudio.HasBufferedFor(SourceSkewTimeout);
+    }
+
+    private static byte[] PadSilence(byte[] pcm, int byteCount)
+    {
+        var padded = new byte[byteCount];
+        Buffer.BlockCopy(pcm, 0, padded, 0, pcm.Length);
+        return padded;
+    }
+
+    private sealed class PendingAudioStream
+    {
+        private readonly LinkedList<ArraySegment<byte>> chunks = new();
+        private long? firstBufferedAt;
+
+        public int ByteCount { get; private set; }
+
+        public AudioFormat? Format { get; private set; }
+
+        public void Enqueue(AudioChunk chunk)
+        {
+            if (chunk.Pcm.Length == 0)
+            {
+                return;
+            }
+
+            firstBufferedAt ??= Environment.TickCount64;
+            Format ??= new AudioFormat(chunk.SampleRate, chunk.Channels);
+            chunks.AddLast(new ArraySegment<byte>(chunk.Pcm));
+            ByteCount += chunk.Pcm.Length;
+        }
+
+        public bool HasBufferedFor(TimeSpan duration)
+        {
+            return firstBufferedAt is not null &&
+                TimeSpan.FromMilliseconds(Environment.TickCount64 - firstBufferedAt.Value) >= duration;
+        }
+
+        public AudioChunk Read(int byteCount)
+        {
+            if (Format is null)
+            {
+                throw new InvalidOperationException("Cannot read audio before a stream format has been captured.");
+            }
+
+            var target = new byte[byteCount];
+            var written = 0;
+            while (written < byteCount && chunks.First is not null)
+            {
+                var chunk = chunks.First.Value;
+                var copyCount = Math.Min(byteCount - written, chunk.Count);
+                Buffer.BlockCopy(chunk.Array!, chunk.Offset, target, written, copyCount);
+                written += copyCount;
+                ByteCount -= copyCount;
+
+                if (copyCount == chunk.Count)
+                {
+                    chunks.RemoveFirst();
+                }
+                else
+                {
+                    chunks.First.Value = new ArraySegment<byte>(
+                        chunk.Array!,
+                        chunk.Offset + copyCount,
+                        chunk.Count - copyCount);
+                }
+            }
+
+            if (ByteCount == 0)
+            {
+                firstBufferedAt = null;
+            }
+
+            return new AudioChunk(target, Format.SampleRate, Format.Channels);
+        }
+    }
+
+    private sealed record AudioFormat(int SampleRate, int Channels);
+
     private sealed record SourceChunk(AudioSourceKind Kind, AudioChunk Chunk);
 
     private enum AudioSourceKind

MeetingAssistant/Recording/IMeetingAudioSource.cs

@@ -3,4 +3,11 @@ namespace MeetingAssistant.Recording;
 public interface IMeetingAudioSource
 {
     IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken);
+
+    IAsyncEnumerable<AudioChunk> CaptureAsync(
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
+    {
+        return CaptureAsync(cancellationToken);
+    }
 }

MeetingAssistant/Recording/MeetingRecordingCoordinator.cs

@@ -485,7 +485,7 @@ public sealed class MeetingRecordingCoordinator
         long byteCount = 0;
         try
         {
-            await foreach (var chunk in audioSource.CaptureAsync(run.CaptureCancellation))
+            await foreach (var chunk in audioSource.CaptureAsync(run.Options, run.CaptureCancellation))
             {
                 chunkCount++;
                 byteCount += chunk.Pcm.Length;

MeetingAssistant/Recording/NaudioCaptureSource.cs

@@ -1,5 +1,4 @@
 using System.Threading.Channels;
-using Microsoft.Extensions.Options;
 using NAudio.CoreAudioApi;
 using NAudio.Wave;
 
@@ -7,23 +6,29 @@ namespace MeetingAssistant.Recording;
 
 public sealed class MicrophoneAudioSource : IMeetingAudioSource
 {
-    private readonly MeetingAssistantOptions options;
     private readonly ILogger<MicrophoneAudioSource> logger;
 
-    public MicrophoneAudioSource(
-        IOptions<MeetingAssistantOptions> options,
-        ILogger<MicrophoneAudioSource> logger)
+    public MicrophoneAudioSource(ILogger<MicrophoneAudioSource> logger)
     {
-        this.options = options.Value;
         this.logger = logger;
     }
 
     public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
     {
-        return CaptureAsync(new WasapiCapture(), cancellationToken);
+        return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
     }
 
-    private IAsyncEnumerable<AudioChunk> CaptureAsync(IWaveIn capture, CancellationToken cancellationToken)
+    public IAsyncEnumerable<AudioChunk> CaptureAsync(
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
+    {
+        return CaptureAsync(new WasapiCapture(), options, cancellationToken);
+    }
+
+    private IAsyncEnumerable<AudioChunk> CaptureAsync(
+        IWaveIn capture,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
     {
         capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
         return CaptureWith(capture, "microphone", logger, cancellationToken);
@@ -94,18 +99,21 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
 
 public sealed class SystemAudioSource : IMeetingAudioSource
 {
-    private readonly MeetingAssistantOptions options;
     private readonly ILogger<SystemAudioSource> logger;
 
-    public SystemAudioSource(
-        IOptions<MeetingAssistantOptions> options,
-        ILogger<SystemAudioSource> logger)
+    public SystemAudioSource(ILogger<SystemAudioSource> logger)
     {
-        this.options = options.Value;
         this.logger = logger;
     }
 
     public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
+    {
+        return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
+    }
+
+    public IAsyncEnumerable<AudioChunk> CaptureAsync(
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
     {
         var capture = new WasapiLoopbackCapture
         {