Improve meeting audio mixing pipeline

MeetingAssistant.Tests/AudioMixingTests.cs

@@ -6,40 +6,129 @@ namespace MeetingAssistant.Tests;
 public sealed class AudioMixingTests
 {
     [Fact]
-    public async Task CompositeAudioSourceMixesMicrophoneAndSystemAudioIntoOnePcmStream()
+    public async Task CompositeAudioSourceCleansMicrophoneBeforeAddingSystemAudio()
     {
-        var microphone = new FixedAudioSource(Pcm16(10_000));
-        var system = new FixedAudioSource(Pcm16(20_000));
-        var source = new CompositeMeetingAudioSource(
-            microphone,
-            system,
-            NullLogger<CompositeMeetingAudioSource>.Instance);
+        var microphone = new FixedAudioSource(Pcm16(12_000));
+        var system = new FixedAudioSource(Pcm16(10_000));
+        var source = CreateSource(microphone, system, new FixedEchoCanceller(Pcm16(2_000)));
 
         var chunks = await ReadChunks(source);
 
         Assert.Single(chunks);
-        Assert.Equal(30_000, BitConverter.ToInt16(chunks[0].Pcm));
+        Assert.Equal(12_000, BitConverter.ToInt16(chunks[0].Pcm));
     }
 
     [Fact]
-    public async Task CompositeAudioSourceClampsMixedSamples()
+    public async Task CompositeAudioSourceClampsAfterConfiguredGain()
     {
         var microphone = new FixedAudioSource(Pcm16(30_000));
         var system = new FixedAudioSource(Pcm16(10_000));
-        var source = new CompositeMeetingAudioSource(
-            microphone,
-            system,
-            NullLogger<CompositeMeetingAudioSource>.Instance);
+        var source = CreateSource(microphone, system);
 
         var chunks = await ReadChunks(source);
 
         Assert.Equal(short.MaxValue, BitConverter.ToInt16(chunks[0].Pcm));
     }
 
-    private static async Task<IReadOnlyList<AudioChunk>> ReadChunks(IMeetingAudioSource source)
+    [Fact]
+    public async Task CompositeAudioSourceUsesRunOptionsForChildSourcesAndGains()
+    {
+        var microphone = new OptionsCapturingAudioSource(Pcm16(10_000));
+        var system = new OptionsCapturingAudioSource(Pcm16(10_000));
+        var source = CreateSource(microphone, system);
+        var options = new MeetingAssistantOptions
+        {
+            Recording =
+            {
+                SampleRate = 24000,
+                Channels = 1,
+                MicrophoneMixGain = 0.5,
+                SystemAudioMixGain = 0.25
+            }
+        };
+
+        var chunks = await ReadChunks(source, options);
+
+        var chunk = Assert.Single(chunks);
+        Assert.Equal(7_500, BitConverter.ToInt16(chunk.Pcm));
+        Assert.Same(options, microphone.CapturedOptions);
+        Assert.Same(options, system.CapturedOptions);
+    }
+
+    [Fact]
+    public async Task CompositeAudioSourceWaitsForMatchingStreamsBeforeMixing()
+    {
+        var microphone = new DelayedAudioSource(TimeSpan.Zero, Pcm16(2_000));
+        var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(100), Pcm16(10_000));
+        var source = CreateSource(microphone, system);
+
+        var chunks = await ReadChunks(source);
+
+        var chunk = Assert.Single(chunks);
+        Assert.Equal(12_000, BitConverter.ToInt16(chunk.Pcm));
+    }
+
+    [Fact]
+    public async Task CompositeAudioSourceEmitsMicrophoneWithSilentSystemWhenSystemAudioIsQuiet()
+    {
+        var microphone = new DelayedAudioSource(TimeSpan.Zero, Pcm16(2_000));
+        var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(500), Pcm16(10_000));
+        var source = CreateSource(microphone, system);
+
+        var chunks = await ReadChunks(source);
+
+        Assert.Collection(
+            chunks,
+            first =>
+            {
+                Assert.Equal(2_000, BitConverter.ToInt16(first.Pcm));
+            },
+            second =>
+            {
+                Assert.Equal(10_000, BitConverter.ToInt16(second.Pcm));
+            });
+    }
+
+    [Fact]
+    public async Task CompositeAudioSourceFlushesContinuouslyArrivingMicrophoneBeforeLateSystemAudio()
+    {
+        var microphone = new RepeatedAudioSource(50, TimeSpan.FromMilliseconds(10), Pcm16(2_000));
+        var system = new DelayedAudioSource(TimeSpan.FromMilliseconds(750), Pcm16(10_000));
+        var source = CreateSource(microphone, system);
+
+        var chunks = await ReadChunks(source);
+
+        Assert.NotEmpty(chunks);
+        Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm));
+    }
+
+    [Fact]
+    public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal()
+    {
+        var canceller = new AdaptiveFilterAcousticEchoCanceller();
+        var system = Enumerable.Repeat(Pcm16(8_000), 80).SelectMany(bytes => bytes).ToArray();
+        var microphone = Enumerable.Repeat(Pcm16(8_000), 80).SelectMany(bytes => bytes).ToArray();
+
+        AudioChunk cleaned = new(microphone, 16000, 1);
+        for (var i = 0; i < 40; i++)
+        {
+            cleaned = canceller.CleanMicrophone(
+                new AudioChunk(microphone, 16000, 1),
+                new AudioChunk(system, 16000, 1));
+        }
+
+        var average = Enumerable.Range(0, cleaned.Pcm.Length / sizeof(short))
+            .Select(index => (double)Math.Abs(BitConverter.ToInt16(cleaned.Pcm, index * sizeof(short))))
+            .Average();
+        Assert.True(average < 1_000);
+    }
+
+    private static async Task<IReadOnlyList<AudioChunk>> ReadChunks(
+        IMeetingAudioSource source,
+        MeetingAssistantOptions? options = null)
     {
         var chunks = new List<AudioChunk>();
-        await foreach (var chunk in source.CaptureAsync(CancellationToken.None))
+        await foreach (var chunk in source.CaptureAsync(options ?? new MeetingAssistantOptions(), CancellationToken.None))
         {
             chunks.Add(chunk);
         }
@@ -52,6 +141,18 @@ public sealed class AudioMixingTests
         return BitConverter.GetBytes(sample);
     }
 
+    private static CompositeMeetingAudioSource CreateSource(
+        IMeetingAudioSource microphone,
+        IMeetingAudioSource system,
+        IAcousticEchoCanceller? echoCanceller = null)
+    {
+        return new CompositeMeetingAudioSource(
+            microphone,
+            system,
+            new FixedEchoCancellerFactory(echoCanceller ?? new NoopAcousticEchoCanceller()),
+            NullLogger<CompositeMeetingAudioSource>.Instance);
+    }
+
     private sealed class FixedAudioSource : IMeetingAudioSource
     {
         private readonly AudioChunk chunk;
@@ -68,4 +169,115 @@ public sealed class AudioMixingTests
             yield return chunk;
         }
     }
+
+    private sealed class DelayedAudioSource : IMeetingAudioSource
+    {
+        private readonly TimeSpan delay;
+        private readonly AudioChunk chunk;
+
+        public DelayedAudioSource(TimeSpan delay, byte[] pcm)
+        {
+            this.delay = delay;
+            chunk = new AudioChunk(pcm, 16000, 1);
+        }
+
+        public async IAsyncEnumerable<AudioChunk> CaptureAsync(
+            [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            if (delay > TimeSpan.Zero)
+            {
+                await Task.Delay(delay, cancellationToken);
+            }
+
+            yield return chunk;
+        }
+    }
+
+    private sealed class RepeatedAudioSource : IMeetingAudioSource
+    {
+        private readonly int count;
+        private readonly TimeSpan interval;
+        private readonly AudioChunk chunk;
+
+        public RepeatedAudioSource(int count, TimeSpan interval, byte[] pcm)
+        {
+            this.count = count;
+            this.interval = interval;
+            chunk = new AudioChunk(pcm, 16000, 1);
+        }
+
+        public async IAsyncEnumerable<AudioChunk> CaptureAsync(
+            [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            for (var index = 0; index < count; index++)
+            {
+                if (interval > TimeSpan.Zero)
+                {
+                    await Task.Delay(interval, cancellationToken);
+                }
+
+                yield return chunk;
+            }
+        }
+    }
+
+    private sealed class OptionsCapturingAudioSource : IMeetingAudioSource
+    {
+        private readonly AudioChunk chunk;
+
+        public OptionsCapturingAudioSource(byte[] pcm)
+        {
+            chunk = new AudioChunk(pcm, 16000, 1);
+        }
+
+        public MeetingAssistantOptions? CapturedOptions { get; private set; }
+
+        public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
+        {
+            return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
+        }
+
+        public async IAsyncEnumerable<AudioChunk> CaptureAsync(
+            MeetingAssistantOptions options,
+            [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            await Task.Yield();
+            CapturedOptions = options;
+            yield return chunk;
+        }
+    }
+
+    private sealed class FixedEchoCanceller : IAcousticEchoCanceller
+    {
+        private readonly byte[] cleanedPcm;
+
+        public FixedEchoCanceller(byte[] cleanedPcm)
+        {
+            this.cleanedPcm = cleanedPcm;
+        }
+
+        public void Reset()
+        {
+        }
+
+        public AudioChunk CleanMicrophone(AudioChunk microphone, AudioChunk systemAudio)
+        {
+            return microphone with { Pcm = cleanedPcm };
+        }
+    }
+
+    private sealed class FixedEchoCancellerFactory : IAcousticEchoCancellerFactory
+    {
+        private readonly IAcousticEchoCanceller echoCanceller;
+
+        public FixedEchoCancellerFactory(IAcousticEchoCanceller echoCanceller)
+        {
+            this.echoCanceller = echoCanceller;
+        }
+
+        public IAcousticEchoCanceller Create()
+        {
+            return echoCanceller;
+        }
+    }
 }

MeetingAssistant.Tests/RecordingCoordinatorTests.cs

@@ -1613,6 +1613,37 @@ public sealed class RecordingCoordinatorTests
         Assert.True(new FileInfo(session.AudioPath).Length > 44);
     }
 
+    [Fact]
+    public async Task TemporaryRecordedAudioStoreDoesNotWriteDiagnosticSidecars()
+    {
+        var recordingFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
+        var store = new TemporaryRecordedAudioStore(
+            Options.Create(new MeetingAssistantOptions
+            {
+                Recording = new RecordingOptions
+                {
+                    SampleRate = 16000,
+                    Channels = 1,
+                    TemporaryRecordingsFolder = recordingFolder
+                }
+            }),
+            NullLogger<TemporaryRecordedAudioStore>.Instance);
+
+        await using var session = await store.CreateSessionAsync(CancellationToken.None);
+        await session.AppendAsync(new AudioChunk([3, 0], 16000, 1), CancellationToken.None);
+        await session.CompleteAsync(CancellationToken.None);
+
+        var microphonePath = Path.Combine(
+            recordingFolder,
+            $"{Path.GetFileNameWithoutExtension(session.AudioPath)}-microphone.wav");
+        var systemPath = Path.Combine(
+            recordingFolder,
+            $"{Path.GetFileNameWithoutExtension(session.AudioPath)}-system.wav");
+        Assert.True(new FileInfo(session.AudioPath).Length > 44);
+        Assert.False(File.Exists(microphonePath));
+        Assert.False(File.Exists(systemPath));
+    }
+
     [Fact]
     public async Task TemporaryRecordedAudioStoreDeletesStaleRecordingsOnStartup()
     {