Harden speaker identity samples

2026-05-28 12:02:44 +02:00
parent c84f8a984a
commit a72cda0c03
23 changed files with 1251 additions and 123 deletions
@@ -118,9 +118,33 @@ public sealed class AzureSpeechSpeakerIdentityMatcherTests
        Assert.Null(match);
    }

+    [Fact]
+    public async Task MatcherReturnsNullWhenSecondaryValidatorRejectsPrimaryMatch()
+    {
+        var diarizationClient = new FakeSpeakerIdentityDiarizationClient(
+        [
+            new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Guest-1", "known"),
+            new TranscriptionSegment(TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(3), "Guest-1", "unknown")
+        ]);
+        var validator = new RejectingSpeakerIdentityMatchValidator();
+        var matcher = CreateMatcher(diarizationClient, validator: validator);
+        var snippet = CreateWavSnippet();
+
+        var match = await matcher.MatchAsync(
+            new SpeakerIdentityMatchRequest(
+                "Guest03",
+                snippet,
+                [new SpeakerIdentityMatchCandidate(42, "Chris", 5, [snippet])]),
+            CancellationToken.None);
+
+        Assert.Null(match);
+        Assert.Equal(1, validator.MatchValidationCount);
+    }
+
    private static AzureSpeechSpeakerIdentityMatcher CreateMatcher(
        ISpeakerIdentityDiarizationClient diarizationClient,
-        TimeSpan? matchTimeout = null)
+        TimeSpan? matchTimeout = null,
+        ISpeakerIdentityMatchValidator? validator = null)
    {
        return new AzureSpeechSpeakerIdentityMatcher(
            diarizationClient,
@@ -132,7 +156,8 @@ public sealed class AzureSpeechSpeakerIdentityMatcherTests
                    MatchTimeout = matchTimeout ?? TimeSpan.FromMinutes(1)
                }
            }),
-            NullLogger<AzureSpeechSpeakerIdentityMatcher>.Instance);
+            NullLogger<AzureSpeechSpeakerIdentityMatcher>.Instance,
+            validator ?? new NoopSpeakerIdentityMatchValidator());
    }

    private static byte[] CreateWavSnippet()
@@ -177,4 +202,22 @@ public sealed class AzureSpeechSpeakerIdentityMatcherTests
            return [];
        }
    }
+
+    private sealed class RejectingSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
+    {
+        public int MatchValidationCount { get; private set; }
+
+        public Task<bool> ValidateSampleAsync(byte[] wavBytes, CancellationToken cancellationToken)
+        {
+            return Task.FromResult(true);
+        }
+
+        public Task<bool> ValidateMatchAsync(
+            SpeakerIdentityMatchValidationRequest request,
+            CancellationToken cancellationToken)
+        {
+            MatchValidationCount++;
+            return Task.FromResult(false);
+        }
+    }
 }
@@ -0,0 +1,137 @@
+using MeetingAssistant;
+using MeetingAssistant.Recording;
+using MeetingAssistant.Speakers;
+using MeetingAssistant.Transcription;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using NAudio.Wave;
+
+namespace MeetingAssistant.Tests;
+
+public sealed class PyannoteSpeakerIdentityMatchValidatorTests
+{
+    [Fact]
+    public async Task ValidatorRejectsSampleWhenPyannoteReportsMultipleSpeakers()
+    {
+        var commandRunner = new CapturingCommandRunner(
+            """
+            __MEETING_ASSISTANT_PYANNOTE_JSON_START__
+            [{"start":0.0,"end":10.0,"speaker":"SPEAKER_00"},{"start":10.0,"end":20.0,"speaker":"SPEAKER_01"}]
+            __MEETING_ASSISTANT_PYANNOTE_JSON_END__
+            """);
+        var validator = CreateValidator(commandRunner);
+
+        var valid = await validator.ValidateSampleAsync(CreateWav(TimeSpan.FromSeconds(20)), CancellationToken.None);
+
+        Assert.False(valid);
+    }
+
+    [Fact]
+    public async Task ValidatorAcceptsMatchWhenPyannoteAssignsKnownAndUnknownSamplesToSameSpeaker()
+    {
+        var commandRunner = new CapturingCommandRunner(
+            """
+            __MEETING_ASSISTANT_PYANNOTE_JSON_START__
+            [{"start":0.0,"end":1.0,"speaker":"SPEAKER_00"},{"start":2.0,"end":3.0,"speaker":"SPEAKER_00"}]
+            __MEETING_ASSISTANT_PYANNOTE_JSON_END__
+            """);
+        var validator = CreateValidator(commandRunner);
+        var wav = CreateWav(TimeSpan.FromSeconds(1));
+
+        var valid = await validator.ValidateMatchAsync(
+            new SpeakerIdentityMatchValidationRequest("Guest03", 42, wav, [wav]),
+            CancellationToken.None);
+
+        Assert.True(valid);
+    }
+
+    [Fact]
+    public async Task DisabledValidatorDoesNotRunPyannote()
+    {
+        var commandRunner = new CapturingCommandRunner("");
+        var validator = CreateValidator(commandRunner, enabled: false);
+
+        var valid = await validator.ValidateSampleAsync(CreateWav(TimeSpan.FromSeconds(1)), CancellationToken.None);
+
+        Assert.True(valid);
+        Assert.Empty(commandRunner.Commands);
+    }
+
+    private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
+        CapturingCommandRunner commandRunner,
+        bool enabled = true)
+    {
+        var finalizer = new PyannoteTranscriptFinalizer(
+            commandRunner,
+            Options.Create(new MeetingAssistantOptions()),
+            NullLogger<PyannoteTranscriptFinalizer>.Instance);
+        return new PyannoteSpeakerIdentityMatchValidator(
+            finalizer,
+            Options.Create(new MeetingAssistantOptions
+            {
+                SpeakerIdentification = new SpeakerIdentificationOptions
+                {
+                    PyannoteValidation = new SpeakerIdentityPyannoteValidationOptions
+                    {
+                        Enabled = enabled,
+                        MinimumSingleSpeakerCoverage = 0.90,
+                        MinimumMatchingKnownSnippetRatio = 1,
+                        Diarization = new PyannoteDiarizationOptions
+                        {
+                            Enabled = true,
+                            BuildImage = false,
+                            DockerCommand = "docker",
+                            Image = "meeting-assistant-pyannote:local",
+                            ModelsFolder = Path.Combine(
+                                Path.GetTempPath(),
+                                "meeting-assistant-tests",
+                                Guid.NewGuid().ToString("N"),
+                                "models"),
+                            Token = "hf_test",
+                            TokenEnv = "",
+                            CommandTimeout = TimeSpan.FromMinutes(1),
+                            AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
+                        }
+                    }
+                }
+            }),
+            NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
+    }
+
+    private static byte[] CreateWav(TimeSpan duration)
+    {
+        const int sampleRate = 16000;
+        using var stream = new MemoryStream();
+        using (var writer = new WaveFileWriter(stream, new WaveFormat(sampleRate, 16, 1)))
+        {
+            var bytes = new byte[(int)(duration.TotalSeconds * sampleRate * sizeof(short))];
+            writer.Write(bytes, 0, bytes.Length);
+        }
+
+        return stream.ToArray();
+    }
+
+    private sealed class CapturingCommandRunner : ICommandRunner
+    {
+        private readonly string output;
+
+        public CapturingCommandRunner(string output)
+        {
+            this.output = output;
+        }
+
+        public IReadOnlyList<CapturedCommand> Commands { get; private set; } = [];
+
+        public Task<CommandResult> RunAsync(
+            string fileName,
+            IReadOnlyList<string> arguments,
+            CancellationToken cancellationToken,
+            IReadOnlyDictionary<string, string>? environment = null)
+        {
+            Commands = Commands.Append(new CapturedCommand(fileName, arguments)).ToList();
+            return Task.FromResult(new CommandResult(0, output, ""));
+        }
+    }
+
+    private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
+}
@@ -984,7 +984,8 @@ public sealed class RecordingCoordinatorTests
                SpeakerIdentification = new SpeakerIdentificationOptions
                {
                    InitialDelay = TimeSpan.Zero,
-                    Interval = TimeSpan.FromMilliseconds(10)
+                    Interval = TimeSpan.FromMilliseconds(10),
+                    MinimumSampleSpeechDuration = TimeSpan.Zero
                }
            }),
            NullLogger<MeetingRecordingCoordinator>.Instance,
@@ -1031,7 +1032,8 @@ public sealed class RecordingCoordinatorTests
                SpeakerIdentification = new SpeakerIdentificationOptions
                {
                    InitialDelay = TimeSpan.Zero,
-                    Interval = TimeSpan.FromMilliseconds(10)
+                    Interval = TimeSpan.FromMilliseconds(10),
+                    MinimumSampleSpeechDuration = TimeSpan.Zero
                }
            }),
            NullLogger<MeetingRecordingCoordinator>.Instance,
@@ -1066,7 +1068,8 @@ public sealed class RecordingCoordinatorTests
                SpeakerIdentification = new SpeakerIdentificationOptions
                {
                    InitialDelay = TimeSpan.Zero,
-                    Interval = TimeSpan.FromMilliseconds(20)
+                    Interval = TimeSpan.FromMilliseconds(20),
+                    MinimumSampleSpeechDuration = TimeSpan.Zero
                }
            }),
            NullLogger<MeetingRecordingCoordinator>.Instance,
@@ -1103,7 +1106,8 @@ public sealed class RecordingCoordinatorTests
                SpeakerIdentification = new SpeakerIdentificationOptions
                {
                    InitialDelay = TimeSpan.Zero,
-                    Interval = TimeSpan.FromMilliseconds(20)
+                    Interval = TimeSpan.FromMilliseconds(20),
+                    MinimumSampleSpeechDuration = TimeSpan.Zero
                }
            }),
            NullLogger<MeetingRecordingCoordinator>.Instance,
@@ -1672,6 +1676,7 @@ public sealed class RecordingCoordinatorTests
            values["MeetingAssistant:SpeakerIdentification:Enabled"] = "true";
            values["MeetingAssistant:SpeakerIdentification:InitialDelay"] = "00:00:00";
            values["MeetingAssistant:SpeakerIdentification:Interval"] = "00:00:00.050";
+            values["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:00:00";
        }

        var configuration = new ConfigurationBuilder()
@@ -1708,7 +1713,7 @@ public sealed class RecordingCoordinatorTests
            var deadline = DateTimeOffset.UtcNow.AddSeconds(5);
            while (DateTimeOffset.UtcNow < deadline)
            {
-                if (segments.Any(segment => segment.Text?.Contains(text, StringComparison.Ordinal) == true))
+                if (segments.Any(segment => segment?.Text?.Contains(text, StringComparison.Ordinal) == true))
                {
                    return;
                }
@@ -0,0 +1,76 @@
+using MeetingAssistant.Recording;
+using MeetingAssistant.Transcription;
+using NAudio.Wave;
+
+namespace MeetingAssistant.Tests;
+
+public sealed class SpeakerAudioSampleCollectorTests
+{
+    [Fact]
+    public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
+    {
+        var collector = new SpeakerAudioSampleCollector(
+            TimeSpan.FromMinutes(2),
+            maxSamplesPerSpeaker: 3,
+            minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
+            maximumSegmentGap: TimeSpan.FromSeconds(1));
+        collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
+
+        collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
+        collector.TryAdd(Segment(12, 24, "Guest01", "six seven eight nine ten."));
+
+        Assert.Empty(collector.Snapshot());
+
+        collector.TryAdd(Segment(24, 31, "Guest01", "eleven twelve thirteen fourteen fifteen."));
+
+        var sample = Assert.Single(collector.Snapshot());
+        Assert.Equal("Guest01", sample.Speaker);
+        Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
+        Assert.Equal(TimeSpan.FromSeconds(31), sample.Segment.End);
+        Assert.True(ReadDuration(sample.WavBytes) >= TimeSpan.FromSeconds(30));
+    }
+
+    [Fact]
+    public void CollectorDoesNotCombineSpeechAcrossDifferentSpeakerInterruption()
+    {
+        var collector = new SpeakerAudioSampleCollector(
+            TimeSpan.FromMinutes(2),
+            maxSamplesPerSpeaker: 3,
+            minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
+            maximumSegmentGap: TimeSpan.FromSeconds(1));
+        collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
+
+        collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
+        collector.TryAdd(Segment(20, 22, "Guest02", "interrupting now."));
+        collector.TryAdd(Segment(22, 35, "Guest01", "six seven eight nine ten."));
+
+        Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
+    }
+
+    private static TranscriptionSegment Segment(
+        double start,
+        double end,
+        string speaker,
+        string text)
+    {
+        return new TranscriptionSegment(
+            TimeSpan.FromSeconds(start),
+            TimeSpan.FromSeconds(end),
+            speaker,
+            text);
+    }
+
+    private static AudioChunk CreateAudio(TimeSpan duration)
+    {
+        const int sampleRate = 16000;
+        const int channels = 1;
+        var bytes = new byte[(int)(duration.TotalSeconds * sampleRate * channels * sizeof(short))];
+        return new AudioChunk(bytes, sampleRate, channels);
+    }
+
+    private static TimeSpan ReadDuration(byte[] wavBytes)
+    {
+        using var reader = new WaveFileReader(new MemoryStream(wavBytes));
+        return reader.TotalTime;
+    }
+}
@@ -199,6 +199,60 @@ public sealed class SpeakerIdentityServiceTests
        Assert.Contains("Guest01 was identified as Manuel", File.ReadAllText(learned.References.Single().TranscriptPath));
    }

+    [Fact]
+    public async Task UnmatchedSpeakerIsNotLearnedWhenSecondaryValidationRejectsSample()
+    {
+        await using var fixture = await SpeakerIdentityFixture.CreateAsync();
+        fixture.MatchValidator.SampleIsValid = false;
+        var service = fixture.CreateService();
+
+        await service.ProcessFinishedTranscriptAsync(
+            fixture.CreateRequest(
+                ["Manuel"],
+                [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "unknown")]),
+            CancellationToken.None);
+
+        Assert.Empty(fixture.Context.SpeakerIdentities);
+    }
+
+    [Fact]
+    public async Task FinishedSnippetExtractionWaitsForMinimumContinuousSpeechDuration()
+    {
+        await using var fixture = await SpeakerIdentityFixture.CreateAsync(options =>
+            options.MinimumSampleSpeechDuration = TimeSpan.FromSeconds(30));
+        var service = fixture.CreateService();
+
+        await service.ProcessFinishedTranscriptAsync(
+            fixture.CreateRequest(
+                ["Manuel"],
+                [new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(20), "Guest01", "short")]),
+            CancellationToken.None);
+
+        Assert.Equal(0, fixture.SnippetExtractor.ExtractCalls);
+        Assert.Empty(fixture.Context.SpeakerIdentities);
+    }
+
+    [Fact]
+    public async Task FinishedSnippetExtractionDoesNotCombineAcrossSpeakerInterruption()
+    {
+        await using var fixture = await SpeakerIdentityFixture.CreateAsync(options =>
+            options.MinimumSampleSpeechDuration = TimeSpan.FromSeconds(30));
+        var service = fixture.CreateService();
+
+        await service.ProcessFinishedTranscriptAsync(
+            fixture.CreateRequest(
+                ["Manuel"],
+                [
+                    new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(20), "Guest01", "first"),
+                    new TranscriptionSegment(TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(22), "Guest02", "interrupting"),
+                    new TranscriptionSegment(TimeSpan.FromSeconds(22), TimeSpan.FromSeconds(35), "Guest01", "second")
+                ]),
+            CancellationToken.None);
+
+        Assert.Equal(0, fixture.SnippetExtractor.ExtractCalls);
+        Assert.Empty(fixture.Context.SpeakerIdentities);
+    }
+
    [Fact]
    public async Task SuppliedSpeakerSamplesAreUsedBeforeExtractingFromAudioFile()
    {
@@ -450,6 +504,87 @@ public sealed class SpeakerIdentityServiceTests
        File.Delete(dbPath);
    }

+    [Fact]
+    public async Task SpeakerOverrideCreatesIdentityWhenLegacyTranscriptionCountHasNoDatabaseDefault()
+    {
+        var dbPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", $"{Guid.NewGuid():N}.db");
+        Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!);
+        await using var context = new SpeakerIdentityDbContext(new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
+            .UseSqlite($"Data Source={dbPath};Pooling=False")
+            .Options);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE TABLE "SpeakerIdentities" (
+                "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerIdentities" PRIMARY KEY AUTOINCREMENT,
+                "CanonicalName" TEXT NULL,
+                "TranscriptionCount" INTEGER NOT NULL,
+                "CreatedAt" TEXT NOT NULL,
+                "UpdatedAt" TEXT NOT NULL
+            );
+            """);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE TABLE "SpeakerCandidateNames" (
+                "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerCandidateNames" PRIMARY KEY AUTOINCREMENT,
+                "SpeakerIdentityId" INTEGER NOT NULL,
+                "Name" TEXT NOT NULL,
+                CONSTRAINT "FK_SpeakerCandidateNames_SpeakerIdentities_SpeakerIdentityId"
+                    FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
+            );
+            """);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE TABLE "SpeakerSnippets" (
+                "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerSnippets" PRIMARY KEY AUTOINCREMENT,
+                "SpeakerIdentityId" INTEGER NOT NULL,
+                "WavBytes" BLOB NOT NULL,
+                "CreatedAt" TEXT NOT NULL,
+                CONSTRAINT "FK_SpeakerSnippets_SpeakerIdentities_SpeakerIdentityId"
+                    FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
+            );
+            """);
+        var options = new SpeakerIdentificationOptions
+        {
+            DatabasePath = dbPath,
+            MinimumSampleSpeechDuration = TimeSpan.Zero
+        };
+        var service = new SpeakerIdentityService(
+            new TestSpeakerIdentityDbContextFactory(dbPath),
+            new FakeSpeakerSnippetExtractor(),
+            new FakeSpeakerIdentityMatcher(),
+            Options.Create(new MeetingAssistantOptions { SpeakerIdentification = options }),
+            NullLogger<SpeakerIdentityService>.Instance,
+            new FakeSpeakerIdentityMatchValidator());
+        var segment = new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(5), "Guest-01", "hello from Daniel");
+        var root = Path.GetDirectoryName(dbPath)!;
+        var transcriptPath = Path.Combine(root, "transcript.md");
+        await File.WriteAllTextAsync(transcriptPath, "Transcript");
+        var request = new SpeakerIdentificationRequest(
+            "test.wav",
+            new MeetingNote(
+                Path.Combine(root, "meeting.md"),
+                new MeetingNoteFrontmatter
+                {
+                    Title = "Test Meeting",
+                    StartTime = DateTimeOffset.Parse("2026-05-28T10:00:00+02:00"),
+                    Attendees = ["Hecht, Daniel"],
+                    Transcript = transcriptPath,
+                    AssistantContext = Path.Combine(root, "context.md"),
+                    Summary = Path.Combine(root, "summary.md")
+                },
+                ""),
+            [segment],
+            [new SpeakerAudioSample("Guest-01", segment, [8, 8, 8], 95)]);
+
+        await service.ApplySpeakerOverrideAsync(request, "Guest-01", "Hecht, Daniel", CancellationToken.None);
+
+        context.ChangeTracker.Clear();
+        var saved = await context.SpeakerIdentities.SingleAsync();
+        Assert.Equal("Hecht, Daniel", saved.CanonicalName);
+        Assert.Equal(0, saved.TranscriptionCount);
+        File.Delete(dbPath);
+    }
+
    private sealed class SpeakerIdentityFixture : IAsyncDisposable
    {
        private readonly string tempDirectory;
@@ -472,6 +607,8 @@ public sealed class SpeakerIdentityServiceTests

        public FakeSpeakerIdentityMatcher Matcher { get; } = new();

+        public FakeSpeakerIdentityMatchValidator MatchValidator { get; } = new();
+
        public FakeSpeakerSnippetExtractor SnippetExtractor { get; } = new();

        public static async Task<SpeakerIdentityFixture> CreateAsync(
@@ -484,7 +621,8 @@ public sealed class SpeakerIdentityServiceTests
            {
                DatabasePath = dbPath,
                MaxSnippetsPerSpeaker = 3,
-                MatchBatchSize = 6
+                MatchBatchSize = 6,
+                MinimumSampleSpeechDuration = TimeSpan.Zero
            };
            configureOptions?.Invoke(options);
            var context = new SpeakerIdentityDbContext(new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
@@ -501,7 +639,8 @@ public sealed class SpeakerIdentityServiceTests
                SnippetExtractor,
                Matcher,
                Options.Create(new MeetingAssistantOptions { SpeakerIdentification = options }),
-                NullLogger<SpeakerIdentityService>.Instance);
+                NullLogger<SpeakerIdentityService>.Instance,
+                MatchValidator);
        }

        public SpeakerIdentityAttendeeCanonicalizer CreateAttendeeCanonicalizer()
@@ -627,6 +766,11 @@ public sealed class SpeakerIdentityServiceTests
            IReadOnlyList<TranscriptionSegment> speakerSegments,
            CancellationToken cancellationToken)
        {
+            if (speakerSegments.Count == 0)
+            {
+                return Task.FromResult<byte[]>([]);
+            }
+
            ExtractCalls++;
            return Task.FromResult<byte[]>([7, 8, 9]);
        }
@@ -657,4 +801,23 @@ public sealed class SpeakerIdentityServiceTests
            return Task.FromResult<SpeakerIdentityMatch?>(new SpeakerIdentityMatch(MatchIdentityId.Value));
        }
    }
+
+    private sealed class FakeSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
+    {
+        public bool SampleIsValid { get; set; } = true;
+
+        public Task<bool> ValidateSampleAsync(
+            byte[] wavBytes,
+            CancellationToken cancellationToken)
+        {
+            return Task.FromResult(SampleIsValid);
+        }
+
+        public Task<bool> ValidateMatchAsync(
+            SpeakerIdentityMatchValidationRequest request,
+            CancellationToken cancellationToken)
+        {
+            return Task.FromResult(true);
+        }
+    }
 }