Harden speaker identity samples

MeetingAssistant/MeetingAssistantOptions.cs

@@ -293,6 +293,10 @@ public sealed class SpeakerIdentificationOptions
 
     public int MaxSnippetsPerSpeaker { get; set; } = 3;
 
+    public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
+
+    public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
+
     public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
 
     public TimeSpan LiveSampleBufferDuration { get; set; } = TimeSpan.FromMinutes(10);
@@ -302,6 +306,23 @@ public sealed class SpeakerIdentificationOptions
     public TimeSpan MatchTimeout { get; set; } = TimeSpan.FromMinutes(3);
 
     public AzureSpeechOptions AzureSpeech { get; set; } = new();
+
+    public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
+}
+
+public sealed class SpeakerIdentityPyannoteValidationOptions
+{
+    public bool Enabled { get; set; }
+
+    public double MinimumSingleSpeakerCoverage { get; set; } = 0.90;
+
+    public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
+
+    public PyannoteDiarizationOptions Diarization { get; set; } = new()
+    {
+        Enabled = true,
+        AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
+    };
 }
 
 public sealed class AgentOptions

MeetingAssistant/Program.cs

@@ -50,6 +50,7 @@ builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOpti
 });
 builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtractor>();
 builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
+builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
 builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
 builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
 builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();

MeetingAssistant/Recording/MeetingRecordingCoordinator.cs

@@ -1334,7 +1334,11 @@ public sealed class MeetingRecordingCoordinator
             Options = options;
             LaunchProfileName = launchProfileName;
             pipelines.Add(pipeline);
-            speakerSampleCollector = new SpeakerAudioSampleCollector(liveSampleBufferDuration, maxSpeakerSamples);
+            speakerSampleCollector = new SpeakerAudioSampleCollector(
+                liveSampleBufferDuration,
+                maxSpeakerSamples,
+                options.SpeakerIdentification.MinimumSampleSpeechDuration,
+                options.SpeakerIdentification.MaximumSampleSegmentGap);
         }
 
         public CancellationTokenSource CaptureCancellationSource { get; }

MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs

@@ -9,11 +9,33 @@ internal sealed class SpeakerAudioSampleCollector
     private readonly RollingAudioBuffer audioBuffer;
     private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
     private readonly int maxSamplesPerSpeaker;
+    private readonly TimeSpan minimumUninterruptedSpeechDuration;
+    private readonly TimeSpan maximumSegmentGap;
+    private PendingSpeakerSpan? pendingSpan;
 
     public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
+        : this(
+            bufferDuration,
+            maxSamplesPerSpeaker,
+            TimeSpan.FromSeconds(30),
+            TimeSpan.FromSeconds(1))
+    {
+    }
+
+    public SpeakerAudioSampleCollector(
+        TimeSpan bufferDuration,
+        int maxSamplesPerSpeaker,
+        TimeSpan minimumUninterruptedSpeechDuration,
+        TimeSpan maximumSegmentGap)
     {
         audioBuffer = new RollingAudioBuffer(bufferDuration);
         this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
+        this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
+            ? minimumUninterruptedSpeechDuration
+            : TimeSpan.Zero;
+        this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
+            ? maximumSegmentGap
+            : TimeSpan.Zero;
     }
 
     public void AppendAudio(AudioChunk chunk)
@@ -26,6 +48,7 @@ internal sealed class SpeakerAudioSampleCollector
         lock (gate)
         {
             samplesBySpeaker.Clear();
+            pendingSpan = null;
             audioBuffer.Reset();
         }
     }
@@ -37,25 +60,31 @@ internal sealed class SpeakerAudioSampleCollector
             return;
         }
 
-        var score = Score(segment);
+        TranscriptionSegment sampleSegment;
+        lock (gate)
+        {
+            sampleSegment = ExtendPendingSpan(segment);
+        }
+
+        var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
         if (score <= 0)
         {
             return;
         }
 
-        var wavBytes = audioBuffer.TryExtractWav(segment.Start, segment.End);
+        var wavBytes = audioBuffer.TryExtractWav(sampleSegment.Start, sampleSegment.End);
         if (wavBytes.Length == 0)
         {
             return;
         }
 
-        var sample = new SpeakerAudioSample(segment.Speaker, segment, wavBytes, score);
+        var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score);
         lock (gate)
         {
-            if (!samplesBySpeaker.TryGetValue(segment.Speaker, out var samples))
+            if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
             {
                 samples = [];
-                samplesBySpeaker[segment.Speaker] = samples;
+                samplesBySpeaker[sampleSegment.Speaker] = samples;
             }
 
             samples.Add(sample);
@@ -86,10 +115,29 @@ internal sealed class SpeakerAudioSampleCollector
             !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
     }
 
-    private static double Score(TranscriptionSegment segment)
+    private TranscriptionSegment ExtendPendingSpan(TranscriptionSegment segment)
+    {
+        if (pendingSpan is null ||
+            !SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
+        {
+            pendingSpan = new PendingSpeakerSpan(
+                segment.Speaker,
+                segment.Start,
+                segment.End,
+                [segment.Text]);
+            return pendingSpan.ToSegment();
+        }
+
+        pendingSpan = pendingSpan.Extend(segment);
+        return pendingSpan.ToSegment();
+    }
+
+    private static double Score(
+        TranscriptionSegment segment,
+        TimeSpan minimumUninterruptedSpeechDuration)
     {
         var durationSeconds = (segment.End - segment.Start).TotalSeconds;
-        if (durationSeconds < 2 || durationSeconds > 30)
+        if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
         {
             return 0;
         }
@@ -97,13 +145,13 @@ internal sealed class SpeakerAudioSampleCollector
         var words = segment.Text
             .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
             .Length;
-        if (words < 3 || words > 80)
+        if (words < 3)
         {
             return 0;
         }
 
-        var durationScore = 1 - Math.Min(Math.Abs(durationSeconds - 8) / 8, 1);
-        var wordScore = 1 - Math.Min(Math.Abs(words - 18) / 18.0, 1);
+        var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
+        var wordScore = Math.Min(words / 60.0, 1);
         var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
             segment.Text.TrimEnd().EndsWith('?') ||
             segment.Text.TrimEnd().EndsWith('!')
@@ -111,4 +159,29 @@ internal sealed class SpeakerAudioSampleCollector
                 : 0;
         return durationScore * 70 + wordScore * 30 + sentenceBonus;
     }
+
+    private sealed record PendingSpeakerSpan(
+        string Speaker,
+        TimeSpan Start,
+        TimeSpan End,
+        IReadOnlyList<string> TextParts)
+    {
+        public PendingSpeakerSpan Extend(TranscriptionSegment segment)
+        {
+            return this with
+            {
+                End = segment.End > End ? segment.End : End,
+                TextParts = TextParts.Append(segment.Text).ToList()
+            };
+        }
+
+        public TranscriptionSegment ToSegment()
+        {
+            return new TranscriptionSegment(
+                Start,
+                End,
+                Speaker,
+                string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
+        }
+    }
 }

MeetingAssistant/Speakers/AzureSpeechSpeakerIdentityMatcher.cs

@@ -15,15 +15,18 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
     private readonly ISpeakerIdentityDiarizationClient diarizationClient;
     private readonly SpeakerIdentificationOptions options;
     private readonly ILogger<AzureSpeechSpeakerIdentityMatcher> logger;
+    private readonly ISpeakerIdentityMatchValidator matchValidator;
 
     public AzureSpeechSpeakerIdentityMatcher(
         ISpeakerIdentityDiarizationClient diarizationClient,
         Microsoft.Extensions.Options.IOptions<MeetingAssistantOptions> options,
-        ILogger<AzureSpeechSpeakerIdentityMatcher> logger)
+        ILogger<AzureSpeechSpeakerIdentityMatcher> logger,
+        ISpeakerIdentityMatchValidator matchValidator)
     {
         this.diarizationClient = diarizationClient;
         this.options = options.Value.SpeakerIdentification;
         this.logger = logger;
+        this.matchValidator = matchValidator;
     }
 
     public async Task<SpeakerIdentityMatch?> MatchAsync(
@@ -35,6 +38,14 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
             return null;
         }
 
+        if (!await matchValidator.ValidateSampleAsync(request.UnknownSnippet, cancellationToken))
+        {
+            logger.LogInformation(
+                "Speaker identity matching {DiarizedSpeaker} skipped because secondary validation rejected the unknown sample",
+                request.DiarizedSpeaker);
+            return null;
+        }
+
         logger.LogInformation(
             "Speaker identity matching {DiarizedSpeaker} against {CandidateCount} candidate(s)",
             request.DiarizedSpeaker,
@@ -84,6 +95,20 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
                 var requiredMatches = known.Count() > 1 ? 2 : 1;
                 if (matchingSnippetCount >= requiredMatches)
                 {
+                    var validationRequest = new SpeakerIdentityMatchValidationRequest(
+                        request.DiarizedSpeaker,
+                        known.Key,
+                        request.UnknownSnippet,
+                        known.Select(segment => segment.Snippet).ToList());
+                    if (!await matchValidator.ValidateMatchAsync(validationRequest, cancellationToken))
+                    {
+                        logger.LogInformation(
+                            "Speaker identity matching {DiarizedSpeaker} rejected identity {IdentityId} after secondary validation",
+                            request.DiarizedSpeaker,
+                            known.Key);
+                        continue;
+                    }
+
                     logger.LogInformation(
                         "Speaker identity matching {DiarizedSpeaker} matched identity {IdentityId}",
                         request.DiarizedSpeaker,
@@ -99,7 +124,7 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
         }
         finally
         {
-            TryDelete(tempPath);
+            SpeakerCompositeWav.TryDelete(tempPath);
         }
     }
 
@@ -130,7 +155,8 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
 
     private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request)
     {
-        using var firstReader = OpenFirstReadableWave(request);
+        using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(
+            request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet));
         if (firstReader is null)
         {
             return new CompositeLayout(null, []);
@@ -145,118 +171,33 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
         {
             foreach (var snippet in candidate.Snippets.Where(storedSnippet => storedSnippet.Length > 0))
             {
-                var segment = AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
+                var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
                 if (segment is null)
                 {
                     continue;
                 }
 
-                knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value));
+                knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value, snippet));
                 current = segment.Value.End + silence;
-                WriteSilence(writer, firstReader.WaveFormat, silence);
+                SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
             }
         }
 
-        var unknownSegment = AppendSnippet(writer, firstReader.WaveFormat, request.UnknownSnippet, current);
+        var unknownSegment = SpeakerCompositeWav.AppendSnippet(
+            writer,
+            firstReader.WaveFormat,
+            request.UnknownSnippet,
+            current);
         return new CompositeLayout(unknownSegment, knownSegments);
     }
 
-    private static WaveFileReader? OpenFirstReadableWave(SpeakerIdentityMatchRequest request)
-    {
-        foreach (var bytes in request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet))
-        {
-            if (bytes.Length == 0)
-            {
-                continue;
-            }
-
-            try
-            {
-                return new WaveFileReader(new MemoryStream(bytes));
-            }
-            catch (FormatException)
-            {
-            }
-        }
-
-        return null;
-    }
-
-    private static TimeSegment? AppendSnippet(
-        WaveFileWriter writer,
-        WaveFormat targetFormat,
-        byte[] snippet,
-        TimeSpan start)
-    {
-        if (snippet.Length == 0)
-        {
-            return null;
-        }
-
-        using var reader = new WaveFileReader(new MemoryStream(snippet));
-        if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
-        {
-            throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
-        }
-
-        reader.CopyTo(writer);
-        return new TimeSegment(start, start + reader.TotalTime);
-    }
-
-    private static void WriteSilence(WaveFileWriter writer, WaveFormat format, TimeSpan duration)
-    {
-        if (duration <= TimeSpan.Zero)
-        {
-            return;
-        }
-
-        var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
-        writer.Write(bytes, 0, bytes.Length);
-    }
-
     private static string? FindBestSpeaker(TimeSegment segment, IReadOnlyList<TranscriptionSegment> segments)
     {
-        var bestOverlap = 0d;
-        string? bestSpeaker = null;
-        foreach (var transcriptSegment in segments)
-        {
-            var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
-                - Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
-            if (overlap > bestOverlap)
-            {
-                bestOverlap = overlap;
-                bestSpeaker = transcriptSegment.Speaker;
-            }
-        }
-
-        return bestSpeaker;
-    }
-
-    private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
-    {
-        return left.Encoding == right.Encoding
-            && left.SampleRate == right.SampleRate
-            && left.Channels == right.Channels
-            && left.BitsPerSample == right.BitsPerSample;
-    }
-
-    private static void TryDelete(string path)
-    {
-        try
-        {
-            if (File.Exists(path))
-            {
-                File.Delete(path);
-            }
-        }
-        catch (IOException)
-        {
-        }
+        return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
     }
 
     private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<KnownCompositeSegment> KnownSegments);
 
-    private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment);
+    private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment, byte[] Snippet);
 
-    private readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
 }

MeetingAssistant/Speakers/NoopSpeakerIdentityMatchValidator.cs

@@ -0,0 +1,18 @@
+namespace MeetingAssistant.Speakers;
+
+public sealed class NoopSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
+{
+    public Task<bool> ValidateSampleAsync(
+        byte[] wavBytes,
+        CancellationToken cancellationToken)
+    {
+        return Task.FromResult(true);
+    }
+
+    public Task<bool> ValidateMatchAsync(
+        SpeakerIdentityMatchValidationRequest request,
+        CancellationToken cancellationToken)
+    {
+        return Task.FromResult(true);
+    }
+}

MeetingAssistant/Speakers/PyannoteSpeakerIdentityMatchValidator.cs

@@ -0,0 +1,237 @@
+using MeetingAssistant.Transcription;
+using Microsoft.Extensions.Options;
+using NAudio.Wave;
+
+namespace MeetingAssistant.Speakers;
+
+public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
+{
+    private readonly PyannoteTranscriptFinalizer finalizer;
+    private readonly SpeakerIdentityPyannoteValidationOptions options;
+    private readonly ILogger<PyannoteSpeakerIdentityMatchValidator> logger;
+
+    public PyannoteSpeakerIdentityMatchValidator(
+        PyannoteTranscriptFinalizer finalizer,
+        IOptions<MeetingAssistantOptions> options,
+        ILogger<PyannoteSpeakerIdentityMatchValidator> logger)
+    {
+        this.finalizer = finalizer;
+        this.options = options.Value.SpeakerIdentification.PyannoteValidation;
+        this.logger = logger;
+    }
+
+    public async Task<bool> ValidateSampleAsync(
+        byte[] wavBytes,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled)
+        {
+            return true;
+        }
+
+        var segments = await DiarizeBytesAsync(wavBytes, cancellationToken);
+        var result = HasSingleSpeaker(segments);
+        if (!result)
+        {
+            logger.LogInformation(
+                "pyannote rejected speaker identity sample because it did not contain one dominant speaker");
+        }
+
+        return result;
+    }
+
+    public async Task<bool> ValidateMatchAsync(
+        SpeakerIdentityMatchValidationRequest request,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled)
+        {
+            return true;
+        }
+
+        var tempPath = Path.Combine(
+            Path.GetTempPath(),
+            "meeting-assistant-speaker-match-pyannote",
+            $"{Guid.NewGuid():N}.wav");
+        try
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
+            var layout = WriteCompositeWav(
+                tempPath,
+                request.KnownSnippets,
+                request.UnknownSnippet,
+                TimeSpan.FromSeconds(1));
+            if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
+            {
+                return false;
+            }
+
+            var segments = await DiarizePathAsync(tempPath, cancellationToken);
+            var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
+            if (string.IsNullOrWhiteSpace(unknownSpeaker))
+            {
+                logger.LogInformation(
+                    "pyannote rejected speaker identity match for {DiarizedSpeaker} because the unknown sample had no speaker",
+                    request.DiarizedSpeaker);
+                return false;
+            }
+
+            var compared = 0;
+            var matching = 0;
+            foreach (var knownSegment in layout.KnownSegments)
+            {
+                var knownSpeaker = FindBestSpeaker(knownSegment, segments);
+                if (string.IsNullOrWhiteSpace(knownSpeaker))
+                {
+                    continue;
+                }
+
+                compared++;
+                if (string.Equals(knownSpeaker, unknownSpeaker, StringComparison.Ordinal))
+                {
+                    matching++;
+                }
+            }
+
+            var minimumRatio = Math.Clamp(options.MinimumMatchingKnownSnippetRatio, 0, 1);
+            var accepted = compared > 0 && (double)matching / compared >= minimumRatio;
+            if (!accepted)
+            {
+                logger.LogInformation(
+                    "pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched",
+                    request.DiarizedSpeaker,
+                    request.IdentityId,
+                    matching,
+                    compared);
+            }
+
+            return accepted;
+        }
+        finally
+        {
+            SpeakerCompositeWav.TryDelete(tempPath);
+        }
+    }
+
+    private async Task<IReadOnlyList<TranscriptionSegment>> DiarizeBytesAsync(
+        byte[] wavBytes,
+        CancellationToken cancellationToken)
+    {
+        var tempPath = Path.Combine(
+            Path.GetTempPath(),
+            "meeting-assistant-speaker-sample-pyannote",
+            $"{Guid.NewGuid():N}.wav");
+        try
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
+            await File.WriteAllBytesAsync(tempPath, wavBytes, cancellationToken);
+            return await DiarizePathAsync(tempPath, cancellationToken);
+        }
+        finally
+        {
+            SpeakerCompositeWav.TryDelete(tempPath);
+        }
+    }
+
+    private async Task<IReadOnlyList<TranscriptionSegment>> DiarizePathAsync(
+        string wavPath,
+        CancellationToken cancellationToken)
+    {
+        TimeSpan duration;
+        try
+        {
+            using var reader = new WaveFileReader(wavPath);
+            duration = reader.TotalTime;
+        }
+        catch (Exception exception) when (exception is IOException or InvalidDataException or FormatException)
+        {
+            logger.LogInformation(exception, "pyannote speaker identity validation could not read WAV input");
+            return [];
+        }
+
+        if (duration <= TimeSpan.Zero)
+        {
+            return [];
+        }
+
+        return await finalizer.FinalizeAsync(
+            wavPath,
+            [new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
+            options.Diarization,
+            SpeechRecognitionPipelineOptions.Default,
+            cancellationToken);
+    }
+
+    private bool HasSingleSpeaker(IReadOnlyList<TranscriptionSegment> segments)
+    {
+        var durationsBySpeaker = segments
+            .Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker) && segment.End > segment.Start)
+            .GroupBy(segment => segment.Speaker)
+            .Select(group => new
+            {
+                Speaker = group.Key,
+                Duration = group.Sum(segment => (segment.End - segment.Start).TotalSeconds)
+            })
+            .Where(group => group.Duration > 0)
+            .ToList();
+        if (durationsBySpeaker.Count == 0)
+        {
+            return false;
+        }
+
+        if (durationsBySpeaker.Count == 1)
+        {
+            return true;
+        }
+
+        var total = durationsBySpeaker.Sum(group => group.Duration);
+        var dominant = durationsBySpeaker.Max(group => group.Duration);
+        return total > 0 && dominant / total >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
+    }
+
+    private CompositeLayout WriteCompositeWav(
+        string path,
+        IReadOnlyList<byte[]> knownSnippets,
+        byte[] unknownSnippet,
+        TimeSpan silence)
+    {
+        using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(knownSnippets.Append(unknownSnippet));
+        if (firstReader is null)
+        {
+            return new CompositeLayout(null, []);
+        }
+
+        using var writer = new WaveFileWriter(path, firstReader.WaveFormat);
+        var current = TimeSpan.Zero;
+        var knownSegments = new List<TimeSegment>();
+
+        foreach (var snippet in knownSnippets.Where(storedSnippet => storedSnippet.Length > 0))
+        {
+            var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
+            if (segment is null)
+            {
+                continue;
+            }
+
+            knownSegments.Add(segment.Value);
+            current = segment.Value.End + silence;
+            SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
+        }
+
+        var unknownSegment = SpeakerCompositeWav.AppendSnippet(
+            writer,
+            firstReader.WaveFormat,
+            unknownSnippet,
+            current);
+        return new CompositeLayout(unknownSegment, knownSegments);
+    }
+
+    private static string? FindBestSpeaker(
+        TimeSegment segment,
+        IReadOnlyList<TranscriptionSegment> segments)
+    {
+        return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
+    }
+
+    private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<TimeSegment> KnownSegments);
+}

MeetingAssistant/Speakers/SpeakerCompositeWav.cs

@@ -0,0 +1,107 @@
+using MeetingAssistant.Transcription;
+using NAudio.Wave;
+
+namespace MeetingAssistant.Speakers;
+
+internal static class SpeakerCompositeWav
+{
+    public static WaveFileReader? OpenFirstReadableWave(IEnumerable<byte[]> snippets)
+    {
+        foreach (var bytes in snippets)
+        {
+            if (bytes.Length == 0)
+            {
+                continue;
+            }
+
+            try
+            {
+                return new WaveFileReader(new MemoryStream(bytes));
+            }
+            catch (FormatException)
+            {
+            }
+        }
+
+        return null;
+    }
+
+    public static TimeSegment? AppendSnippet(
+        WaveFileWriter writer,
+        WaveFormat targetFormat,
+        byte[] snippet,
+        TimeSpan start)
+    {
+        if (snippet.Length == 0)
+        {
+            return null;
+        }
+
+        using var reader = new WaveFileReader(new MemoryStream(snippet));
+        if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
+        {
+            throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
+        }
+
+        reader.CopyTo(writer);
+        return new TimeSegment(start, start + reader.TotalTime);
+    }
+
+    public static void WriteSilence(
+        WaveFileWriter writer,
+        WaveFormat format,
+        TimeSpan duration)
+    {
+        if (duration <= TimeSpan.Zero)
+        {
+            return;
+        }
+
+        var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
+        writer.Write(bytes, 0, bytes.Length);
+    }
+
+    public static string? FindBestSpeaker(
+        TimeSegment segment,
+        IReadOnlyList<TranscriptionSegment> segments)
+    {
+        var bestOverlap = 0d;
+        string? bestSpeaker = null;
+        foreach (var transcriptSegment in segments)
+        {
+            var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
+                - Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
+            if (overlap > bestOverlap)
+            {
+                bestOverlap = overlap;
+                bestSpeaker = transcriptSegment.Speaker;
+            }
+        }
+
+        return bestSpeaker;
+    }
+
+    public static void TryDelete(string path)
+    {
+        try
+        {
+            if (File.Exists(path))
+            {
+                File.Delete(path);
+            }
+        }
+        catch (IOException)
+        {
+        }
+    }
+
+    private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
+    {
+        return left.Encoding == right.Encoding
+            && left.SampleRate == right.SampleRate
+            && left.Channels == right.Channels
+            && left.BitsPerSample == right.BitsPerSample;
+    }
+}
+
+internal readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);

MeetingAssistant/Speakers/SpeakerIdentificationModels.cs

@@ -38,6 +38,12 @@ public sealed record SpeakerIdentityMatchCandidate(
 
 public sealed record SpeakerIdentityMatch(int IdentityId);
 
+public sealed record SpeakerIdentityMatchValidationRequest(
+    string DiarizedSpeaker,
+    int IdentityId,
+    byte[] UnknownSnippet,
+    IReadOnlyList<byte[]> KnownSnippets);
+
 public interface ISpeakerIdentityMatcher
 {
     Task<SpeakerIdentityMatch?> MatchAsync(
@@ -45,6 +51,17 @@ public interface ISpeakerIdentityMatcher
         CancellationToken cancellationToken);
 }
 
+public interface ISpeakerIdentityMatchValidator
+{
+    Task<bool> ValidateSampleAsync(
+        byte[] wavBytes,
+        CancellationToken cancellationToken);
+
+    Task<bool> ValidateMatchAsync(
+        SpeakerIdentityMatchValidationRequest request,
+        CancellationToken cancellationToken);
+}
+
 public interface ISpeakerSnippetExtractor
 {
     Task<byte[]> ExtractSnippetAsync(

MeetingAssistant/Speakers/SpeakerIdentity.cs

@@ -13,6 +13,8 @@ public sealed class SpeakerIdentity
 
     public DateTimeOffset UpdatedAt { get; set; }
 
+    public int TranscriptionCount { get; set; }
+
     public List<SpeakerCandidateName> CandidateNames { get; set; } = [];
 
     public List<SpeakerAlias> Aliases { get; set; } = [];
@@ -154,8 +156,9 @@ public sealed class SpeakerIdentityDbContext : DbContext
     {
         modelBuilder.Entity<SpeakerIdentity>(entity =>
         {
-            entity.Property<int>("TranscriptionCount")
-                .HasDefaultValue(0);
+            entity.Property(identity => identity.TranscriptionCount)
+                .IsRequired()
+                .ValueGeneratedNever();
 
             entity.HasMany(identity => identity.CandidateNames)
                 .WithOne(candidate => candidate.SpeakerIdentity)

MeetingAssistant/Speakers/SpeakerIdentityService.cs

@@ -10,6 +10,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
     private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
     private readonly ISpeakerSnippetExtractor snippetExtractor;
     private readonly ISpeakerIdentityMatcher matcher;
+    private readonly ISpeakerIdentityMatchValidator matchValidator;
     private readonly SpeakerIdentificationOptions options;
     private readonly ILogger<SpeakerIdentityService> logger;
 
@@ -18,11 +19,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
         ISpeakerSnippetExtractor snippetExtractor,
         ISpeakerIdentityMatcher matcher,
         IOptions<MeetingAssistantOptions> options,
-        ILogger<SpeakerIdentityService> logger)
+        ILogger<SpeakerIdentityService> logger,
+        ISpeakerIdentityMatchValidator matchValidator)
     {
         this.dbContextFactory = dbContextFactory;
         this.snippetExtractor = snippetExtractor;
         this.matcher = matcher;
+        this.matchValidator = matchValidator;
         this.options = options.Value.SpeakerIdentification;
         this.logger = logger;
     }
@@ -198,7 +201,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
                 ? suppliedSnippet
                 : await snippetExtractor.ExtractSnippetAsync(
                     request.AudioPath,
-                    group.ToList(),
+                    SpeakerSampleSpanSelector.SelectBestContinuousSpan(
+                        request.Segments,
+                        speaker,
+                        options.MaximumSampleSegmentGap,
+                        options.MinimumSampleSpeechDuration),
                     cancellationToken);
             if (snippet.Length == 0)
             {
@@ -544,6 +551,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
 
         foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
         {
+            if (!await matchValidator.ValidateSampleAsync(snippet, cancellationToken))
+            {
+                logger.LogInformation(
+                    "Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample",
+                    speaker);
+                continue;
+            }
+
             var now = DateTimeOffset.UtcNow;
             var canonicalName = remainingCandidates.Count == 1 ? remainingCandidates[0] : null;
             var identity = new SpeakerIdentity

MeetingAssistant/Speakers/SpeakerSampleSpanSelector.cs

@@ -0,0 +1,79 @@
+using MeetingAssistant.Transcription;
+
+namespace MeetingAssistant.Speakers;
+
+internal static class SpeakerSampleSpanSelector
+{
+    public static bool CanExtend(
+        string currentSpeaker,
+        TimeSpan currentEnd,
+        TranscriptionSegment nextSegment,
+        TimeSpan maximumSegmentGap)
+    {
+        return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
+            nextSegment.Start - currentEnd <= maximumSegmentGap;
+    }
+
+    public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
+        IReadOnlyList<TranscriptionSegment> segments,
+        string speaker,
+        TimeSpan maximumSegmentGap,
+        TimeSpan minimumDuration)
+    {
+        var best = new List<TranscriptionSegment>();
+        var current = new List<TranscriptionSegment>();
+
+        foreach (var segment in segments.OrderBy(segment => segment.Start))
+        {
+            if (current.Count == 0)
+            {
+                if (IsSpeaker(segment, speaker))
+                {
+                    current.Add(segment);
+                    best = LongerSpan(current, best);
+                }
+
+                continue;
+            }
+
+            if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
+            {
+                current.Clear();
+                if (!IsSpeaker(segment, speaker))
+                {
+                    continue;
+                }
+            }
+
+            current.Add(segment);
+            best = LongerSpan(current, best);
+        }
+
+        return SpanDuration(best) >= minimumDuration
+            ? best
+            : [];
+    }
+
+    public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
+    {
+        return segments.Count == 0
+            ? TimeSpan.Zero
+            : segments[^1].End - segments[0].Start;
+    }
+
+    private static bool IsSpeaker(
+        TranscriptionSegment segment,
+        string speaker)
+    {
+        return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static List<TranscriptionSegment> LongerSpan(
+        List<TranscriptionSegment> current,
+        List<TranscriptionSegment> best)
+    {
+        return SpanDuration(current) > SpanDuration(best)
+            ? current.ToList()
+            : best;
+    }
+}

MeetingAssistant/appsettings.json

@@ -103,10 +103,30 @@
       "MaxMatchCandidates": 100,
       "MatchIdentityActiveAge": "365.00:00:00",
       "MaxSnippetsPerSpeaker": 3,
+      "MinimumSampleSpeechDuration": "00:00:30",
+      "MaximumSampleSegmentGap": "00:00:01",
       "SilenceBetweenSnippetsSeconds": 1,
       "LiveSampleBufferDuration": "00:10:00",
       "MergeRecentIdentityAge": "14.00:00:00",
-      "MatchTimeout": "00:03:00"
+      "MatchTimeout": "00:03:00",
+      "PyannoteValidation": {
+        "Enabled": false,
+        "MinimumSingleSpeakerCoverage": 0.9,
+        "MinimumMatchingKnownSnippetRatio": 0.5,
+        "Diarization": {
+          "Enabled": true,
+          "DockerCommand": "docker",
+          "BaseImage": "python:3.11-slim",
+          "Image": "meeting-assistant-pyannote:local",
+          "BuildImage": true,
+          "ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Pyannote\\models",
+          "Model": "pyannote/speaker-diarization-3.1",
+          "AnnotationSource": "SpeakerDiarization",
+          "AlignmentMode": "PyannoteTurns",
+          "TokenEnv": "HF_TOKEN",
+          "CommandTimeout": "00:30:00"
+        }
+      }
     },
     "Automation": {
       "RulesPath": "C:\\Manuel\\meeting-assistant\\meeting-rules.local.yaml"