Add inactivity safeguard and speaker diagnostics

MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs

@@ -1,5 +1,6 @@
 using MeetingAssistant.Speakers;
 using MeetingAssistant.Transcription;
+using Microsoft.Extensions.Logging;
 
 namespace MeetingAssistant.Recording;
 
@@ -11,6 +12,7 @@ internal sealed class SpeakerAudioSampleCollector
     private readonly int maxSamplesPerSpeaker;
     private readonly TimeSpan minimumUninterruptedSpeechDuration;
     private readonly TimeSpan maximumSegmentGap;
+    private readonly ILogger? logger;
     private PendingSpeakerSpan? pendingSpan;
 
     public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
@@ -18,7 +20,8 @@ internal sealed class SpeakerAudioSampleCollector
             bufferDuration,
             maxSamplesPerSpeaker,
             TimeSpan.FromSeconds(30),
-            TimeSpan.FromSeconds(1))
+            TimeSpan.FromSeconds(1),
+            logger: null)
     {
     }
 
@@ -26,7 +29,8 @@ internal sealed class SpeakerAudioSampleCollector
         TimeSpan bufferDuration,
         int maxSamplesPerSpeaker,
         TimeSpan minimumUninterruptedSpeechDuration,
-        TimeSpan maximumSegmentGap)
+        TimeSpan maximumSegmentGap,
+        ILogger? logger = null)
     {
         audioBuffer = new RollingAudioBuffer(bufferDuration);
         this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
@@ -36,6 +40,7 @@ internal sealed class SpeakerAudioSampleCollector
         this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
             ? maximumSegmentGap
             : TimeSpan.Zero;
+        this.logger = logger;
     }
 
     public void AppendAudio(AudioChunk chunk)
@@ -53,32 +58,61 @@ internal sealed class SpeakerAudioSampleCollector
         }
     }
 
-    public void TryAdd(TranscriptionSegment segment)
+    public SpeakerAudioSample? TryAdd(TranscriptionSegment segment)
     {
         if (!IsDiarizedSpeaker(segment.Speaker))
         {
-            return;
+            logger?.LogInformation(
+                "Discarding speaker identity sample for {Speaker} because the segment has no diarized speaker label",
+                segment.Speaker);
+            return null;
         }
 
         TranscriptionSegment sampleSegment;
+        PendingSpanReset? reset;
         lock (gate)
         {
-            sampleSegment = ExtendPendingSpan(segment);
+            (sampleSegment, reset) = ExtendPendingSpan(segment);
+        }
+
+        if (reset is not null)
+        {
+            logger?.LogInformation(
+                "Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
+                reset.PreviousSpeaker,
+                segment.Speaker,
+                reset.PreviousEnd,
+                segment.Start,
+                reset.Gap,
+                maximumSegmentGap);
         }
 
         var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
-        if (score <= 0)
+        if (!score.Accepted)
         {
-            return;
+            logger?.LogInformation(
+                "Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
+                sampleSegment.Speaker,
+                score.Reason,
+                sampleSegment.End - sampleSegment.Start,
+                minimumUninterruptedSpeechDuration,
+                score.WordCount);
+            return null;
         }
 
         var wavBytes = audioBuffer.TryExtractWav(sampleSegment.Start, sampleSegment.End);
         if (wavBytes.Length == 0)
         {
-            return;
+            logger?.LogInformation(
+                "Discarding speaker identity sample for {Speaker} because no audio could be extracted from rolling buffer: start {Start}, end {End}, duration {Duration}",
+                sampleSegment.Speaker,
+                sampleSegment.Start,
+                sampleSegment.End,
+                sampleSegment.End - sampleSegment.Start);
+            return null;
         }
 
-        var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score);
+        var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
         lock (gate)
         {
             if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
@@ -88,13 +122,27 @@ internal sealed class SpeakerAudioSampleCollector
             }
 
             samples.Add(sample);
+            var beforeCount = samples.Count;
             var bestSamples = samples
                 .OrderByDescending(candidate => candidate.Score)
                 .Take(maxSamplesPerSpeaker)
                 .ToList();
+            var retained = bestSamples.Contains(sample);
             samples.Clear();
             samples.AddRange(bestSamples);
+            if (!retained)
+            {
+                logger?.LogInformation(
+                    "Discarding speaker identity sample for {Speaker} because it was not among the best {MaxSamplesPerSpeaker} retained sample(s): score {Score}, candidate count {CandidateCount}",
+                    sample.Speaker,
+                    maxSamplesPerSpeaker,
+                    sample.Score,
+                    beforeCount);
+                return null;
+            }
         }
+
+        return sample;
     }
 
     public IReadOnlyList<SpeakerAudioSample> Snapshot()
@@ -115,39 +163,43 @@ internal sealed class SpeakerAudioSampleCollector
             !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
     }
 
-    private TranscriptionSegment ExtendPendingSpan(TranscriptionSegment segment)
+    private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
     {
         if (pendingSpan is null ||
             !SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
         {
+            var reset = pendingSpan is null
+                ? null
+                : new PendingSpanReset(
+                    pendingSpan.Speaker,
+                    pendingSpan.End,
+                    segment.Start - pendingSpan.End);
             pendingSpan = new PendingSpeakerSpan(
                 segment.Speaker,
                 segment.Start,
                 segment.End,
                 [segment.Text]);
-            return pendingSpan.ToSegment();
+            return (pendingSpan.ToSegment(), reset);
         }
 
         pendingSpan = pendingSpan.Extend(segment);
-        return pendingSpan.ToSegment();
+        return (pendingSpan.ToSegment(), null);
     }
 
-    private static double Score(
+    private static SampleScore Score(
         TranscriptionSegment segment,
         TimeSpan minimumUninterruptedSpeechDuration)
     {
         var durationSeconds = (segment.End - segment.Start).TotalSeconds;
         if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
         {
-            return 0;
+            return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
         }
 
-        var words = segment.Text
-            .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
-            .Length;
+        var words = WordCount(segment.Text);
         if (words < 3)
         {
-            return 0;
+            return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
         }
 
         var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
@@ -157,9 +209,20 @@ internal sealed class SpeakerAudioSampleCollector
             segment.Text.TrimEnd().EndsWith('!')
                 ? 5
                 : 0;
-        return durationScore * 70 + wordScore * 30 + sentenceBonus;
+        return new SampleScore(true, durationScore * 70 + wordScore * 30 + sentenceBonus, null, words);
     }
 
+    private static int WordCount(string text)
+    {
+        return text
+            .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+            .Length;
+    }
+
+    private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
+
+    private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
+
     private sealed record PendingSpeakerSpan(
         string Speaker,
         TimeSpan Start,