using MeetingAssistant.Speakers; using MeetingAssistant.Transcription; using Microsoft.Extensions.Logging; namespace MeetingAssistant.Recording; internal sealed class SpeakerAudioSampleCollector { private readonly object gate = new(); private readonly RollingAudioBuffer audioBuffer; private readonly Dictionary> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase); private readonly int maxSamplesPerSpeaker; private readonly TimeSpan minimumUninterruptedSpeechDuration; private readonly TimeSpan maximumSegmentGap; private readonly ILogger? logger; private PendingSpeakerSpan? pendingSpan; public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker) : this( bufferDuration, maxSamplesPerSpeaker, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(1), logger: null) { } public SpeakerAudioSampleCollector( TimeSpan bufferDuration, int maxSamplesPerSpeaker, TimeSpan minimumUninterruptedSpeechDuration, TimeSpan maximumSegmentGap, ILogger? logger = null) { audioBuffer = new RollingAudioBuffer(bufferDuration); this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker); this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero ? minimumUninterruptedSpeechDuration : TimeSpan.Zero; this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero ? maximumSegmentGap : TimeSpan.Zero; this.logger = logger; } public void AppendAudio(AudioChunk chunk) { audioBuffer.Append(chunk); } public void Reset() { lock (gate) { samplesBySpeaker.Clear(); pendingSpan = null; audioBuffer.Reset(); } } public SpeakerAudioSample? TryAdd(TranscriptionSegment segment) { if (!IsDiarizedSpeaker(segment.Speaker)) { logger?.LogInformation( "Discarding speaker identity sample for {Speaker} because the segment has no diarized speaker label", segment.Speaker); return null; } TranscriptionSegment sampleSegment; PendingSpanReset? reset; lock (gate) { (sampleSegment, reset) = ExtendPendingSpan(segment); } if (reset is not null) { logger?.LogInformation( "Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}", reset.PreviousSpeaker, segment.Speaker, reset.PreviousEnd, segment.Start, reset.Gap, maximumSegmentGap); } var score = Score(sampleSegment, minimumUninterruptedSpeechDuration); if (!score.Accepted) { logger?.LogInformation( "Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}", sampleSegment.Speaker, score.Reason, sampleSegment.End - sampleSegment.Start, minimumUninterruptedSpeechDuration, score.WordCount); return null; } var wavBytes = audioBuffer.TryExtractWav(sampleSegment.Start, sampleSegment.End); if (wavBytes.Length == 0) { logger?.LogInformation( "Discarding speaker identity sample for {Speaker} because no audio could be extracted from rolling buffer: start {Start}, end {End}, duration {Duration}", sampleSegment.Speaker, sampleSegment.Start, sampleSegment.End, sampleSegment.End - sampleSegment.Start); return null; } var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value); lock (gate) { if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples)) { samples = []; samplesBySpeaker[sampleSegment.Speaker] = samples; } samples.Add(sample); var beforeCount = samples.Count; var bestSamples = samples .OrderByDescending(candidate => candidate.Score) .Take(maxSamplesPerSpeaker) .ToList(); var retained = bestSamples.Contains(sample); samples.Clear(); samples.AddRange(bestSamples); if (!retained) { logger?.LogInformation( "Discarding speaker identity sample for {Speaker} because it was not among the best {MaxSamplesPerSpeaker} retained sample(s): score {Score}, candidate count {CandidateCount}", sample.Speaker, maxSamplesPerSpeaker, sample.Score, beforeCount); return null; } } return sample; } public IReadOnlyList Snapshot() { lock (gate) { return samplesBySpeaker.Values .SelectMany(samples => samples) .OrderBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase) .ThenByDescending(sample => sample.Score) .ToList(); } } private static bool IsDiarizedSpeaker(string speaker) { return !string.IsNullOrWhiteSpace(speaker) && !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase); } private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment) { if (pendingSpan is null || !SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap)) { var reset = pendingSpan is null ? null : new PendingSpanReset( pendingSpan.Speaker, pendingSpan.End, segment.Start - pendingSpan.End); pendingSpan = new PendingSpeakerSpan( segment.Speaker, segment.Start, segment.End, [segment.Text]); return (pendingSpan.ToSegment(), reset); } pendingSpan = pendingSpan.Extend(segment); return (pendingSpan.ToSegment(), null); } private static SampleScore Score( TranscriptionSegment segment, TimeSpan minimumUninterruptedSpeechDuration) { var durationSeconds = (segment.End - segment.Start).TotalSeconds; if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds) { return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text)); } var words = WordCount(segment.Text); if (words < 3) { return new SampleScore(false, 0, "word count is below the minimum useful sample length", words); } var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2); var wordScore = Math.Min(words / 60.0, 1); var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') || segment.Text.TrimEnd().EndsWith('?') || segment.Text.TrimEnd().EndsWith('!') ? 5 : 0; return new SampleScore(true, durationScore * 70 + wordScore * 30 + sentenceBonus, null, words); } private static int WordCount(string text) { return text .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Length; } private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount); private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap); private sealed record PendingSpeakerSpan( string Speaker, TimeSpan Start, TimeSpan End, IReadOnlyList TextParts) { public PendingSpeakerSpan Extend(TranscriptionSegment segment) { return this with { End = segment.End > End ? segment.End : End, TextParts = TextParts.Append(segment.Text).ToList() }; } public TranscriptionSegment ToSegment() { return new TranscriptionSegment( Start, End, Speaker, string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part)))); } } }