meeting-assistant/MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs

using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging;

namespace MeetingAssistant.Recording;

internal sealed class SpeakerAudioSampleCollector
{
    private readonly object gate = new();
    private readonly RollingAudioBuffer audioBuffer;
    private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
    private readonly int maxSamplesPerSpeaker;
    private readonly TimeSpan minimumUninterruptedSpeechDuration;
    private readonly TimeSpan maximumSegmentGap;
    private readonly ILogger? logger;
    private PendingSpeakerSpan? pendingSpan;

    public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
        : this(
            bufferDuration,
            maxSamplesPerSpeaker,
            TimeSpan.FromSeconds(30),
            TimeSpan.FromSeconds(1),
            logger: null)
    {
    }

    public SpeakerAudioSampleCollector(
        TimeSpan bufferDuration,
        int maxSamplesPerSpeaker,
        TimeSpan minimumUninterruptedSpeechDuration,
        TimeSpan maximumSegmentGap,
        ILogger? logger = null)
    {
        audioBuffer = new RollingAudioBuffer(bufferDuration);
        this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
        this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
            ? minimumUninterruptedSpeechDuration
            : TimeSpan.Zero;
        this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
            ? maximumSegmentGap
            : TimeSpan.Zero;
        this.logger = logger;
    }

    public void AppendAudio(AudioChunk chunk)
    {
        audioBuffer.Append(chunk);
    }

    public void Reset()
    {
        lock (gate)
        {
            samplesBySpeaker.Clear();
            pendingSpan = null;
            audioBuffer.Reset();
        }
    }

    public SpeakerAudioSample? TryAdd(TranscriptionSegment segment)
    {
        if (!IsDiarizedSpeaker(segment.Speaker))
        {
            logger?.LogInformation(
                "Discarding speaker identity sample for {Speaker} because the segment has no diarized speaker label",
                segment.Speaker);
            return null;
        }

        TranscriptionSegment sampleSegment;
        PendingSpanReset? reset;
        lock (gate)
        {
            (sampleSegment, reset) = ExtendPendingSpan(segment);
        }

        if (reset is not null)
        {
            logger?.LogInformation(
                "Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
                reset.PreviousSpeaker,
                segment.Speaker,
                reset.PreviousEnd,
                segment.Start,
                reset.Gap,
                maximumSegmentGap);
        }

        var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
        if (!score.Accepted)
        {
            logger?.LogInformation(
                "Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
                sampleSegment.Speaker,
                score.Reason,
                sampleSegment.End - sampleSegment.Start,
                minimumUninterruptedSpeechDuration,
                score.WordCount);
            return null;
        }

        var wavBytes = audioBuffer.TryExtractWav(sampleSegment.Start, sampleSegment.End);
        if (wavBytes.Length == 0)
        {
            logger?.LogInformation(
                "Discarding speaker identity sample for {Speaker} because no audio could be extracted from rolling buffer: start {Start}, end {End}, duration {Duration}",
                sampleSegment.Speaker,
                sampleSegment.Start,
                sampleSegment.End,
                sampleSegment.End - sampleSegment.Start);
            return null;
        }

        var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
        lock (gate)
        {
            if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
            {
                samples = [];
                samplesBySpeaker[sampleSegment.Speaker] = samples;
            }

            samples.Add(sample);
            var beforeCount = samples.Count;
            var bestSamples = samples
                .OrderByDescending(candidate => candidate.Score)
                .Take(maxSamplesPerSpeaker)
                .ToList();
            var retained = bestSamples.Contains(sample);
            samples.Clear();
            samples.AddRange(bestSamples);
            if (!retained)
            {
                logger?.LogInformation(
                    "Discarding speaker identity sample for {Speaker} because it was not among the best {MaxSamplesPerSpeaker} retained sample(s): score {Score}, candidate count {CandidateCount}",
                    sample.Speaker,
                    maxSamplesPerSpeaker,
                    sample.Score,
                    beforeCount);
                return null;
            }
        }

        return sample;
    }

    public IReadOnlyList<SpeakerAudioSample> Snapshot()
    {
        lock (gate)
        {
            return samplesBySpeaker.Values
                .SelectMany(samples => samples)
                .OrderBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase)
                .ThenByDescending(sample => sample.Score)
                .ToList();
        }
    }

    private static bool IsDiarizedSpeaker(string speaker)
    {
        return !string.IsNullOrWhiteSpace(speaker) &&
            !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
    }

    private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
    {
        if (pendingSpan is null ||
            !SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
        {
            var reset = pendingSpan is null
                ? null
                : new PendingSpanReset(
                    pendingSpan.Speaker,
                    pendingSpan.End,
                    segment.Start - pendingSpan.End);
            pendingSpan = new PendingSpeakerSpan(
                segment.Speaker,
                segment.Start,
                segment.End,
                [segment.Text]);
            return (pendingSpan.ToSegment(), reset);
        }

        pendingSpan = pendingSpan.Extend(segment);
        return (pendingSpan.ToSegment(), null);
    }

    private static SampleScore Score(
        TranscriptionSegment segment,
        TimeSpan minimumUninterruptedSpeechDuration)
    {
        var durationSeconds = (segment.End - segment.Start).TotalSeconds;
        if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
        {
            return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
        }

        var words = WordCount(segment.Text);
        if (words < 3)
        {
            return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
        }

        var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
        var wordScore = Math.Min(words / 60.0, 1);
        var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
            segment.Text.TrimEnd().EndsWith('?') ||
            segment.Text.TrimEnd().EndsWith('!')
                ? 5
                : 0;
        return new SampleScore(true, durationScore * 70 + wordScore * 30 + sentenceBonus, null, words);
    }

    private static int WordCount(string text)
    {
        return text
            .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
            .Length;
    }

    private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);

    private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);

    private sealed record PendingSpeakerSpan(
        string Speaker,
        TimeSpan Start,
        TimeSpan End,
        IReadOnlyList<string> TextParts)
    {
        public PendingSpeakerSpan Extend(TranscriptionSegment segment)
        {
            return this with
            {
                End = segment.End > End ? segment.End : End,
                TextParts = TextParts.Append(segment.Text).ToList()
            };
        }

        public TranscriptionSegment ToSegment()
        {
            return new TranscriptionSegment(
                Start,
                End,
                Speaker,
                string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
        }
    }
}