meeting-assistant/MeetingAssistant/Speakers/ResemblyzerVoiceClusterMatcher.cs

namespace MeetingAssistant.Speakers;

public sealed record ResemblyzerVoiceVectorCandidate(
    int IdentityId,
    IReadOnlyList<float[]> Vectors);

public sealed record ResemblyzerVoiceClusterMatchResult(
    int? IdentityId,
    string Reason,
    double Cohesion,
    double? BestSimilarity,
    double? RunnerUpSimilarity)
{
    public bool Accepted => IdentityId.HasValue;
}

public sealed class ResemblyzerVoiceClusterMatcher
{
    private readonly ResemblyzerSpeakerRecognitionOptions options;
    private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;

    public ResemblyzerVoiceClusterMatcher(
        ResemblyzerSpeakerRecognitionOptions options,
        ILogger<ResemblyzerVoiceClusterMatcher> logger)
    {
        this.options = options;
        this.logger = logger;
    }

    public ResemblyzerVoiceClusterMatchResult Match(
        IReadOnlyList<float[]> queryVectors,
        IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
    {
        var requiredCount = options.RequiredVectorsPerSpeaker;
        if (queryVectors.Count < requiredCount)
        {
            return Reject(
                $"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
                cohesion: 0,
                bestSimilarity: null,
                runnerUpSimilarity: null);
        }

        var normalizedQuery = queryVectors
            .Take(requiredCount)
            .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
            .ToList();
        var cohesion = MeanPairwiseCosine(normalizedQuery);
        if (cohesion < options.MinimumClusterCohesion)
        {
            return Reject(
                $"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
                cohesion,
                bestSimilarity: null,
                runnerUpSimilarity: null);
        }

        var scored = candidates
            .Select(candidate => ScoreCandidate(normalizedQuery, candidate))
            .Where(candidate => candidate is not null)
            .Select(candidate => candidate!.Value)
            .OrderByDescending(candidate => candidate.Similarity)
            .ThenBy(candidate => candidate.IdentityId)
            .ToList();
        if (scored.Count == 0)
        {
            return Reject("no compatible identity vectors were available", cohesion, null, null);
        }

        var best = scored[0];
        var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
        if (best.Similarity < options.MinimumIdentitySimilarity)
        {
            return Reject(
                $"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
                cohesion,
                best.Similarity,
                runnerUp);
        }

        if (runnerUp is { } runnerUpSimilarity &&
            best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
        {
            return Reject(
                $"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
                cohesion,
                best.Similarity,
                runnerUpSimilarity);
        }

        logger.LogInformation(
            "Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
            best.IdentityId,
            cohesion,
            best.Similarity,
            runnerUp);
        return new ResemblyzerVoiceClusterMatchResult(
            best.IdentityId,
            "accepted",
            cohesion,
            best.Similarity,
            runnerUp);
    }

    private ResemblyzerVoiceClusterMatchResult Reject(
        string reason,
        double cohesion,
        double? bestSimilarity,
        double? runnerUpSimilarity)
    {
        logger.LogInformation(
            "Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
            reason,
            cohesion,
            bestSimilarity,
            runnerUpSimilarity);
        return new ResemblyzerVoiceClusterMatchResult(
            null,
            reason,
            cohesion,
            bestSimilarity,
            runnerUpSimilarity);
    }

    private (int IdentityId, double Similarity)? ScoreCandidate(
        IReadOnlyList<float[]> normalizedQuery,
        ResemblyzerVoiceVectorCandidate candidate)
    {
        if (candidate.Vectors.Count == 0)
        {
            return null;
        }

        try
        {
            var normalizedCandidate = candidate.Vectors
                .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
                .ToList();
            var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
            var similarities = normalizedQuery
                .Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
                .Order()
                .ToList();
            return (candidate.IdentityId, Median(similarities));
        }
        catch (InvalidDataException exception)
        {
            logger.LogWarning(
                exception,
                "Skipping invalid Resemblyzer evidence for identity {IdentityId}",
                candidate.IdentityId);
            return null;
        }
    }

    private static float[] Centroid(IReadOnlyList<float[]> vectors)
    {
        var centroid = new float[ResemblyzerVectorContract.Dimensions];
        foreach (var vector in vectors)
        {
            for (var index = 0; index < centroid.Length; index++)
            {
                centroid[index] += vector[index];
            }
        }

        for (var index = 0; index < centroid.Length; index++)
        {
            centroid[index] /= vectors.Count;
        }

        return centroid;
    }

    private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
    {
        if (vectors.Count < 2)
        {
            return 1;
        }

        double total = 0;
        var pairs = 0;
        for (var first = 0; first < vectors.Count - 1; first++)
        {
            for (var second = first + 1; second < vectors.Count; second++)
            {
                total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
                pairs++;
            }
        }

        return total / pairs;
    }

    private static double Median(IReadOnlyList<double> sortedValues)
    {
        var middle = sortedValues.Count / 2;
        return sortedValues.Count % 2 == 0
            ? (sortedValues[middle - 1] + sortedValues[middle]) / 2
            : sortedValues[middle];
    }

}