namespace MeetingAssistant.Speakers; public sealed record ResemblyzerVoiceVectorCandidate( int IdentityId, IReadOnlyList Vectors); public sealed record ResemblyzerVoiceClusterMatchResult( int? IdentityId, string Reason, double Cohesion, double? BestSimilarity, double? RunnerUpSimilarity) { public bool Accepted => IdentityId.HasValue; } public sealed class ResemblyzerVoiceClusterMatcher { private readonly ResemblyzerSpeakerRecognitionOptions options; private readonly ILogger logger; public ResemblyzerVoiceClusterMatcher( ResemblyzerSpeakerRecognitionOptions options, ILogger logger) { this.options = options; this.logger = logger; } public ResemblyzerVoiceClusterMatchResult Match( IReadOnlyList queryVectors, IReadOnlyList candidates) { var requiredCount = options.RequiredVectorsPerSpeaker; if (queryVectors.Count < requiredCount) { return Reject( $"insufficient query vectors ({queryVectors.Count}/{requiredCount})", cohesion: 0, bestSimilarity: null, runnerUpSimilarity: null); } var normalizedQuery = queryVectors .Take(requiredCount) .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector")) .ToList(); var cohesion = MeanPairwiseCosine(normalizedQuery); if (cohesion < options.MinimumClusterCohesion) { return Reject( $"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}", cohesion, bestSimilarity: null, runnerUpSimilarity: null); } var scored = candidates .Select(candidate => ScoreCandidate(normalizedQuery, candidate)) .Where(candidate => candidate is not null) .Select(candidate => candidate!.Value) .OrderByDescending(candidate => candidate.Similarity) .ThenBy(candidate => candidate.IdentityId) .ToList(); if (scored.Count == 0) { return Reject("no compatible identity vectors were available", cohesion, null, null); } var best = scored[0]; var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null; if (best.Similarity < options.MinimumIdentitySimilarity) { return Reject( $"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}", cohesion, best.Similarity, runnerUp); } if (runnerUp is { } runnerUpSimilarity && best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin) { return Reject( $"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}", cohesion, best.Similarity, runnerUpSimilarity); } logger.LogInformation( "Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}", best.IdentityId, cohesion, best.Similarity, runnerUp); return new ResemblyzerVoiceClusterMatchResult( best.IdentityId, "accepted", cohesion, best.Similarity, runnerUp); } private ResemblyzerVoiceClusterMatchResult Reject( string reason, double cohesion, double? bestSimilarity, double? runnerUpSimilarity) { logger.LogInformation( "Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}", reason, cohesion, bestSimilarity, runnerUpSimilarity); return new ResemblyzerVoiceClusterMatchResult( null, reason, cohesion, bestSimilarity, runnerUpSimilarity); } private (int IdentityId, double Similarity)? ScoreCandidate( IReadOnlyList normalizedQuery, ResemblyzerVoiceVectorCandidate candidate) { if (candidate.Vectors.Count == 0) { return null; } try { var normalizedCandidate = candidate.Vectors .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector")) .ToList(); var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid"); var similarities = normalizedQuery .Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid)) .Order() .ToList(); return (candidate.IdentityId, Median(similarities)); } catch (InvalidDataException exception) { logger.LogWarning( exception, "Skipping invalid Resemblyzer evidence for identity {IdentityId}", candidate.IdentityId); return null; } } private static float[] Centroid(IReadOnlyList vectors) { var centroid = new float[ResemblyzerVectorContract.Dimensions]; foreach (var vector in vectors) { for (var index = 0; index < centroid.Length; index++) { centroid[index] += vector[index]; } } for (var index = 0; index < centroid.Length; index++) { centroid[index] /= vectors.Count; } return centroid; } private static double MeanPairwiseCosine(IReadOnlyList vectors) { if (vectors.Count < 2) { return 1; } double total = 0; var pairs = 0; for (var first = 0; first < vectors.Count - 1; first++) { for (var second = first + 1; second < vectors.Count; second++) { total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]); pairs++; } } return total / pairs; } private static double Median(IReadOnlyList sortedValues) { var middle = sortedValues.Count / 2; return sortedValues.Count % 2 == 0 ? (sortedValues[middle - 1] + sortedValues[middle]) / 2 : sortedValues[middle]; } }