feat: add local Resemblyzer speaker recognition

2026-09-11 13:49:27 +02:00
parent 43fc8aaec0
commit f86af983e8
48 changed files with 5250 additions and 273 deletions
@@ -0,0 +1,141 @@
+using MeetingAssistant;
+using MeetingAssistant.Speakers;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace MeetingAssistant.Tests;
+
+public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
+{
+    [Fact]
+    public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
+    {
+        var identity = IdentityWithVectors(
+            Enumerable.Range(0, 16)
+                .Select(index => UnitVector(0, index + 2, 0.04f))
+                .Concat(Enumerable.Range(0, 4)
+                    .Select(index => UnitVector(1, index + 30, 0.04f)))
+                .ToList());
+        var pruner = CreatePruner();
+
+        var result = pruner.Prune(identity);
+
+        Assert.Equal(4, result);
+        Assert.Equal(16, identity.VoiceVectors.Count);
+        Assert.All(
+            identity.VoiceVectors,
+            stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
+    }
+
+    [Fact]
+    public void AmbiguousDenseClustersPreserveAllVectors()
+    {
+        var identity = IdentityWithVectors(
+            Enumerable.Range(0, 10)
+                .Select(index => UnitVector(0, index + 2, 0.04f))
+                .Concat(Enumerable.Range(0, 10)
+                    .Select(index => UnitVector(1, index + 30, 0.04f)))
+                .ToList());
+        var pruner = CreatePruner();
+
+        var result = pruner.Prune(identity);
+
+        Assert.Equal(0, result);
+        Assert.Equal(20, identity.VoiceVectors.Count);
+    }
+
+    [Fact]
+    public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
+    {
+        var identity = IdentityWithVectors(
+            Enumerable.Range(0, 15)
+                .Select(index => UnitVector(0, index + 2, 0.04f))
+                .Concat(Enumerable.Range(0, 4)
+                    .Select(index => UnitVector(1, index + 30, 0.04f)))
+                .ToList());
+        var pruner = CreatePruner();
+
+        var result = pruner.Prune(identity);
+
+        Assert.Equal(0, result);
+        Assert.Equal(19, identity.VoiceVectors.Count);
+    }
+
+    [Fact]
+    public void PruningPreservesOtherModelsAndMalformedStoredRows()
+    {
+        var identity = IdentityWithVectors(
+            Enumerable.Range(0, 16)
+                .Select(index => UnitVector(0, index + 2, 0.04f))
+                .Concat(Enumerable.Range(0, 4)
+                    .Select(index => UnitVector(1, index + 30, 0.04f)))
+                .ToList());
+        SpeakerVoiceVectors.AddDistinct(
+            identity,
+            [UnitVector(2, 60, 0.04f)],
+            "older-model",
+            1000,
+            DateTimeOffset.UtcNow);
+        identity.VoiceVectors.Add(new SpeakerVoiceVector
+        {
+            ModelId = "resemblyzer-0.1.4-pretrained",
+            Dimensions = 256,
+            VectorBytes = [1],
+            Fingerprint = "malformed",
+            CreatedAt = DateTimeOffset.UtcNow
+        });
+        identity.VoiceVectors.Add(new SpeakerVoiceVector
+        {
+            ModelId = "resemblyzer-0.1.4-pretrained",
+            Dimensions = 256,
+            VectorBytes = new byte[256 * sizeof(float)],
+            Fingerprint = "zero-magnitude",
+            CreatedAt = DateTimeOffset.UtcNow
+        });
+        var pruner = CreatePruner();
+
+        var result = pruner.Prune(identity);
+
+        Assert.Equal(4, result);
+        Assert.Equal(19, identity.VoiceVectors.Count);
+        Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
+        Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
+        Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
+    }
+
+    private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
+    {
+        return new ResemblyzerVoiceVectorOutlierPruner(
+            new ResemblyzerSpeakerRecognitionOptions
+            {
+                ModelId = "resemblyzer-0.1.4-pretrained",
+                OutlierPruningMinimumVectors = 20,
+                OutlierPruningNeighborSimilarity = 0.90,
+                OutlierPruningMinimumNeighbors = 3,
+                OutlierPruningMinimumClusterRatio = 0.60
+            },
+            NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
+    }
+
+    private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
+    {
+        var identity = new SpeakerIdentity();
+        SpeakerVoiceVectors.AddDistinct(
+            identity,
+            vectors,
+            "resemblyzer-0.1.4-pretrained",
+            1000,
+            DateTimeOffset.UtcNow);
+        return identity;
+    }
+
+    private static float[] UnitVector(
+        int primaryDimension,
+        int secondaryDimension,
+        float secondaryValue)
+    {
+        var vector = new float[256];
+        vector[primaryDimension] = 1;
+        vector[secondaryDimension] = secondaryValue;
+        return vector;
+    }
+}