feat: add local Resemblyzer speaker recognition

MeetingAssistant/LaunchProfiles/LaunchProfileOptionsProvider.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.Configuration;
+using MeetingAssistant.Speakers;
 
 namespace MeetingAssistant.LaunchProfiles;
 
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
         var options = BindDefaultOptions();
         if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
         {
+            ValidateSpeakerSampleDurations(options, DefaultProfileName);
             return new LaunchProfile(DefaultProfileName, options);
         }
 
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
 
         profileSection.Bind(options);
         ApplyArrayOverrides(profileSection, options);
+        ValidateSpeakerSampleDurations(options, profileName);
         return new LaunchProfile(profileName, options);
     }
 
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
             : name.Trim();
     }
 
+    private static void ValidateSpeakerSampleDurations(
+        MeetingAssistantOptions options,
+        string profileName)
+    {
+        SpeakerSampleDurationConfiguration.ValidateOrThrow(
+            options.SpeakerIdentification,
+            $"Launch profile '{profileName}' has invalid speaker sample durations");
+        ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
+            options.SpeakerIdentification.Resemblyzer,
+            $"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
+    }
+
     private static void ApplyArrayOverrides(
         IConfigurationSection profileSection,
         MeetingAssistantOptions options)

MeetingAssistant/MeetingAssistantOptions.cs

@@ -345,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
 
     public int MaxSnippetsPerSpeaker { get; set; } = 3;
 
-    public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
+    public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
 
-    public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
+    public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
 
     public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
 
@@ -360,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
     public AzureSpeechOptions AzureSpeech { get; set; } = new();
 
     public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
+
+    public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
+}
+
+public sealed class ResemblyzerSpeakerRecognitionOptions
+{
+    public bool Enabled { get; set; }
+
+    public int RequiredVectorsPerSpeaker { get; set; } = 5;
+
+    public int MaxVectorsPerIdentity { get; set; } = 1000;
+
+    public int OutlierPruningMinimumVectors { get; set; } = 20;
+
+    public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
+
+    public int OutlierPruningMinimumNeighbors { get; set; } = 3;
+
+    public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
+
+    public double MinimumClusterCohesion { get; set; } = 0.75;
+
+    public double MinimumIdentitySimilarity { get; set; } = 0.75;
+
+    public double MinimumSimilarityMargin { get; set; } = 0.05;
+
+    public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
+
+    public string PackageVersion { get; set; } = "0.1.4";
+
+    public string PythonCommand { get; set; } = "python";
+
+    public string TorchVersion { get; set; } = "2.14.0+cpu";
+
+    public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
+
+    public string WebRtcVadVersion { get; set; } = "2.0.14";
+
+    public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
+
+    public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
 }
 
 public sealed class SpeakerIdentityPyannoteValidationOptions

MeetingAssistant/Program.cs

@@ -16,7 +16,10 @@ using Microsoft.Extensions.Options;
 
 var builder = WebApplication.CreateBuilder(args);
 builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
-builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
+builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
+builder.Services.AddOptions<MeetingAssistantOptions>()
+    .Bind(builder.Configuration.GetSection("MeetingAssistant"))
+    .ValidateOnStart();
 builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
 #if WINDOWS
 builder.Services.AddSingleton<MicrophoneDeviceSelection>();
@@ -91,8 +94,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
 builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
 builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
 builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
-builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
-builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
+builder.Services.AddSingleton<SpeakerIdentityService>();
+builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
+builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
+    services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
+    services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
+builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
+    services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
+    services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
+builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
+builder.Services.AddSingleton(services =>
+{
+    var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
+        .Value.SpeakerIdentification.Resemblyzer;
+    return resemblyzer.Enabled
+        ? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
+        : SpeakerSampleCollectionPolicy.ExistingBackend;
+});
+builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
+    services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
+        .Value.SpeakerIdentification.Resemblyzer.Enabled
+        ? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
+        : services.GetRequiredService<SpeakerIdentityService>());
+builder.Services.AddSingleton<SpeakerIdentityMergeService>();
+builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
+builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
+    services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
+        .Value.SpeakerIdentification.Resemblyzer.Enabled
+        ? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
+        : services.GetRequiredService<SpeakerIdentityMergeService>());
 builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
 builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
 builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
@@ -134,6 +164,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
 builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
 builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
 builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
+builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
 #if WINDOWS
 builder.Services.AddHostedService<GlobalHotkeyService>();
 builder.Services.AddHostedService<UnoTaskbarIconService>();

MeetingAssistant/Recording/MeetingRecordingCoordinator.cs

@@ -34,6 +34,7 @@ public sealed class MeetingRecordingCoordinator
     private readonly IMeetingInactivityClock inactivityClock;
     private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
     private readonly MeetingAssistantOptions options;
+    private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
     private readonly ILogger<MeetingRecordingCoordinator> logger;
     private readonly SemaphoreSlim gate = new(1, 1);
     private RecordingRun? currentRun;
@@ -63,7 +64,8 @@ public sealed class MeetingRecordingCoordinator
         IMeetingRunArtifactCleaner? artifactCleaner = null,
         IMeetingInactivityPromptService? inactivityPromptService = null,
         IMeetingInactivityClock? inactivityClock = null,
-        IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
+        IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
+        SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
     {
         this.audioSource = audioSource;
         this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -86,6 +88,8 @@ public sealed class MeetingRecordingCoordinator
         this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
         this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
         this.options = options.Value;
+        this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
+            ?? SpeakerSampleCollectionPolicy.ExistingBackend;
         this.logger = logger;
     }
 
@@ -290,7 +294,9 @@ public sealed class MeetingRecordingCoordinator
                 startedAt,
                 launchProfile.Name,
                 runOptions.SpeakerIdentification.LiveSampleBufferDuration,
-                runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
+                speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
+                    runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
+                speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
                 logger);
             run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
             if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
@@ -1169,11 +1175,11 @@ public sealed class MeetingRecordingCoordinator
         try
         {
             var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
-            var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
+            var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
             if (checkpoint is null)
             {
                 logger.LogInformation(
-                    "Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
+                    "Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
                 return;
             }
 
@@ -2079,6 +2085,7 @@ public sealed class MeetingRecordingCoordinator
             string launchProfileName,
             TimeSpan liveSampleBufferDuration,
             int maxSpeakerSamples,
+            bool requireNonOverlappingSpeakerSamples,
             ILogger logger)
         {
             CaptureCancellationSource = captureCancellation;
@@ -2099,8 +2106,9 @@ public sealed class MeetingRecordingCoordinator
                 liveSampleBufferDuration,
                 maxSpeakerSamples,
                 options.SpeakerIdentification.MinimumSampleSpeechDuration,
-                options.SpeakerIdentification.MaximumSampleSegmentGap,
-                logger);
+                options.SpeakerIdentification.MaximumSampleDuration,
+                logger,
+                requireNonOverlappingSpeakerSamples);
         }
 
         public CancellationTokenSource CaptureCancellationSource { get; }
@@ -2508,9 +2516,11 @@ public sealed class MeetingRecordingCoordinator
             return speakerSampleCollector.Snapshot();
         }
 
-        public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
+        public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
+            MeetingNote meetingNote,
+            IReadOnlyList<SpeakerAudioSample> samples)
         {
-            var speakers = GetUnmappedSampleSpeakers();
+            var speakers = GetUnmappedSampleSpeakers(samples);
             if (speakers.Count == 0)
             {
                 return null;
@@ -2518,7 +2528,11 @@ public sealed class MeetingRecordingCoordinator
 
             var checkpoint = new LiveIdentificationCheckpoint(
                 speakers,
-                BuildAttendeeSignature(meetingNote));
+                BuildAttendeeSignature(meetingNote),
+                Options.SpeakerIdentification.Resemblyzer.Enabled
+                    ? speakers.Select(speaker => samples.Count(sample =>
+                        string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
+                    : []);
             lock (liveIdentificationGate)
             {
                 return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
@@ -2576,9 +2590,9 @@ public sealed class MeetingRecordingCoordinator
                 StringComparer.OrdinalIgnoreCase);
         }
 
-        private IReadOnlyList<string> GetUnmappedSampleSpeakers()
+        private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
         {
-            return GetSpeakerSamplesSnapshot()
+            return samples
                 .Select(sample => sample.Speaker)
                 .Where(speaker => !string.IsNullOrWhiteSpace(speaker))
                 .Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
@@ -2634,11 +2648,13 @@ public sealed class MeetingRecordingCoordinator
 
         public sealed record LiveIdentificationCheckpoint(
             IReadOnlyList<string> Speakers,
-            string AttendeeSignature)
+            string AttendeeSignature,
+            IReadOnlyList<int> SampleCounts)
         {
             public bool Matches(LiveIdentificationCheckpoint other)
             {
                 return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
+                    SampleCounts.SequenceEqual(other.SampleCounts) &&
                     Speakers.Count == other.Speakers.Count &&
                     Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
             }

MeetingAssistant/Recording/SpeakerAudioSampleCollector.cs

@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
     private readonly object gate = new();
     private readonly RollingAudioBuffer audioBuffer;
     private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
     private readonly int maxSamplesPerSpeaker;
-    private readonly TimeSpan minimumUninterruptedSpeechDuration;
-    private readonly TimeSpan maximumSegmentGap;
+    private readonly TimeSpan minimumSampleSpeechDuration;
+    private readonly TimeSpan maximumSampleDuration;
+    private readonly bool requireNonOverlappingSamples;
     private readonly ILogger? logger;
-    private PendingSpeakerSpan? pendingSpan;
-
-    public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
-        : this(
-            bufferDuration,
-            maxSamplesPerSpeaker,
-            TimeSpan.FromSeconds(30),
-            TimeSpan.FromSeconds(1),
-            logger: null)
-    {
-    }
+    private SpeakerSampleSpan? pendingSpan;
 
     public SpeakerAudioSampleCollector(
         TimeSpan bufferDuration,
         int maxSamplesPerSpeaker,
-        TimeSpan minimumUninterruptedSpeechDuration,
-        TimeSpan maximumSegmentGap,
-        ILogger? logger = null)
+        TimeSpan minimumSampleSpeechDuration,
+        TimeSpan maximumSampleDuration,
+        ILogger? logger = null,
+        bool requireNonOverlappingSamples = false)
     {
+        SpeakerSampleDurationConfiguration.ValidateOrThrow(
+            minimumSampleSpeechDuration,
+            maximumSampleDuration,
+            "Speaker sample collector configuration is invalid");
         audioBuffer = new RollingAudioBuffer(bufferDuration);
         this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
-        this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
-            ? minimumUninterruptedSpeechDuration
-            : TimeSpan.Zero;
-        this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
-            ? maximumSegmentGap
-            : TimeSpan.Zero;
+        this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
+        this.maximumSampleDuration = maximumSampleDuration;
+        this.requireNonOverlappingSamples = requireNonOverlappingSamples;
         this.logger = logger;
     }
 
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
         lock (gate)
         {
             samplesBySpeaker.Clear();
+            lastAcceptedEndBySpeaker.Clear();
             pendingSpan = null;
             audioBuffer.Reset();
         }
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
             return null;
         }
 
-        TranscriptionSegment sampleSegment;
-        PendingSpanReset? reset;
+        SpeakerSampleSpan? sampleSpan;
+        SpeakerSampleSpan? previousSpan;
         lock (gate)
         {
-            (sampleSegment, reset) = ExtendPendingSpan(segment);
+            if (requireNonOverlappingSamples &&
+                lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
+                segment.Start < lastAcceptedEnd)
+            {
+                logger?.LogInformation(
+                    "Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
+                    segment.Speaker,
+                    lastAcceptedEnd);
+                return null;
+            }
+
+            (sampleSpan, previousSpan) = ExtendPendingSpan(segment);
         }
 
-        if (reset is not null)
+        if (sampleSpan is null)
         {
             logger?.LogInformation(
-                "Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
-                reset.PreviousSpeaker,
-                segment.Speaker,
-                reset.PreviousEnd,
-                segment.Start,
-                reset.Gap,
-                maximumSegmentGap);
+                "Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
+                segment.Speaker);
+            return null;
         }
 
-        var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
+        var sampleSegment = sampleSpan.ToSegment();
+
+        if (previousSpan is not null)
+        {
+            logger?.LogInformation(
+                "Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
+                previousSpan.Speaker,
+                segment.Speaker,
+                previousSpan.End,
+                segment.Start);
+        }
+
+        var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
         if (!score.Accepted)
         {
             logger?.LogInformation(
-                "Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
+                "Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
                 sampleSegment.Speaker,
                 score.Reason,
+                sampleSpan.SpeechDuration,
                 sampleSegment.End - sampleSegment.Start,
-                minimumUninterruptedSpeechDuration,
+                minimumSampleSpeechDuration,
                 score.WordCount);
             return null;
         }
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
         var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
         lock (gate)
         {
+            if (requireNonOverlappingSamples &&
+                ReferenceEquals(pendingSpan, sampleSpan))
+            {
+                pendingSpan = null;
+                lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
+            }
+
             if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
             {
                 samples = [];
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
             !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
     }
 
-    private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
+    private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
     {
-        if (pendingSpan is null ||
-            !SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
+        if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
         {
-            var reset = pendingSpan is null
-                ? null
-                : new PendingSpanReset(
-                    pendingSpan.Speaker,
-                    pendingSpan.End,
-                    segment.Start - pendingSpan.End);
-            pendingSpan = new PendingSpeakerSpan(
-                segment.Speaker,
-                segment.Start,
-                segment.End,
-                [segment.Text]);
-            return (pendingSpan.ToSegment(), reset);
+            pendingSpan = extended;
+            return (pendingSpan, null);
         }
 
-        pendingSpan = pendingSpan.Extend(segment);
-        return (pendingSpan.ToSegment(), null);
+        var previousSpan = pendingSpan;
+        pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
+        return (pendingSpan, previousSpan);
     }
 
     private static SampleScore Score(
         TranscriptionSegment segment,
-        TimeSpan minimumUninterruptedSpeechDuration)
+        TimeSpan speechDuration,
+        TimeSpan minimumSampleSpeechDuration)
     {
-        var durationSeconds = (segment.End - segment.Start).TotalSeconds;
-        if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
+        var durationSeconds = speechDuration.TotalSeconds;
+        if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
         {
             return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
         }
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
             return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
         }
 
-        var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
+        var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
         var wordScore = Math.Min(words / 60.0, 1);
         var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
             segment.Text.TrimEnd().EndsWith('?') ||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
 
     private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
 
-    private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
-
-    private sealed record PendingSpeakerSpan(
-        string Speaker,
-        TimeSpan Start,
-        TimeSpan End,
-        IReadOnlyList<string> TextParts)
-    {
-        public PendingSpeakerSpan Extend(TranscriptionSegment segment)
-        {
-            return this with
-            {
-                End = segment.End > End ? segment.End : End,
-                TextParts = TextParts.Append(segment.Text).ToList()
-            };
-        }
-
-        public TranscriptionSegment ToSegment()
-        {
-            return new TranscriptionSegment(
-                Start,
-                End,
-                Speaker,
-                string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
-        }
-    }
 }

MeetingAssistant/Speakers/ResemblyzerSpeakerIdentificationService.cs

@@ -0,0 +1,631 @@
+using MeetingAssistant.MeetingNotes;
+using MeetingAssistant.Transcription;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Speakers;
+
+public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
+{
+    private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
+    private readonly ISpeakerSnippetExtractor snippetExtractor;
+    private readonly IResemblyzerVoiceEncoder encoder;
+    private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
+    private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
+    private readonly SpeakerIdentificationOptions options;
+    private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
+    private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
+
+    public ResemblyzerSpeakerIdentificationService(
+        IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
+        ISpeakerSnippetExtractor snippetExtractor,
+        IResemblyzerVoiceEncoder encoder,
+        ResemblyzerVoiceClusterMatcher clusterMatcher,
+        ResemblyzerVoiceVectorOutlierPruner outlierPruner,
+        IOptions<MeetingAssistantOptions> options,
+        ILogger<ResemblyzerSpeakerIdentificationService> logger)
+    {
+        this.dbContextFactory = dbContextFactory;
+        this.snippetExtractor = snippetExtractor;
+        this.encoder = encoder;
+        this.clusterMatcher = clusterMatcher;
+        this.outlierPruner = outlierPruner;
+        this.options = options.Value.SpeakerIdentification;
+        resemblyzerOptions = this.options.Resemblyzer;
+        this.logger = logger;
+    }
+
+    public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
+        SpeakerIdentificationRequest request,
+        CancellationToken cancellationToken)
+    {
+        return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
+    }
+
+    public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
+        SpeakerIdentificationRequest request,
+        CancellationToken cancellationToken)
+    {
+        return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
+    }
+
+    public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
+        SpeakerIdentificationRequest request,
+        CancellationToken cancellationToken)
+    {
+        return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
+    }
+
+    public async Task ApplySpeakerOverrideAsync(
+        SpeakerIdentificationRequest request,
+        string sourceSpeaker,
+        string targetSpeaker,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled ||
+            string.IsNullOrWhiteSpace(sourceSpeaker) ||
+            string.IsNullOrWhiteSpace(targetSpeaker) ||
+            string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
+        {
+            return;
+        }
+
+        var sourceLabel = sourceSpeaker.Trim();
+        var targetName = targetSpeaker.Trim();
+        await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
+        var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
+        var target = identities
+            .Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
+            .OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
+            .ThenBy(identity => identity.Id)
+            .FirstOrDefault();
+        var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
+        var sourceCandidate = identities
+            .Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
+            .Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
+            .Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
+            .OrderBy(identity => identity.Id)
+            .FirstOrDefault();
+        var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
+        if (target is null && sourceCandidate is null && vectors.Count == 0)
+        {
+            logger.LogWarning(
+                "Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
+                sourceLabel,
+                targetName);
+            return;
+        }
+
+        var now = DateTimeOffset.UtcNow;
+        if (target is null)
+        {
+            target = sourceCandidate ?? new SpeakerIdentity
+            {
+                CreatedAt = now
+            };
+            if (target.Id == 0)
+            {
+                context.SpeakerIdentities.Add(target);
+            }
+        }
+        else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
+        {
+            SpeakerIdentityMerger.MergeIntoAndPrune(
+                target,
+                sourceCandidate,
+                options.MaxSnippetsPerSpeaker,
+                resemblyzerOptions.MaxVectorsPerIdentity,
+                outlierPruner);
+            context.SpeakerIdentities.Remove(sourceCandidate);
+        }
+
+        target.CanonicalName = targetName;
+        SpeakerIdentityNaming.SetCandidates(target, [targetName]);
+        SpeakerIdentityReferences.AddIfMissing(target, reference, now);
+        outlierPruner.AddAndPrune(
+            target,
+            vectors,
+            now);
+        target.UpdatedAt = now;
+        await context.SaveChangesAsync(cancellationToken);
+        await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
+            target.References,
+            sourceLabel,
+            targetName,
+            cancellationToken);
+    }
+
+    public async Task DeleteSpeakerIdentityAsync(
+        string identity,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
+        {
+            return;
+        }
+
+        await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
+        var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
+        var target = identities.FirstOrDefault(candidate =>
+            SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
+        if (target is null)
+        {
+            return;
+        }
+
+        context.SpeakerIdentities.Remove(target);
+        await context.SaveChangesAsync(cancellationToken);
+    }
+
+    private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
+        SpeakerIdentificationRequest request,
+        bool final,
+        bool allowAudioFallback,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled || request.Segments.Count == 0)
+        {
+            return EmptyResult(request.Segments);
+        }
+
+        await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
+        var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
+        var knownMappings = request.KnownSpeakerMappings ??
+            new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
+        if (final && knownMappings.Count > 0)
+        {
+            await PersistMappedSpeakerEvidenceAsync(
+                context,
+                request,
+                knownMappings,
+                cancellationToken);
+        }
+
+        var identifiedNames = knownMappings.Values
+            .Where(name => !string.IsNullOrWhiteSpace(name))
+            .Select(name => name.Trim())
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        foreach (var segmentSpeaker in request.Segments
+            .Select(segment => segment.Speaker)
+            .Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
+        {
+            identifiedNames.Add(segmentSpeaker.Trim());
+        }
+
+        var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
+        var pendingAudits = new List<PendingIdentificationAudit>();
+        var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
+        var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
+        foreach (var speaker in request.Segments
+            .Select(segment => segment.Speaker)
+            .Where(speaker => !string.IsNullOrWhiteSpace(speaker))
+            .Distinct(StringComparer.OrdinalIgnoreCase))
+        {
+            if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
+            {
+                continue;
+            }
+
+            var vectors = await ResolveAutomaticVectorsAsync(
+                request,
+                speaker,
+                allowAudioFallback,
+                cancellationToken);
+            if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
+            {
+                logger.LogInformation(
+                    "Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
+                    speaker,
+                    vectors.Count,
+                    resemblyzerOptions.RequiredVectorsPerSpeaker);
+                continue;
+            }
+
+            var (identity, decision) = await FindMatchAsync(
+                context,
+                attendees,
+                identifiedNames,
+                vectors,
+                cancellationToken);
+            if (identity is null)
+            {
+                if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
+                {
+                    unmatchedSpeakers.Add((speaker, vectors));
+                }
+
+                continue;
+            }
+
+            var now = DateTimeOffset.UtcNow;
+            var previousCanonicalName = identity.CanonicalName;
+            var previousReferenceCount = identity.References.Count;
+            outlierPruner.AddAndPrune(
+                identity,
+                vectors,
+                now);
+            SpeakerIdentityReferences.AddIfMissing(
+                identity,
+                CreateReference(request.MeetingNote, now),
+                now);
+            if (identity.References.Count != previousReferenceCount)
+            {
+                identity.UpdatedAt = now;
+            }
+
+            if (final)
+            {
+                UpdateMatchedIdentity(identity, attendees);
+                if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
+                    !string.IsNullOrWhiteSpace(identity.CanonicalName))
+                {
+                    pendingAudits.Add(new PendingIdentificationAudit(
+                        identity,
+                        speaker,
+                        identity.CanonicalName));
+                }
+
+                foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
+                {
+                    matchedAcceptedNames.Add(acceptedName);
+                }
+            }
+
+            var displayName = identity.GetDisplayName();
+            if (!string.IsNullOrWhiteSpace(displayName))
+            {
+                mappings[speaker] = displayName;
+                identifiedNames.Add(displayName);
+                foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
+                {
+                    identifiedNames.Add(acceptedName);
+                }
+                attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
+                    displayName,
+                    SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
+            }
+        }
+
+        if (final)
+        {
+            LearnUnmatchedSpeakers(
+                context,
+                request.MeetingNote,
+                attendees,
+                matchedAcceptedNames,
+                unmatchedSpeakers,
+                pendingAudits);
+        }
+
+        await context.SaveChangesAsync(cancellationToken);
+        await AppendAuditsAsync(pendingAudits, cancellationToken);
+        var relabeled = request.Segments
+            .Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
+                ? segment with { Speaker = name }
+                : segment)
+            .ToList();
+        return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
+    }
+
+    private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
+        SpeakerIdentityDbContext context,
+        IReadOnlyList<string> attendees,
+        IReadOnlySet<string> identifiedNames,
+        IReadOnlyList<float[]> queryVectors,
+        CancellationToken cancellationToken)
+    {
+        var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
+        var identities = await LoadIdentities(context)
+            .OrderByDescending(identity => identity.References.Count)
+            .ThenBy(identity => identity.Id)
+            .ToListAsync(cancellationToken);
+        var candidates = identities
+            .Select(identity => new
+            {
+                Identity = identity,
+                IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
+                IsActive = identity.UpdatedAt >= activeCutoff
+            })
+            .Where(candidate => candidate.IsAttendee || candidate.IsActive)
+            .Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
+            .Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
+                string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
+            .OrderByDescending(candidate => candidate.IsAttendee)
+            .ThenByDescending(candidate => candidate.Identity.ReferenceCount)
+            .ThenBy(candidate => candidate.Identity.Id)
+            .Take(Math.Max(1, options.MaxMatchCandidates))
+            .Select(candidate => new ResemblyzerVoiceVectorCandidate(
+                candidate.Identity.Id,
+                SpeakerVoiceVectors.DecodeCompatible(
+                    candidate.Identity,
+                    resemblyzerOptions.ModelId,
+                    logger)))
+            .Where(candidate => candidate.Vectors.Count > 0)
+            .ToList();
+        var match = clusterMatcher.Match(queryVectors, candidates);
+        return (
+            match.IdentityId is { } identityId
+                ? identities.Single(identity => identity.Id == identityId)
+                : null,
+            match);
+    }
+
+    private void LearnUnmatchedSpeakers(
+        SpeakerIdentityDbContext context,
+        MeetingNote meetingNote,
+        IReadOnlyList<string> attendees,
+        IReadOnlySet<string> matchedAcceptedNames,
+        IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
+        ICollection<PendingIdentificationAudit> pendingAudits)
+    {
+        var candidates = attendees
+            .Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+        if (candidates.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var (speaker, vectors) in unmatchedSpeakers)
+        {
+            var now = DateTimeOffset.UtcNow;
+            var identity = new SpeakerIdentity
+            {
+                CanonicalName = candidates.Count == 1 ? candidates[0] : null,
+                CreatedAt = now,
+                UpdatedAt = now,
+                CandidateNames = candidates
+                    .Select(name => new SpeakerCandidateName { Name = name })
+                    .ToList(),
+                References = [CreateReference(meetingNote, now)]
+            };
+            outlierPruner.AddAndPrune(
+                identity,
+                vectors,
+                now);
+            context.SpeakerIdentities.Add(identity);
+            logger.LogInformation(
+                "Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
+                speaker,
+                identity.VoiceVectors.Count,
+                string.Join(", ", candidates));
+            if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
+            {
+                pendingAudits.Add(new PendingIdentificationAudit(
+                    identity,
+                    speaker,
+                    identity.CanonicalName));
+            }
+        }
+    }
+
+    private async Task AppendAuditsAsync(
+        IEnumerable<PendingIdentificationAudit> pendingAudits,
+        CancellationToken cancellationToken)
+    {
+        foreach (var audit in pendingAudits)
+        {
+            try
+            {
+                await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
+                    audit.Identity.References,
+                    audit.Speaker,
+                    audit.Name,
+                    cancellationToken);
+            }
+            catch (Exception exception) when (exception is not OperationCanceledException)
+            {
+                logger.LogError(
+                    exception,
+                    "Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
+                    audit.Identity.Id);
+            }
+        }
+    }
+
+    private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
+        SpeakerIdentificationRequest request,
+        string speaker,
+        bool allowAudioFallback,
+        CancellationToken cancellationToken)
+    {
+        var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
+        var samples = await ResolveWavSamplesAsync(
+            request,
+            speaker,
+            resemblyzerOptions.MaxVectorsPerIdentity,
+            allowAudioFallback,
+            cancellationToken);
+        if (samples.Count < requiredCount)
+        {
+            return [];
+        }
+
+        return await encoder.EncodeAsync(samples, cancellationToken);
+    }
+
+    private async Task PersistMappedSpeakerEvidenceAsync(
+        SpeakerIdentityDbContext context,
+        SpeakerIdentificationRequest request,
+        IReadOnlyDictionary<string, string> knownMappings,
+        CancellationToken cancellationToken)
+    {
+        var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
+        foreach (var (speaker, mappedName) in knownMappings)
+        {
+            var identity = identities
+                .Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
+                .OrderBy(candidate =>
+                    string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
+                .ThenBy(candidate => candidate.Id)
+                .FirstOrDefault();
+            if (identity is null)
+            {
+                logger.LogWarning(
+                    "Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
+                    speaker,
+                    mappedName);
+                continue;
+            }
+
+            var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
+            if (vectors.Count == 0)
+            {
+                continue;
+            }
+
+            var now = DateTimeOffset.UtcNow;
+            var previousReferenceCount = identity.References.Count;
+            var vectorUpdate = outlierPruner.AddAndPrune(
+                identity,
+                vectors,
+                now);
+            SpeakerIdentityReferences.AddIfMissing(
+                identity,
+                CreateReference(request.MeetingNote, now),
+                now);
+            if (identity.References.Count != previousReferenceCount)
+            {
+                identity.UpdatedAt = now;
+            }
+
+            logger.LogInformation(
+                "Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
+                vectorUpdate.AddedCount,
+                speaker,
+                identity.Id);
+        }
+    }
+
+    private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
+        SpeakerIdentificationRequest request,
+        string speaker,
+        CancellationToken cancellationToken)
+    {
+        var samples = await ResolveWavSamplesAsync(
+            request,
+            speaker,
+            resemblyzerOptions.MaxVectorsPerIdentity,
+            allowAudioFallback: true,
+            cancellationToken);
+        if (samples.Count == 0)
+        {
+            return [];
+        }
+
+        return await encoder.EncodeAsync(samples, cancellationToken);
+    }
+
+    private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
+        SpeakerIdentificationRequest request,
+        string speaker,
+        int maxSamples,
+        bool allowAudioFallback,
+        CancellationToken cancellationToken)
+    {
+        var suppliedSamples = request.Samples?
+            .Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
+            .Where(sample => sample.WavBytes.Length > 0)
+            .OrderByDescending(sample => sample.Score)
+            .Take(maxSamples)
+            .ToList()
+            ?? [];
+        var wavSamples = suppliedSamples
+            .Select(sample => sample.WavBytes)
+            .ToList();
+        if (!allowAudioFallback || wavSamples.Count >= maxSamples)
+        {
+            return wavSamples;
+        }
+
+        var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
+            request.Segments,
+            speaker,
+            options.MinimumSampleSpeechDuration,
+            options.MaximumSampleDuration,
+            request.Segments.Count);
+        foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
+        {
+            var wavBytes = await snippetExtractor.ExtractSnippetAsync(
+                request.AudioPath,
+                span,
+                cancellationToken);
+            if (wavBytes.Length > 0)
+            {
+                wavSamples.Add(wavBytes);
+            }
+
+            if (wavSamples.Count >= maxSamples)
+            {
+                break;
+            }
+        }
+
+        logger.LogInformation(
+            "Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
+            wavSamples.Count,
+            maxSamples,
+            speaker,
+            suppliedSamples.Count,
+            wavSamples.Count - suppliedSamples.Count);
+        return wavSamples;
+    }
+
+    private static bool OverlapsSuppliedSample(
+        IReadOnlyList<TranscriptionSegment> span,
+        IReadOnlyList<SpeakerAudioSample> suppliedSamples)
+    {
+        if (span.Count == 0)
+        {
+            return false;
+        }
+
+        var start = span[0].Start;
+        var end = span[^1].End;
+        return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
+    }
+
+    private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
+    {
+        return context.SpeakerIdentities
+            .AsSplitQuery()
+            .Include(identity => identity.Aliases)
+            .Include(identity => identity.CandidateNames)
+            .Include(identity => identity.Snippets)
+            .Include(identity => identity.VoiceVectors)
+            .Include(identity => identity.References);
+    }
+
+    private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
+    {
+        identity.UpdatedAt = DateTimeOffset.UtcNow;
+        if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
+        {
+            return;
+        }
+
+        SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
+    }
+
+    private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
+    {
+        return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
+    }
+
+    private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
+    {
+        return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
+    }
+
+    private sealed record PendingIdentificationAudit(
+        SpeakerIdentity Identity,
+        string Speaker,
+        string Name);
+
+}

MeetingAssistant/Speakers/ResemblyzerSpeakerIdentityMergeService.cs

@@ -0,0 +1,172 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Speakers;
+
+public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
+{
+    private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
+    private readonly ResemblyzerVoiceClusterMatcher matcher;
+    private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
+    private readonly SpeakerIdentificationOptions options;
+    private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
+    private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
+
+    public ResemblyzerSpeakerIdentityMergeService(
+        IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
+        ResemblyzerVoiceClusterMatcher matcher,
+        ResemblyzerVoiceVectorOutlierPruner outlierPruner,
+        IOptions<MeetingAssistantOptions> options,
+        ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
+    {
+        this.dbContextFactory = dbContextFactory;
+        this.matcher = matcher;
+        this.outlierPruner = outlierPruner;
+        this.options = options.Value.SpeakerIdentification;
+        resemblyzerOptions = this.options.Resemblyzer;
+        this.logger = logger;
+    }
+
+    public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
+        TimeSpan? recentIdentityAge,
+        CancellationToken cancellationToken)
+    {
+        await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
+        await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
+        var identities = await context.SpeakerIdentities
+            .AsSplitQuery()
+            .Include(identity => identity.Aliases)
+            .Include(identity => identity.CandidateNames)
+            .Include(identity => identity.Snippets)
+            .Include(identity => identity.VoiceVectors)
+            .Include(identity => identity.References)
+            .OrderByDescending(identity => identity.References.Count)
+            .ThenBy(identity => identity.Id)
+            .ToListAsync(cancellationToken);
+        var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
+        var recentIds = identities
+            .Where(identity => identity.CreatedAt >= cutoff)
+            .Select(identity => identity.Id)
+            .ToList();
+        var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
+        var attempts = 0;
+        var mergedPairs = 0;
+        var pendingAudits = new List<PendingMergeAudit>();
+
+        foreach (var sourceId in recentIds)
+        {
+            var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
+            if (source is null)
+            {
+                continue;
+            }
+
+            var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
+                    source,
+                    resemblyzerOptions.ModelId,
+                    logger)
+                .Take(required * 2)
+                .ToList();
+            if (sourceVectors.Count < required * 2)
+            {
+                logger.LogInformation(
+                    "Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
+                    source.Id,
+                    sourceVectors.Count,
+                    required * 2);
+                continue;
+            }
+
+            var candidates = identities
+                .Where(identity => identity.Id != source.Id)
+                .Where(identity => identity.VoiceVectors.Any(vector =>
+                    string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
+                .Take(Math.Max(1, options.MaxMatchCandidates))
+                .Select(identity => new ResemblyzerVoiceVectorCandidate(
+                    identity.Id,
+                    SpeakerVoiceVectors.DecodeCompatible(
+                        identity,
+                        resemblyzerOptions.ModelId,
+                        logger)))
+                .Where(candidate => candidate.Vectors.Count > 0)
+                .ToList();
+            if (candidates.Count == 0)
+            {
+                continue;
+            }
+
+            attempts++;
+            var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
+            if (!first.Accepted || first.IdentityId is not { } targetId)
+            {
+                continue;
+            }
+
+            attempts++;
+            var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
+            if (!second.Accepted || second.IdentityId != targetId)
+            {
+                logger.LogInformation(
+                    "Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
+                    source.Id,
+                    first.IdentityId,
+                    second.IdentityId);
+                continue;
+            }
+
+            var target = identities.SingleOrDefault(identity => identity.Id == targetId);
+            if (target is null)
+            {
+                continue;
+            }
+
+            var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
+            var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
+            SpeakerIdentityMerger.MergeIntoAndPrune(
+                target,
+                source,
+                options.MaxSnippetsPerSpeaker,
+                resemblyzerOptions.MaxVectorsPerIdentity,
+                outlierPruner);
+            pendingAudits.Add(new PendingMergeAudit(
+                target,
+                targetName,
+                sourceName));
+            context.SpeakerIdentities.Remove(source);
+            identities.Remove(source);
+            mergedPairs++;
+        }
+
+        await context.SaveChangesAsync(cancellationToken);
+        foreach (var audit in pendingAudits)
+        {
+            try
+            {
+                await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
+                    audit.Identity.References,
+                    audit.TargetName,
+                    audit.SourceName,
+                    cancellationToken);
+            }
+            catch (Exception exception) when (exception is not OperationCanceledException)
+            {
+                logger.LogError(
+                    exception,
+                    "Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
+                    audit.Identity.Id);
+            }
+        }
+
+        return new SpeakerIdentityMergeResult(
+            recentIds.Count,
+            identities.Count,
+            attempts,
+            mergedPairs);
+    }
+
+    private sealed record PendingMergeAudit(
+        SpeakerIdentity Identity,
+        string TargetName,
+        string SourceName);
+
+}

MeetingAssistant/Speakers/ResemblyzerVoiceClusterMatcher.cs

@@ -0,0 +1,204 @@
+namespace MeetingAssistant.Speakers;
+
+public sealed record ResemblyzerVoiceVectorCandidate(
+    int IdentityId,
+    IReadOnlyList<float[]> Vectors);
+
+public sealed record ResemblyzerVoiceClusterMatchResult(
+    int? IdentityId,
+    string Reason,
+    double Cohesion,
+    double? BestSimilarity,
+    double? RunnerUpSimilarity)
+{
+    public bool Accepted => IdentityId.HasValue;
+}
+
+public sealed class ResemblyzerVoiceClusterMatcher
+{
+    private readonly ResemblyzerSpeakerRecognitionOptions options;
+    private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
+
+    public ResemblyzerVoiceClusterMatcher(
+        ResemblyzerSpeakerRecognitionOptions options,
+        ILogger<ResemblyzerVoiceClusterMatcher> logger)
+    {
+        this.options = options;
+        this.logger = logger;
+    }
+
+    public ResemblyzerVoiceClusterMatchResult Match(
+        IReadOnlyList<float[]> queryVectors,
+        IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
+    {
+        var requiredCount = options.RequiredVectorsPerSpeaker;
+        if (queryVectors.Count < requiredCount)
+        {
+            return Reject(
+                $"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
+                cohesion: 0,
+                bestSimilarity: null,
+                runnerUpSimilarity: null);
+        }
+
+        var normalizedQuery = queryVectors
+            .Take(requiredCount)
+            .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
+            .ToList();
+        var cohesion = MeanPairwiseCosine(normalizedQuery);
+        if (cohesion < options.MinimumClusterCohesion)
+        {
+            return Reject(
+                $"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
+                cohesion,
+                bestSimilarity: null,
+                runnerUpSimilarity: null);
+        }
+
+        var scored = candidates
+            .Select(candidate => ScoreCandidate(normalizedQuery, candidate))
+            .Where(candidate => candidate is not null)
+            .Select(candidate => candidate!.Value)
+            .OrderByDescending(candidate => candidate.Similarity)
+            .ThenBy(candidate => candidate.IdentityId)
+            .ToList();
+        if (scored.Count == 0)
+        {
+            return Reject("no compatible identity vectors were available", cohesion, null, null);
+        }
+
+        var best = scored[0];
+        var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
+        if (best.Similarity < options.MinimumIdentitySimilarity)
+        {
+            return Reject(
+                $"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
+                cohesion,
+                best.Similarity,
+                runnerUp);
+        }
+
+        if (runnerUp is { } runnerUpSimilarity &&
+            best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
+        {
+            return Reject(
+                $"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
+                cohesion,
+                best.Similarity,
+                runnerUpSimilarity);
+        }
+
+        logger.LogInformation(
+            "Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
+            best.IdentityId,
+            cohesion,
+            best.Similarity,
+            runnerUp);
+        return new ResemblyzerVoiceClusterMatchResult(
+            best.IdentityId,
+            "accepted",
+            cohesion,
+            best.Similarity,
+            runnerUp);
+    }
+
+    private ResemblyzerVoiceClusterMatchResult Reject(
+        string reason,
+        double cohesion,
+        double? bestSimilarity,
+        double? runnerUpSimilarity)
+    {
+        logger.LogInformation(
+            "Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
+            reason,
+            cohesion,
+            bestSimilarity,
+            runnerUpSimilarity);
+        return new ResemblyzerVoiceClusterMatchResult(
+            null,
+            reason,
+            cohesion,
+            bestSimilarity,
+            runnerUpSimilarity);
+    }
+
+    private (int IdentityId, double Similarity)? ScoreCandidate(
+        IReadOnlyList<float[]> normalizedQuery,
+        ResemblyzerVoiceVectorCandidate candidate)
+    {
+        if (candidate.Vectors.Count == 0)
+        {
+            return null;
+        }
+
+        try
+        {
+            var normalizedCandidate = candidate.Vectors
+                .Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
+                .ToList();
+            var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
+            var similarities = normalizedQuery
+                .Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
+                .Order()
+                .ToList();
+            return (candidate.IdentityId, Median(similarities));
+        }
+        catch (InvalidDataException exception)
+        {
+            logger.LogWarning(
+                exception,
+                "Skipping invalid Resemblyzer evidence for identity {IdentityId}",
+                candidate.IdentityId);
+            return null;
+        }
+    }
+
+    private static float[] Centroid(IReadOnlyList<float[]> vectors)
+    {
+        var centroid = new float[ResemblyzerVectorContract.Dimensions];
+        foreach (var vector in vectors)
+        {
+            for (var index = 0; index < centroid.Length; index++)
+            {
+                centroid[index] += vector[index];
+            }
+        }
+
+        for (var index = 0; index < centroid.Length; index++)
+        {
+            centroid[index] /= vectors.Count;
+        }
+
+        return centroid;
+    }
+
+    private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
+    {
+        if (vectors.Count < 2)
+        {
+            return 1;
+        }
+
+        double total = 0;
+        var pairs = 0;
+        for (var first = 0; first < vectors.Count - 1; first++)
+        {
+            for (var second = first + 1; second < vectors.Count; second++)
+            {
+                total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
+                pairs++;
+            }
+        }
+
+        return total / pairs;
+    }
+
+    private static double Median(IReadOnlyList<double> sortedValues)
+    {
+        var middle = sortedValues.Count / 2;
+        return sortedValues.Count % 2 == 0
+            ? (sortedValues[middle - 1] + sortedValues[middle]) / 2
+            : sortedValues[middle];
+    }
+
+}

MeetingAssistant/Speakers/ResemblyzerVoiceVectorOutlierPruner.cs

@@ -0,0 +1,208 @@
+namespace MeetingAssistant.Speakers;
+
+public sealed record ResemblyzerVoiceVectorUpdateResult(
+    int AddedCount,
+    int RemovedCount)
+{
+    public bool Changed => AddedCount > 0 || RemovedCount > 0;
+}
+
+public sealed class ResemblyzerVoiceVectorOutlierPruner
+{
+    private readonly ResemblyzerSpeakerRecognitionOptions options;
+    private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
+
+    public ResemblyzerVoiceVectorOutlierPruner(
+        ResemblyzerSpeakerRecognitionOptions options,
+        ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
+    {
+        this.options = options;
+        this.logger = logger;
+    }
+
+    public int Prune(SpeakerIdentity identity)
+    {
+        var compatible = new List<DecodedSpeakerVoiceVector>();
+        foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
+            identity,
+            options.ModelId,
+            logger))
+        {
+            try
+            {
+                compatible.Add(new DecodedSpeakerVoiceVector(
+                    entry.Stored,
+                    SpeakerVoiceVectors.Normalize(
+                        entry.Vector,
+                        $"Stored voice vector {entry.Stored.Id}")));
+            }
+            catch (InvalidDataException exception)
+            {
+                logger.LogWarning(
+                    exception,
+                    "Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
+                    entry.Stored.Id,
+                    identity.Id);
+            }
+        }
+        var minimumVectors = options.OutlierPruningMinimumVectors;
+        if (compatible.Count < minimumVectors)
+        {
+            return 0;
+        }
+
+        var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
+        var ranked = clusters
+            .GroupBy(cluster => cluster)
+            .Where(group => group.Key > 0)
+            .Select(group => new { Id = group.Key, Count = group.Count() })
+            .OrderByDescending(group => group.Count)
+            .ThenBy(group => group.Id)
+            .ToList();
+        if (ranked.Count == 0)
+        {
+            return Skip(identity, "no dense cluster was found");
+        }
+
+        if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
+        {
+            return Skip(identity, "no uniquely largest dense cluster was found");
+        }
+
+        var dominant = ranked[0];
+        var ratio = (double)dominant.Count / compatible.Count;
+        if (ratio < options.OutlierPruningMinimumClusterRatio)
+        {
+            return Skip(
+                identity,
+                $"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
+        }
+
+        var removed = compatible
+            .Where((_, index) => clusters[index] != dominant.Id)
+            .Select(item => item.Stored)
+            .ToList();
+        foreach (var vector in removed)
+        {
+            identity.VoiceVectors.Remove(vector);
+        }
+
+        if (removed.Count > 0)
+        {
+            identity.UpdatedAt = DateTimeOffset.UtcNow;
+        }
+
+        logger.LogInformation(
+            "Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
+            removed.Count,
+            identity.Id,
+            dominant.Count,
+            compatible.Count);
+        return removed.Count;
+    }
+
+    public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
+        SpeakerIdentity identity,
+        IReadOnlyList<float[]> vectors,
+        DateTimeOffset createdAt)
+    {
+        var removedBeforeAdding = Prune(identity);
+        var added = SpeakerVoiceVectors.AddDistinct(
+            identity,
+            vectors,
+            options.ModelId,
+            options.MaxVectorsPerIdentity,
+            createdAt);
+        var removedAfterAdding = Prune(identity);
+        var result = new ResemblyzerVoiceVectorUpdateResult(
+            added,
+            removedBeforeAdding + removedAfterAdding);
+        if (result.Changed)
+        {
+            identity.UpdatedAt = createdAt;
+        }
+
+        return result;
+    }
+
+    private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
+    {
+        var neighbors = Enumerable.Range(0, vectors.Count)
+            .Select(index => Enumerable.Range(0, vectors.Count)
+                .Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
+                .ToList())
+            .ToList();
+        var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
+        var labels = new int[vectors.Count];
+        var clusterId = 0;
+
+        for (var point = 0; point < vectors.Count; point++)
+        {
+            if (labels[point] != 0)
+            {
+                continue;
+            }
+
+            if (neighbors[point].Count < minimumNeighbors)
+            {
+                labels[point] = -1;
+                continue;
+            }
+
+            clusterId++;
+            ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
+        }
+
+        return labels;
+    }
+
+    private static void ExpandCluster(
+        int seed,
+        int clusterId,
+        IReadOnlyList<List<int>> neighbors,
+        int[] labels,
+        int minimumNeighbors)
+    {
+        labels[seed] = clusterId;
+        var pending = new Queue<int>(neighbors[seed]);
+        var queued = neighbors[seed].ToHashSet();
+        while (pending.TryDequeue(out var point))
+        {
+            if (labels[point] == -1)
+            {
+                labels[point] = clusterId;
+            }
+
+            if (labels[point] != 0)
+            {
+                continue;
+            }
+
+            labels[point] = clusterId;
+            if (neighbors[point].Count < minimumNeighbors)
+            {
+                continue;
+            }
+
+            foreach (var neighbor in neighbors[point])
+            {
+                if (queued.Add(neighbor))
+                {
+                    pending.Enqueue(neighbor);
+                }
+            }
+        }
+    }
+
+    private int Skip(
+        SpeakerIdentity identity,
+        string reason)
+    {
+        logger.LogInformation(
+            "Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
+            identity.Id,
+            reason);
+        return 0;
+    }
+
+}

MeetingAssistant/Speakers/ResemblyzerWarmupHostedService.cs

@@ -0,0 +1,54 @@
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Speakers;
+
+public sealed class ResemblyzerWarmupHostedService : BackgroundService
+{
+    private readonly IResemblyzerVoiceEncoder encoder;
+    private readonly bool speakerIdentificationEnabled;
+    private readonly ResemblyzerSpeakerRecognitionOptions options;
+    private readonly ILogger<ResemblyzerWarmupHostedService> logger;
+    public ResemblyzerWarmupHostedService(
+        IResemblyzerVoiceEncoder encoder,
+        IOptions<MeetingAssistantOptions> options,
+        ILogger<ResemblyzerWarmupHostedService> logger)
+    {
+        this.encoder = encoder;
+        speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
+        this.options = options.Value.SpeakerIdentification.Resemblyzer;
+        this.logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        if (!speakerIdentificationEnabled || !options.Enabled)
+        {
+            return;
+        }
+
+        await Task.Yield();
+        try
+        {
+            logger.LogInformation(
+                "Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
+                VaultPath.Resolve(options.RuntimeFolder),
+                options.ModelId);
+            await encoder.WarmUpAsync(stoppingToken);
+            logger.LogInformation(
+                "Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
+                VaultPath.Resolve(options.RuntimeFolder),
+                options.ModelId);
+        }
+        catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+        {
+            logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(
+                exception,
+                "Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
+                VaultPath.Resolve(options.RuntimeFolder));
+        }
+    }
+}

MeetingAssistant/Speakers/SpeakerIdentity.cs

@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
 
     public List<SpeakerSnippet> Snippets { get; set; } = [];
 
+    public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
+
     public List<SpeakerIdentityReference> References { get; set; } = [];
 
     [NotMapped]
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
             string.IsNullOrWhiteSpace(reference.TranscriptPath);
     }
 
-    private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
+    public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
     {
         return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
             string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
     public DateTimeOffset CreatedAt { get; set; }
 }
 
+public sealed class SpeakerVoiceVector
+{
+    public int Id { get; set; }
+
+    public int SpeakerIdentityId { get; set; }
+
+    public SpeakerIdentity? SpeakerIdentity { get; set; }
+
+    public string ModelId { get; set; } = "";
+
+    public int Dimensions { get; set; }
+
+    public byte[] VectorBytes { get; set; } = [];
+
+    public string Fingerprint { get; set; } = "";
+
+    public DateTimeOffset CreatedAt { get; set; }
+}
+
 public sealed class SpeakerAlias
 {
     public int Id { get; set; }
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
 
     public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
 
+    public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
+
     public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
                 .HasForeignKey(snippet => snippet.SpeakerIdentityId)
                 .OnDelete(DeleteBehavior.Cascade);
 
+            entity.HasMany(identity => identity.VoiceVectors)
+                .WithOne(vector => vector.SpeakerIdentity)
+                .HasForeignKey(vector => vector.SpeakerIdentityId)
+                .OnDelete(DeleteBehavior.Cascade);
+
             entity.HasMany(identity => identity.References)
                 .WithOne(reference => reference.SpeakerIdentity)
                 .HasForeignKey(reference => reference.SpeakerIdentityId)
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
             .HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
             .IsUnique();
 
+        modelBuilder.Entity<SpeakerVoiceVector>()
+            .HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
+            .IsUnique();
+
+        modelBuilder.Entity<SpeakerVoiceVector>()
+            .HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
+
         modelBuilder.Entity<SpeakerIdentityReference>()
             .HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
             .IsUnique();

MeetingAssistant/Speakers/SpeakerIdentityMergeService.cs

@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
 {
     private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
     private readonly ISpeakerIdentityMatcher matcher;
+    private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
     private readonly SpeakerIdentificationOptions options;
     private readonly ILogger<SpeakerIdentityMergeService> logger;
 
     public SpeakerIdentityMergeService(
         IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
         ISpeakerIdentityMatcher matcher,
+        ResemblyzerVoiceVectorOutlierPruner outlierPruner,
         IOptions<MeetingAssistantOptions> options,
         ILogger<SpeakerIdentityMergeService> logger)
     {
         this.dbContextFactory = dbContextFactory;
         this.matcher = matcher;
+        this.outlierPruner = outlierPruner;
         this.options = options.Value.SpeakerIdentification;
         this.logger = logger;
     }
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
 
         var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
         var identities = await context.SpeakerIdentities
+            .AsSplitQuery()
             .Include(identity => identity.Aliases)
             .Include(identity => identity.CandidateNames)
             .Include(identity => identity.Snippets)
+            .Include(identity => identity.VoiceVectors)
             .Include(identity => identity.References)
             .OrderByDescending(identity => identity.References.Count)
             .ThenBy(identity => identity.Id)
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
 
                 var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
                 var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
-                SpeakerIdentityMerger.MergeInto(
+                SpeakerIdentityMerger.MergeIntoAndPrune(
                     target,
                     source,
-                    options.MaxSnippetsPerSpeaker);
+                    options.MaxSnippetsPerSpeaker,
+                    options.Resemblyzer.MaxVectorsPerIdentity,
+                    outlierPruner);
                 logger.LogInformation(
                     "Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
                     source.Id,

MeetingAssistant/Speakers/SpeakerIdentityMerger.cs

@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
 
 internal static class SpeakerIdentityMerger
 {
+    public static void MergeIntoAndPrune(
+        SpeakerIdentity target,
+        SpeakerIdentity source,
+        int maxSnippets,
+        int maxVoiceVectors,
+        ResemblyzerVoiceVectorOutlierPruner outlierPruner)
+    {
+        outlierPruner.Prune(target);
+        outlierPruner.Prune(source);
+        MergeInto(target, source, maxSnippets, maxVoiceVectors);
+        outlierPruner.Prune(target);
+    }
+
     public static void MergeInto(
         SpeakerIdentity target,
         SpeakerIdentity source,
-        int maxSnippets)
+        int maxSnippets,
+        int maxVoiceVectors = int.MaxValue)
     {
         AddAlias(target, source.CanonicalName);
         foreach (var alias in source.Aliases)
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
             .ToList();
         target.Snippets.Clear();
         target.Snippets.AddRange(retainedSnippets);
+
+        var retainedVectors = target.VoiceVectors
+            .Concat(source.VoiceVectors)
+            .GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
+            .Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
+            .OrderByDescending(vector => vector.CreatedAt)
+            .ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
+            .Take(Math.Max(1, maxVoiceVectors))
+            .Select(vector => new SpeakerVoiceVector
+            {
+                SpeakerIdentity = target,
+                ModelId = vector.ModelId,
+                Dimensions = vector.Dimensions,
+                VectorBytes = vector.VectorBytes.ToArray(),
+                Fingerprint = vector.Fingerprint,
+                CreatedAt = vector.CreatedAt
+            })
+            .ToList();
+        target.VoiceVectors.Clear();
+        target.VoiceVectors.AddRange(retainedVectors);
         target.UpdatedAt = DateTimeOffset.UtcNow;
     }
 

MeetingAssistant/Speakers/SpeakerIdentityNaming.cs

@@ -0,0 +1,103 @@
+using MeetingAssistant.MeetingNotes;
+
+namespace MeetingAssistant.Speakers;
+
+internal static class SpeakerIdentityNaming
+{
+    public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
+    {
+        return new[] { identity.CanonicalName }
+            .Concat(identity.Aliases.Select(alias => alias.Name))
+            .Concat(identity.CandidateNames.Select(candidate => candidate.Name))
+            .Where(name => !string.IsNullOrWhiteSpace(name))
+            .Select(name => name!.Trim())
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+    }
+
+    public static bool MatchesAnyAcceptedName(
+        SpeakerIdentity identity,
+        IReadOnlySet<string> names)
+    {
+        return GetAcceptedNames(identity).Any(names.Contains);
+    }
+
+    public static bool MatchesAttendees(
+        SpeakerIdentity identity,
+        IReadOnlyList<string> attendees)
+    {
+        var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
+        return GetAcceptedNames(identity).Any(attendeeSet.Contains);
+    }
+
+    public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
+    {
+        return attendees
+            .Select(MeetingAttendeeNames.NormalizeDisplayName)
+            .Where(attendee => !string.IsNullOrWhiteSpace(attendee))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    public static bool IsDiarizedSpeakerLabel(string speaker)
+    {
+        var normalized = speaker.Trim();
+        return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
+            normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
+            normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
+    }
+
+    public static bool UpdateCandidateNames(
+        SpeakerIdentity identity,
+        IReadOnlyList<string> attendees)
+    {
+        if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
+        {
+            return false;
+        }
+
+        var currentCandidates = identity.CandidateNames
+            .Select(candidate => candidate.Name)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
+        var aliasToCandidate = identity.Aliases
+            .Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
+            .SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
+            .Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
+                pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
+                pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
+            .ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
+        var intersection = attendees
+            .Select(attendee => currentCandidates.Contains(attendee)
+                ? attendee
+                : aliasToCandidate.GetValueOrDefault(attendee) ??
+                    (identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
+                        ? fallbackAliasCandidate
+                        : null))
+            .Where(candidate => !string.IsNullOrWhiteSpace(candidate))
+            .Select(candidate => candidate!)
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+        var resetToAttendees = intersection.Count == 0;
+        SetCandidates(identity, resetToAttendees ? attendees : intersection);
+        if (intersection.Count == 1)
+        {
+            identity.CanonicalName = intersection[0];
+        }
+
+        return resetToAttendees;
+    }
+
+    public static void SetCandidates(
+        SpeakerIdentity identity,
+        IEnumerable<string> candidates)
+    {
+        identity.CandidateNames.Clear();
+        identity.CandidateNames.AddRange(candidates
+            .Where(candidate => !string.IsNullOrWhiteSpace(candidate))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .Select(candidate => new SpeakerCandidateName { Name = candidate }));
+    }
+}

MeetingAssistant/Speakers/SpeakerIdentitySchema.cs

@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
             ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
             """,
             cancellationToken);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
+                "Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
+                "SpeakerIdentityId" INTEGER NOT NULL,
+                "ModelId" TEXT NOT NULL,
+                "Dimensions" INTEGER NOT NULL,
+                "VectorBytes" BLOB NOT NULL,
+                "Fingerprint" TEXT NOT NULL,
+                "CreatedAt" TEXT NOT NULL,
+                CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
+                    FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
+            );
+            """,
+            cancellationToken);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
+            ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
+            """,
+            cancellationToken);
+        await context.Database.ExecuteSqlRawAsync(
+            """
+            CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
+            ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
+            """,
+            cancellationToken);
     }
 
     private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(

MeetingAssistant/Speakers/SpeakerIdentityService.cs

@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
 
         target.CanonicalName = targetName;
         target.UpdatedAt = now;
-        ResetCandidates(target, [targetName]);
+        SpeakerIdentityNaming.SetCandidates(target, [targetName]);
         AddMeetingReference(target, meetingReference);
         var snippetAdded = AddSnippetIfNeeded(target, snippet);
         await context.SaveChangesAsync(cancellationToken);
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
         await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
         await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
 
-        var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
+        var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
         var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
         var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
         var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             .ToHashSet(StringComparer.OrdinalIgnoreCase);
         foreach (var speaker in request.Segments
             .Select(segment => segment.Speaker)
-            .Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
+            .Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
         {
             alreadyIdentifiedNames.Add(speaker.Trim());
         }
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
                         cancellationToken);
                 }
 
-                foreach (var acceptedName in GetAcceptedNames(identity))
+                foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
                 {
                     matchedAcceptedNames.Add(acceptedName);
                 }
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             {
                 speakerMappings[speaker] = speakerName;
                 alreadyIdentifiedNames.Add(speakerName);
-                foreach (var acceptedName in GetAcceptedNames(identity))
+                foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
                 {
                     alreadyIdentifiedNames.Add(acceptedName);
                 }
 
                 attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
                     speakerName,
-                    GetAcceptedNames(identity).ToList()));
+                    SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
             }
         }
 
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             .Select(identity => new
             {
                 Identity = identity,
-                IsAttendee = MatchesAttendees(identity, attendees),
+                IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
                 IsActive = identity.UpdatedAt >= activeCutoff
             })
             .Where(candidate => candidate.IsAttendee || candidate.IsActive)
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
         string operation,
         CancellationToken cancellationToken)
     {
-        var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
+        var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
             request.Segments,
             speaker,
-            options.MaximumSampleSegmentGap,
-            options.MinimumSampleSpeechDuration);
+            options.MinimumSampleSpeechDuration,
+            options.MaximumSampleDuration);
         logger.LogInformation(
             "{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
             operation,
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             .Include(identity => identity.References)
             .ToListAsync(cancellationToken);
         return identities
-            .Where(identity => GetAcceptedNames(identity).Contains(name))
+            .Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
             .OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
             .ThenBy(identity => identity.Id)
             .FirstOrDefault();
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             .Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
             .ToListAsync(cancellationToken);
         return candidates
-            .Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
+            .Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
             .Where(identity => identity.CandidateNames.Any(candidate =>
                 string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
                 identity.Aliases.Any(alias =>
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
         identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
     }
 
-    private static bool IsSameReference(
-        SpeakerIdentityReference first,
-        SpeakerIdentityReference second)
-    {
-        return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
-            string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
-    }
-
     private static bool MatchesAcceptedNames(
         SpeakerIdentity identity,
         IReadOnlySet<string> names)
     {
-        return GetAcceptedNames(identity).Any(names.Contains);
-    }
-
-    private static bool MatchesAttendees(
-        SpeakerIdentity identity,
-        IReadOnlyList<string> attendees)
-    {
-        var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
-        return GetAcceptedNames(identity).Any(attendeeSet.Contains);
+        return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
     }
 
     private static Task<SpeakerIdentity?> LoadIdentityAsync(
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
             var currentCandidates = identity.CandidateNames
                 .Select(candidate => candidate.Name)
                 .ToHashSet(StringComparer.OrdinalIgnoreCase);
-            var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
-            var aliasToCandidate = identity.Aliases
-                .Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
-                .SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
-                .Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
-                    pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
-                    pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
-                .ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
-            var intersection = attendees
-                .Select(attendee => currentCandidates.Contains(attendee)
-                    ? attendee
-                    : aliasToCandidate.GetValueOrDefault(attendee) ??
-                        (identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
-                            ? fallbackAliasCandidate
-                            : null))
-                .Where(candidate => !string.IsNullOrWhiteSpace(candidate))
-                .Select(candidate => candidate!)
-                .Distinct(StringComparer.OrdinalIgnoreCase)
-                .Order(StringComparer.OrdinalIgnoreCase)
-                .ToList();
-
-            if (intersection.Count == 0)
+            var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
+            if (resetToAttendees)
             {
                 logger.LogInformation(
                     "Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
                     identity.Id,
                     FormatNames(attendees));
-                ResetCandidates(identity, attendees);
                 ReplaceOldestSnippet(identity, snippet);
                 return;
             }
 
             logger.LogInformation(
-                "Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
+                "Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
                 identity.Id,
                 FormatNames(currentCandidates),
                 FormatNames(attendees),
-                FormatNames(intersection));
-            ResetCandidates(identity, intersection);
-            if (intersection.Count == 1)
+                FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
+            if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
             {
-                identity.CanonicalName = intersection[0];
                 logger.LogInformation(
                     "Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
                     identity.Id,
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
         }
     }
 
-    private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
-    {
-        identity.CandidateNames.Clear();
-        identity.CandidateNames.AddRange(candidates
-            .Distinct(StringComparer.OrdinalIgnoreCase)
-            .Order(StringComparer.OrdinalIgnoreCase)
-            .Select(candidate => new SpeakerCandidateName { Name = candidate }));
-    }
-
-    private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
-    {
-        return new[]
-            {
-                identity.CanonicalName
-            }
-            .Concat(identity.Aliases.Select(alias => alias.Name))
-            .Concat(identity.CandidateNames.Select(candidate => candidate.Name))
-            .Where(name => !string.IsNullOrWhiteSpace(name))
-            .Select(name => name!.Trim())
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
-    }
-
-    private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
-    {
-        return attendees
-            .Select(NormalizeAttendee)
-            .Where(attendee => !string.IsNullOrWhiteSpace(attendee))
-            .Distinct(StringComparer.OrdinalIgnoreCase)
-            .Order(StringComparer.OrdinalIgnoreCase)
-            .ToList();
-    }
-
-    private static string NormalizeAttendee(string attendee)
-    {
-        return MeetingAttendeeNames.NormalizeDisplayName(attendee);
-    }
-
-    private static bool IsDiarizedSpeakerLabel(string speaker)
-    {
-        var normalized = speaker.Trim();
-        return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
-            normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
-            normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
-    }
-
     private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
     {
         return SpeakerIdentityReferences.Create(

MeetingAssistant/Speakers/SpeakerSampleCollectionPolicy.cs

@@ -0,0 +1,154 @@
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Speakers;
+
+public sealed record SpeakerSampleCollectionPolicy(
+    int MinimumRetainedSamples,
+    bool RequireNonOverlappingSamples)
+{
+    public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
+
+    public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
+    {
+        return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
+    }
+
+    public int ResolveRetainedSampleLimit(int configuredLimit)
+    {
+        return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
+    }
+}
+
+internal static class SpeakerSampleDurationConfiguration
+{
+    public static void ValidateOrThrow(
+        SpeakerIdentificationOptions options,
+        string context)
+    {
+        ValidateOrThrow(
+            options.MinimumSampleSpeechDuration,
+            options.MaximumSampleDuration,
+            context);
+    }
+
+    public static void ValidateOrThrow(
+        TimeSpan minimumDuration,
+        TimeSpan maximumDuration,
+        string context)
+    {
+        var error = GetError(minimumDuration, maximumDuration);
+        if (error is not null)
+        {
+            throw new InvalidOperationException($"{context}: {error}");
+        }
+    }
+
+    public static string? GetError(
+        TimeSpan minimumDuration,
+        TimeSpan maximumDuration)
+    {
+        if (minimumDuration < TimeSpan.Zero)
+        {
+            return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
+        }
+
+        if (maximumDuration <= TimeSpan.Zero)
+        {
+            return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
+        }
+
+        return minimumDuration > maximumDuration
+            ? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
+            : null;
+    }
+}
+
+internal static class ResemblyzerSpeakerRecognitionConfiguration
+{
+    public static void ValidateOrThrow(
+        ResemblyzerSpeakerRecognitionOptions options,
+        string context)
+    {
+        var error = GetError(options);
+        if (error is not null)
+        {
+            throw new InvalidOperationException($"{context}: {error}");
+        }
+    }
+
+    public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
+    {
+        if (options.RequiredVectorsPerSpeaker < 1)
+        {
+            return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
+        }
+
+        if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
+        {
+            return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
+        }
+
+        if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
+        {
+            return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
+        }
+
+        if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
+        {
+            return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
+        }
+
+        if (options.OutlierPruningMinimumNeighbors < 1 ||
+            options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
+        {
+            return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
+        }
+
+        if (!IsCosine(options.OutlierPruningNeighborSimilarity))
+        {
+            return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
+        }
+
+        if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
+            options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
+        {
+            return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
+        }
+
+        if (!IsCosine(options.MinimumClusterCohesion))
+        {
+            return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
+        }
+
+        if (!IsCosine(options.MinimumIdentitySimilarity))
+        {
+            return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
+        }
+
+        return !double.IsFinite(options.MinimumSimilarityMargin) ||
+            options.MinimumSimilarityMargin is < 0 or > 2
+            ? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
+            : null;
+    }
+
+    private static bool IsCosine(double value)
+    {
+        return double.IsFinite(value) && value is >= -1 and <= 1;
+    }
+}
+
+internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
+    : IValidateOptions<MeetingAssistantOptions>
+{
+    public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
+    {
+        var error = SpeakerSampleDurationConfiguration.GetError(
+            options.SpeakerIdentification.MinimumSampleSpeechDuration,
+            options.SpeakerIdentification.MaximumSampleDuration) ??
+            ResemblyzerSpeakerRecognitionConfiguration.GetError(
+                options.SpeakerIdentification.Resemblyzer);
+        return error is null
+            ? ValidateOptionsResult.Success
+            : ValidateOptionsResult.Fail(error);
+    }
+}

MeetingAssistant/Speakers/SpeakerSampleSpan.cs

@@ -0,0 +1,82 @@
+using MeetingAssistant.Transcription;
+
+namespace MeetingAssistant.Speakers;
+
+internal sealed record SpeakerSampleSpan(
+    string Speaker,
+    TimeSpan Start,
+    TimeSpan End,
+    TimeSpan SpeechDuration,
+    TimeSpan MaximumDuration,
+    IReadOnlyList<TranscriptionSegment> Segments)
+{
+    public static SpeakerSampleSpan? Create(
+        TranscriptionSegment segment,
+        TimeSpan maximumDuration)
+    {
+        if (maximumDuration <= TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(
+                nameof(maximumDuration),
+                maximumDuration,
+                "Speaker sample maximum duration must be greater than zero.");
+        }
+
+        var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
+        return boundedEnd > segment.Start
+            ? new SpeakerSampleSpan(
+                segment.Speaker,
+                segment.Start,
+                boundedEnd,
+                boundedEnd - segment.Start,
+                maximumDuration,
+                [segment with { End = boundedEnd }])
+            : null;
+    }
+
+    public bool TryExtend(
+        TranscriptionSegment segment,
+        out SpeakerSampleSpan extended)
+    {
+        extended = this;
+        if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
+            segment.Start >= Start + MaximumDuration)
+        {
+            return false;
+        }
+
+        var boundedEnd = Min(segment.End, Start + MaximumDuration);
+        if (boundedEnd <= segment.Start)
+        {
+            return false;
+        }
+
+        var uncoveredStart = segment.Start > End ? segment.Start : End;
+        var additionalSpeechDuration = boundedEnd > uncoveredStart
+            ? boundedEnd - uncoveredStart
+            : TimeSpan.Zero;
+        extended = this with
+        {
+            End = boundedEnd > End ? boundedEnd : End,
+            SpeechDuration = SpeechDuration + additionalSpeechDuration,
+            Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
+        };
+        return true;
+    }
+
+    public TranscriptionSegment ToSegment()
+    {
+        return new TranscriptionSegment(
+            Start,
+            End,
+            Speaker,
+            string.Join(' ', Segments
+                .Select(segment => segment.Text)
+                .Where(text => !string.IsNullOrWhiteSpace(text))));
+    }
+
+    private static TimeSpan Min(TimeSpan left, TimeSpan right)
+    {
+        return left < right ? left : right;
+    }
+}

MeetingAssistant/Speakers/SpeakerSampleSpanSelector.cs

@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
 
 internal static class SpeakerSampleSpanSelector
 {
-    public static bool CanExtend(
-        string currentSpeaker,
-        TimeSpan currentEnd,
-        TranscriptionSegment nextSegment,
-        TimeSpan maximumSegmentGap)
-    {
-        return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
-            nextSegment.Start - currentEnd <= maximumSegmentGap;
-    }
-
-    public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
+    public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
         IReadOnlyList<TranscriptionSegment> segments,
         string speaker,
-        TimeSpan maximumSegmentGap,
-        TimeSpan minimumDuration)
+        TimeSpan minimumDuration,
+        TimeSpan maximumSampleDuration)
     {
-        var best = new List<TranscriptionSegment>();
-        var current = new List<TranscriptionSegment>();
-
+        SpeakerSampleSpan? best = null;
+        SpeakerSampleSpan? current = null;
         foreach (var segment in segments.OrderBy(segment => segment.Start))
         {
-            if (current.Count == 0)
+            if (!IsSpeaker(segment, speaker))
             {
-                if (IsSpeaker(segment, speaker))
-                {
-                    current.Add(segment);
-                    best = LongerSpan(current, best);
-                }
-
+                current = null;
                 continue;
             }
 
-            if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
+            current = ExtendOrStart(current, segment, maximumSampleDuration);
+            if (current is not null &&
+                (best is null || current.SpeechDuration > best.SpeechDuration))
             {
-                current.Clear();
-                if (!IsSpeaker(segment, speaker))
-                {
-                    continue;
-                }
+                best = current;
             }
-
-            current.Add(segment);
-            best = LongerSpan(current, best);
         }
 
-        return SpanDuration(best) >= minimumDuration
-            ? best
+        return best is not null && best.SpeechDuration >= minimumDuration
+            ? best.Segments
             : [];
     }
 
+    public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
+        IReadOnlyList<TranscriptionSegment> segments,
+        string speaker,
+        TimeSpan minimumDuration,
+        TimeSpan maximumSampleDuration,
+        int maxSpans)
+    {
+        if (maxSpans <= 0)
+        {
+            return [];
+        }
+
+        var completed = new List<SpeakerSampleSpan>();
+        SpeakerSampleSpan? current = null;
+        TimeSpan? lastCompletedEnd = null;
+        foreach (var segment in segments.OrderBy(segment => segment.Start))
+        {
+            if (!IsSpeaker(segment, speaker))
+            {
+                current = null;
+                continue;
+            }
+
+            if (lastCompletedEnd is { } end && segment.Start < end)
+            {
+                current = null;
+                continue;
+            }
+
+            current = ExtendOrStart(current, segment, maximumSampleDuration);
+            if (current is null || current.SpeechDuration < minimumDuration)
+            {
+                continue;
+            }
+
+            completed.Add(current);
+            lastCompletedEnd = current.End;
+            current = null;
+        }
+
+        return completed
+            .OrderByDescending(span => span.SpeechDuration)
+            .ThenBy(span => span.Start)
+            .Take(maxSpans)
+            .OrderBy(span => span.Start)
+            .Select(span => span.Segments)
+            .ToList();
+    }
+
     public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
     {
         return segments.Count == 0
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
             : segments[^1].End - segments[0].Start;
     }
 
+    private static SpeakerSampleSpan? ExtendOrStart(
+        SpeakerSampleSpan? current,
+        TranscriptionSegment segment,
+        TimeSpan maximumSampleDuration)
+    {
+        return current is not null && current.TryExtend(segment, out var extended)
+            ? extended
+            : SpeakerSampleSpan.Create(segment, maximumSampleDuration);
+    }
+
     private static bool IsSpeaker(
         TranscriptionSegment segment,
         string speaker)
     {
         return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
     }
-
-    private static List<TranscriptionSegment> LongerSpan(
-        List<TranscriptionSegment> current,
-        List<TranscriptionSegment> best)
-    {
-        return SpanDuration(current) > SpanDuration(best)
-            ? current.ToList()
-            : best;
-    }
 }

MeetingAssistant/Speakers/SpeakerVoiceVectors.cs

@@ -0,0 +1,191 @@
+using System.Buffers.Binary;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace MeetingAssistant.Speakers;
+
+internal sealed record DecodedSpeakerVoiceVector(
+    SpeakerVoiceVector Stored,
+    float[] Vector);
+
+internal static class ResemblyzerVectorContract
+{
+    public const int Dimensions = 256;
+}
+
+internal static class SpeakerVoiceVectors
+{
+    public static float[] Decode(SpeakerVoiceVector stored)
+    {
+        if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
+            stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
+        {
+            throw new InvalidDataException(
+                $"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
+        }
+
+        var vector = new float[stored.Dimensions];
+        for (var index = 0; index < vector.Length; index++)
+        {
+            vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
+                stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
+            if (!float.IsFinite(vector[index]))
+            {
+                throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
+            }
+        }
+
+        return vector;
+    }
+
+    public static int AddDistinct(
+        SpeakerIdentity identity,
+        IEnumerable<float[]> vectors,
+        string modelId,
+        int maximumCount,
+        DateTimeOffset createdAt)
+    {
+        var limit = Math.Max(1, maximumCount);
+        var fingerprints = identity.VoiceVectors
+            .Select(vector => vector.Fingerprint)
+            .ToHashSet(StringComparer.Ordinal);
+        var added = 0;
+        foreach (var vector in vectors)
+        {
+            if (identity.VoiceVectors.Count >= limit)
+            {
+                break;
+            }
+
+            var bytes = Encode(vector);
+            var fingerprint = Fingerprint(modelId, bytes);
+            if (!fingerprints.Add(fingerprint))
+            {
+                continue;
+            }
+
+            identity.VoiceVectors.Add(new SpeakerVoiceVector
+            {
+                ModelId = modelId,
+                Dimensions = vector.Length,
+                VectorBytes = bytes,
+                Fingerprint = fingerprint,
+                CreatedAt = createdAt
+            });
+            added++;
+        }
+
+        return added;
+    }
+
+    public static IReadOnlyList<float[]> DecodeCompatible(
+        SpeakerIdentity identity,
+        string modelId,
+        ILogger logger)
+    {
+        return DecodeCompatibleEntries(identity, modelId, logger)
+            .Select(entry => entry.Vector)
+            .ToList();
+    }
+
+    public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
+        SpeakerIdentity identity,
+        string modelId,
+        ILogger logger)
+    {
+        var vectors = new List<DecodedSpeakerVoiceVector>();
+        foreach (var stored in identity.VoiceVectors
+            .Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
+            .OrderBy(vector => vector.CreatedAt)
+            .ThenBy(vector => vector.Id))
+        {
+            try
+            {
+                vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
+            }
+            catch (InvalidDataException exception)
+            {
+                logger.LogWarning(
+                    exception,
+                    "Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
+                    stored.Id,
+                    identity.Id);
+            }
+        }
+
+        return vectors;
+    }
+
+    public static double Cosine(float[] first, float[] second)
+    {
+        double dot = 0;
+        for (var index = 0; index < first.Length; index++)
+        {
+            dot += first[index] * second[index];
+        }
+
+        return dot;
+    }
+
+    public static float[] Normalize(float[]? vector, string description)
+    {
+        if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
+        {
+            throw new InvalidDataException(
+                $"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
+        }
+
+        double magnitudeSquared = 0;
+        foreach (var value in vector)
+        {
+            if (!float.IsFinite(value))
+            {
+                throw new InvalidDataException($"{description} contains a non-finite value.");
+            }
+
+            magnitudeSquared += value * value;
+        }
+
+        var magnitude = Math.Sqrt(magnitudeSquared);
+        if (magnitude <= double.Epsilon)
+        {
+            throw new InvalidDataException($"{description} has zero magnitude.");
+        }
+
+        return vector.Select(value => (float)(value / magnitude)).ToArray();
+    }
+
+    private static byte[] Encode(float[] vector)
+    {
+        if (vector.Length != ResemblyzerVectorContract.Dimensions)
+        {
+            throw new InvalidDataException(
+                $"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
+        }
+
+        var bytes = new byte[vector.Length * sizeof(float)];
+        for (var index = 0; index < vector.Length; index++)
+        {
+            if (!float.IsFinite(vector[index]))
+            {
+                throw new InvalidDataException("Voice vector contains a non-finite value.");
+            }
+
+            BinaryPrimitives.WriteSingleLittleEndian(
+                bytes.AsSpan(index * sizeof(float), sizeof(float)),
+                vector[index]);
+        }
+
+        return bytes;
+    }
+
+    private static string Fingerprint(string modelId, byte[] vectorBytes)
+    {
+        var modelBytes = Encoding.UTF8.GetBytes(modelId);
+        var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
+        modelBytes.CopyTo(input, 0);
+        input[modelBytes.Length] = 0;
+        vectorBytes.CopyTo(input, modelBytes.Length + 1);
+        return Convert.ToHexString(SHA256.HashData(input));
+    }
+}

MeetingAssistant/Speakers/VenvResemblyzerVoiceEncoder.cs

@@ -0,0 +1,391 @@
+using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using MeetingAssistant.Transcription;
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Speakers;
+
+public interface IResemblyzerVoiceEncoder
+{
+    Task<IReadOnlyList<float[]>> EncodeAsync(
+        IReadOnlyList<byte[]> wavSamples,
+        CancellationToken cancellationToken);
+
+    Task WarmUpAsync(CancellationToken cancellationToken);
+}
+
+public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
+{
+    private const string EnvironmentSchemaVersion = "venv-v2";
+    private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
+    private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
+    private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
+    private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
+    private const string LibrosaRequirement = "librosa>=0.9.1";
+    private const string ScipyRequirement = "scipy>=1.2.1";
+    private readonly SemaphoreSlim commandLock = new(1, 1);
+    private readonly ICommandRunner commandRunner;
+    private readonly ResemblyzerSpeakerRecognitionOptions options;
+    private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
+    private string? verifiedEnvironmentPythonPath;
+
+    public VenvResemblyzerVoiceEncoder(
+        ICommandRunner commandRunner,
+        IOptions<MeetingAssistantOptions> options,
+        ILogger<VenvResemblyzerVoiceEncoder> logger)
+    {
+        this.commandRunner = commandRunner;
+        this.options = options.Value.SpeakerIdentification.Resemblyzer;
+        this.logger = logger;
+    }
+
+    public async Task<IReadOnlyList<float[]>> EncodeAsync(
+        IReadOnlyList<byte[]> wavSamples,
+        CancellationToken cancellationToken)
+    {
+        if (wavSamples.Count == 0)
+        {
+            return [];
+        }
+
+        if (wavSamples.Any(sample => sample.Length == 0))
+        {
+            throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
+        }
+
+        var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
+        var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(inputFolder);
+        try
+        {
+            for (var index = 0; index < wavSamples.Count; index++)
+            {
+                await File.WriteAllBytesAsync(
+                    Path.Combine(inputFolder, $"{index:D4}.wav"),
+                    wavSamples[index],
+                    cancellationToken);
+            }
+
+            await commandLock.WaitAsync(cancellationToken);
+            try
+            {
+                return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
+            }
+            finally
+            {
+                commandLock.Release();
+            }
+        }
+        finally
+        {
+            try
+            {
+                Directory.Delete(inputFolder, recursive: true);
+            }
+            catch (DirectoryNotFoundException)
+            {
+            }
+            catch (IOException exception)
+            {
+                logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
+            }
+            catch (UnauthorizedAccessException exception)
+            {
+                logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
+            }
+        }
+    }
+
+    private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
+        string inputFolder,
+        int expectedCount,
+        CancellationToken cancellationToken)
+    {
+        return await RunWithTimeoutAsync(async token =>
+        {
+            var pythonPath = await EnsureEnvironmentAsync(token);
+            var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
+            await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
+            var result = await RunRequiredAsync(
+                pythonPath,
+                [scriptPath, Path.GetFullPath(inputFolder)],
+                "encoding",
+                token);
+            return ParseAndValidateVectors(result.StandardOutput, expectedCount);
+        }, cancellationToken);
+    }
+
+    public async Task WarmUpAsync(CancellationToken cancellationToken)
+    {
+        await commandLock.WaitAsync(cancellationToken);
+        try
+        {
+            await RunWithTimeoutAsync(async token =>
+            {
+                await EnsureEnvironmentAsync(token);
+                return true;
+            }, cancellationToken);
+        }
+        finally
+        {
+            commandLock.Release();
+        }
+    }
+
+    private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
+    {
+        ValidateDependencySettings();
+        var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
+        var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
+        var pythonPath = GetEnvironmentPythonPath(environmentFolder);
+        var readyPath = Path.Combine(environmentFolder, ".ready");
+        if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
+            && File.Exists(pythonPath)
+            && File.Exists(readyPath))
+        {
+            return pythonPath;
+        }
+
+        if (File.Exists(pythonPath) && File.Exists(readyPath))
+        {
+            var verification = await commandRunner.RunAsync(
+                pythonPath,
+                WarmupArguments,
+                cancellationToken);
+            if (verification.ExitCode == 0)
+            {
+                verifiedEnvironmentPythonPath = pythonPath;
+                return pythonPath;
+            }
+
+            logger.LogWarning(
+                "Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
+                verification.ExitCode,
+                verification.StandardError);
+            File.Delete(readyPath);
+        }
+
+        Directory.CreateDirectory(environmentFolder);
+        await RunRequiredAsync(
+            options.PythonCommand,
+            ["-m", "venv", "--clear", environmentFolder],
+            "virtual environment creation",
+            cancellationToken);
+        await RunRequiredAsync(
+            pythonPath,
+            ["-m", "pip", "install", "--upgrade", "pip"],
+            "pip upgrade",
+            cancellationToken);
+        await RunRequiredAsync(
+            pythonPath,
+            [
+                "-m", "pip", "install",
+                "--index-url", options.TorchIndexUrl,
+                TorchRequirement
+            ],
+            "CPU PyTorch installation",
+            cancellationToken);
+        await RunRequiredAsync(
+            pythonPath,
+            [
+                "-m", "pip", "install",
+                .. RuntimeDependencyRequirements
+            ],
+            "Resemblyzer dependency installation",
+            cancellationToken);
+        await RunRequiredAsync(
+            pythonPath,
+            [
+                "-m", "pip", "install", "--no-deps",
+                ResemblyzerRequirement
+            ],
+            "Resemblyzer installation",
+            cancellationToken);
+        await RunRequiredAsync(
+            pythonPath,
+            WarmupArguments,
+            "environment verification",
+            cancellationToken);
+        await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
+        verifiedEnvironmentPythonPath = pythonPath;
+        return pythonPath;
+    }
+
+    private async Task<CommandResult> RunRequiredAsync(
+        string fileName,
+        IReadOnlyList<string> arguments,
+        string operation,
+        CancellationToken cancellationToken)
+    {
+        var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
+        ThrowIfFailed(result, operation);
+        return result;
+    }
+
+    private static string BuildEncodingScript()
+    {
+        return
+            "import json\n"
+            + "import sys\n"
+            + "import numpy as np\n"
+            + "from pathlib import Path\n"
+            + "from resemblyzer import VoiceEncoder, preprocess_wav\n"
+            + "encoder = VoiceEncoder('cpu', verbose=False)\n"
+            + "vectors = []\n"
+            + "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
+            + "    wav = preprocess_wav(path)\n"
+            + "    vector = encoder.embed_utterance(wav)\n"
+            + "    vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
+            + $"print('{JsonStart}')\n"
+            + "print(json.dumps(vectors, allow_nan=False))\n"
+            + $"print('{JsonEnd}')\n";
+    }
+
+    private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
+    {
+        var json = ExtractJson(output);
+        float[][]? vectors;
+        try
+        {
+            vectors = JsonSerializer.Deserialize<float[][]>(json);
+        }
+        catch (JsonException exception)
+        {
+            throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
+        }
+
+        if (vectors is null || vectors.Length != expectedCount)
+        {
+            throw new InvalidDataException(
+                $"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
+        }
+
+        return vectors
+            .Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
+            .ToList();
+    }
+
+    private static string ExtractJson(string output)
+    {
+        var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
+        if (start < 0)
+        {
+            throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
+        }
+
+        start += JsonStart.Length;
+        var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
+        if (end < 0)
+        {
+            throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
+        }
+
+        return output[start..end].Trim();
+    }
+
+    private string BuildEnvironmentKey()
+    {
+        var settings = string.Join('\n', EnvironmentManifest);
+        var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
+        return Convert.ToHexString(hash)[..16].ToLowerInvariant();
+    }
+
+    private string TorchRequirement => $"torch=={options.TorchVersion}";
+
+    private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
+
+    private static string[] WarmupArguments =>
+    [
+        "-c",
+        "from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
+    ];
+
+    private string[] RuntimeDependencyRequirements =>
+    [
+        NumpyBeforePython313Requirement,
+        NumpyFromPython313Requirement,
+        LibrosaRequirement,
+        ScipyRequirement,
+        $"webrtcvad-wheels=={options.WebRtcVadVersion}"
+    ];
+
+    private IEnumerable<string> EnvironmentManifest =>
+    [
+        EnvironmentSchemaVersion,
+        options.PythonCommand,
+        options.TorchIndexUrl,
+        TorchRequirement,
+        .. RuntimeDependencyRequirements,
+        ResemblyzerRequirement
+    ];
+
+    private static string GetEnvironmentPythonPath(string environmentFolder)
+    {
+        return OperatingSystem.IsWindows()
+            ? Path.Combine(environmentFolder, "Scripts", "python.exe")
+            : Path.Combine(environmentFolder, "bin", "python");
+    }
+
+    private void ValidateDependencySettings()
+    {
+        if (string.IsNullOrWhiteSpace(options.PythonCommand))
+        {
+            throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
+        }
+
+        ValidateVersion(options.PackageVersion, "Resemblyzer package");
+        ValidateVersion(options.TorchVersion, "PyTorch");
+        ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
+        if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
+            || indexUri.Scheme != Uri.UriSchemeHttps)
+        {
+            throw new InvalidOperationException(
+                $"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
+        }
+    }
+
+    private static void ValidateVersion(string version, string dependency)
+    {
+        if (!PackageVersionPattern().IsMatch(version))
+        {
+            throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
+        }
+    }
+
+    private async Task<T> RunWithTimeoutAsync<T>(
+        Func<CancellationToken, Task<T>> operation,
+        CancellationToken cancellationToken)
+    {
+        using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
+            ? new CancellationTokenSource(options.CommandTimeout)
+            : null;
+        using var linkedSource = timeoutSource is null
+            ? null
+            : CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
+        try
+        {
+            return await operation(linkedSource?.Token ?? cancellationToken);
+        }
+        catch (OperationCanceledException) when (
+            !cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
+        {
+            throw new TimeoutException(
+                $"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
+        }
+    }
+
+    private static void ThrowIfFailed(CommandResult result, string operation)
+    {
+        if (result.ExitCode != 0)
+        {
+            throw new InvalidOperationException(
+                $"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
+        }
+    }
+
+    [GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
+    private static partial Regex PackageVersionPattern();
+}

MeetingAssistant/Transcription/PyannoteDiarizationWarmupHostedService.cs

@@ -120,7 +120,8 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
     private static IEnumerable<PyannoteRuntimeOptions> GetEnabledSpeakerValidationDiarizationOptions(
         MeetingAssistantOptions options)
     {
-        if (options.SpeakerIdentification.PyannoteValidation.Enabled)
+        if (!options.SpeakerIdentification.Resemblyzer.Enabled &&
+            options.SpeakerIdentification.PyannoteValidation.Enabled)
         {
             yield return options.SpeakerIdentification.PyannoteValidation.Diarization;
         }

MeetingAssistant/Workflow/WorkflowRulesEditorChatPipeline.cs

@@ -24,6 +24,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
     private readonly IMeetingMetadataProvider meetingMetadataProvider;
     private readonly ILaunchProfileOptionsProvider launchProfiles;
     private readonly ISpeakerIdentityMergeService identityMergeService;
+    private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
     private readonly AsrDiagnosticService asrDiagnosticService;
     private readonly IConfiguration configuration;
 
@@ -38,6 +39,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
         IMeetingMetadataProvider meetingMetadataProvider,
         ILaunchProfileOptionsProvider launchProfiles,
         ISpeakerIdentityMergeService identityMergeService,
+        ResemblyzerVoiceVectorOutlierPruner outlierPruner,
         AsrDiagnosticService asrDiagnosticService,
         IConfiguration configuration)
     {
@@ -51,6 +53,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
         this.meetingMetadataProvider = meetingMetadataProvider;
         this.launchProfiles = launchProfiles;
         this.identityMergeService = identityMergeService;
+        this.outlierPruner = outlierPruner;
         this.asrDiagnosticService = asrDiagnosticService;
         this.configuration = configuration;
     }
@@ -78,7 +81,8 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
             identityMergeService,
             asrDiagnosticService,
             configuration,
-            logger: logger);
+            logger: logger,
+            outlierPruner: outlierPruner);
         var messages = conversation
             .Select(ToChatMessage)
             .Append(new ChatMessage(ChatRole.User, userMessage.Trim()))

MeetingAssistant/Workflow/WorkflowRulesEditorTools.cs

@@ -10,6 +10,7 @@ using MeetingAssistant.Transcription;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using YamlDotNet.Core;
 using YamlDotNet.Serialization;
 
@@ -42,6 +43,7 @@ public sealed class WorkflowRulesEditorTools
     private readonly IMeetingMetadataProvider? meetingMetadataProvider;
     private readonly ILaunchProfileOptionsProvider? launchProfiles;
     private readonly ISpeakerIdentityMergeService? identityMergeService;
+    private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
     private readonly AsrDiagnosticService? asrDiagnosticService;
     private readonly IConfiguration? configuration;
     private readonly ILogger? logger;
@@ -61,7 +63,8 @@ public sealed class WorkflowRulesEditorTools
         string? logDirectory = null,
         string? specRootPath = null,
         string? projectAgentsTemplatePath = null,
-        ILogger? logger = null)
+        ILogger? logger = null,
+        ResemblyzerVoiceVectorOutlierPruner? outlierPruner = null)
     {
         this.options = options;
         rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath);
@@ -82,6 +85,9 @@ public sealed class WorkflowRulesEditorTools
         this.meetingMetadataProvider = meetingMetadataProvider;
         this.launchProfiles = launchProfiles;
         this.identityMergeService = identityMergeService;
+        this.outlierPruner = outlierPruner ?? new ResemblyzerVoiceVectorOutlierPruner(
+            speakerOptions.Resemblyzer,
+            NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
         this.asrDiagnosticService = asrDiagnosticService;
         this.configuration = configuration;
         this.logger = logger;
@@ -743,8 +749,8 @@ public sealed class WorkflowRulesEditorTools
         string[]? candidateNames = null)
     {
         return Task.FromResult(
-            "Refused: speaker identities require at least one audio sample. " +
-            "Use a speaker override from an existing transcript speaker sample, or update/merge an existing sampled identity.");
+            "Refused: speaker identities require audio evidence (a WAV sample or voice vector). " +
+            "Use a speaker override from an existing transcript speaker, or update/merge an existing identity.");
     }
 
     public async Task<string> UpdateIdentity(
@@ -826,10 +832,12 @@ public sealed class WorkflowRulesEditorTools
                 return $"Could not find target {targetIdentityId} or source {sourceIdentityId}.";
             }
 
-            SpeakerIdentityMerger.MergeInto(
+            SpeakerIdentityMerger.MergeIntoAndPrune(
                 target,
                 source,
-                speakerOptions.MaxSnippetsPerSpeaker);
+                speakerOptions.MaxSnippetsPerSpeaker,
+                speakerOptions.Resemblyzer.MaxVectorsPerIdentity,
+                outlierPruner);
             context.SpeakerIdentities.Remove(source);
             await context.SaveChangesAsync();
             return ToJson(ToIdentityDetail(target));
@@ -1567,9 +1575,11 @@ public sealed class WorkflowRulesEditorTools
     private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
     {
         return context.SpeakerIdentities
+            .AsSplitQuery()
             .Include(identity => identity.Aliases)
             .Include(identity => identity.CandidateNames)
             .Include(identity => identity.Snippets)
+            .Include(identity => identity.VoiceVectors)
             .Include(identity => identity.References);
     }
 
@@ -1591,6 +1601,7 @@ public sealed class WorkflowRulesEditorTools
             identity.Aliases.Select(alias => alias.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
             identity.CandidateNames.Select(candidate => candidate.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
             identity.Snippets.Count,
+            identity.VoiceVectors.Count,
             identity.References.Count,
             identity.UpdatedAt);
     }
@@ -1650,6 +1661,7 @@ public sealed class WorkflowRulesEditorTools
         IReadOnlyList<string> Aliases,
         IReadOnlyList<string> CandidateNames,
         int SampleCount,
+        int VoiceVectorCount,
         int ReferenceCount,
         DateTimeOffset UpdatedAt);
 

MeetingAssistant/appsettings.json

@@ -120,8 +120,8 @@
       "MaxMatchCandidates": 100,
       "MatchIdentityActiveAge": "365.00:00:00",
       "MaxSnippetsPerSpeaker": 3,
-      "MinimumSampleSpeechDuration": "00:00:30",
-      "MaximumSampleSegmentGap": "00:00:01",
+      "MinimumSampleSpeechDuration": "00:00:10",
+      "MaximumSampleDuration": "00:01:00",
       "SilenceBetweenSnippetsSeconds": 1,
       "LiveSampleBufferDuration": "00:10:00",
       "MergeRecentIdentityAge": "14.00:00:00",
@@ -142,6 +142,26 @@
           "TokenEnv": "HF_TOKEN",
           "CommandTimeout": "01:00:00"
         }
+      },
+      "Resemblyzer": {
+        "Enabled": false,
+        "RequiredVectorsPerSpeaker": 5,
+        "MaxVectorsPerIdentity": 1000,
+        "OutlierPruningMinimumVectors": 20,
+        "OutlierPruningNeighborSimilarity": 0.75,
+        "OutlierPruningMinimumNeighbors": 3,
+        "OutlierPruningMinimumClusterRatio": 0.60,
+        "MinimumClusterCohesion": 0.75,
+        "MinimumIdentitySimilarity": 0.75,
+        "MinimumSimilarityMargin": 0.05,
+        "ModelId": "resemblyzer-0.1.4-pretrained",
+        "PackageVersion": "0.1.4",
+        "PythonCommand": "python",
+        "TorchVersion": "2.14.0+cpu",
+        "TorchIndexUrl": "https://download.pytorch.org/whl/cpu",
+        "WebRtcVadVersion": "2.0.14",
+        "RuntimeFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Resemblyzer",
+        "CommandTimeout": "00:15:00"
       }
     },
     "Automation": {