using MeetingAssistant.MeetingNotes; using MeetingAssistant.Transcription; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; namespace MeetingAssistant.Speakers; public sealed class SpeakerIdentityService : ISpeakerIdentificationService { private readonly IDbContextFactory dbContextFactory; private readonly ISpeakerSnippetExtractor snippetExtractor; private readonly ISpeakerIdentityMatcher matcher; private readonly SpeakerIdentificationOptions options; private readonly ILogger logger; public SpeakerIdentityService( IDbContextFactory dbContextFactory, ISpeakerSnippetExtractor snippetExtractor, ISpeakerIdentityMatcher matcher, IOptions options, ILogger logger) { this.dbContextFactory = dbContextFactory; this.snippetExtractor = snippetExtractor; this.matcher = matcher; this.options = options.Value.SpeakerIdentification; this.logger = logger; } public async Task ProcessFinishedTranscriptAsync( SpeakerIdentificationRequest request, CancellationToken cancellationToken) { return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken); } public async Task IdentifyKnownSpeakersAsync( SpeakerIdentificationRequest request, CancellationToken cancellationToken) { return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken); } public async Task IdentifyFinishedSpeakersAsync( SpeakerIdentificationRequest request, CancellationToken cancellationToken) { return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken); } public async Task ApplySpeakerOverrideAsync( SpeakerIdentificationRequest request, string sourceSpeaker, string targetSpeaker, CancellationToken cancellationToken) { if (!options.Enabled || string.IsNullOrWhiteSpace(sourceSpeaker) || string.IsNullOrWhiteSpace(targetSpeaker) || string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase)) { return; } var sourceLabel = sourceSpeaker.Trim(); var targetName = targetSpeaker.Trim(); await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); var now = DateTimeOffset.UtcNow; var meetingReference = CreateReference(request.MeetingNote, now); var snippet = await ResolveOverrideSnippetAsync(request, sourceLabel, cancellationToken); var target = await FindIdentityByAcceptedNameAsync(context, targetName, cancellationToken); var sourceCandidate = await FindCurrentRunCandidateAsync( context, meetingReference, targetName, cancellationToken); if (target is null) { target = sourceCandidate ?? new SpeakerIdentity { CreatedAt = now, CandidateNames = [] }; if (target.Id == 0) { context.SpeakerIdentities.Add(target); } } else if (sourceCandidate is not null && sourceCandidate.Id != target.Id) { MergeOverrideCandidate(target, sourceCandidate); context.SpeakerIdentities.Remove(sourceCandidate); } target.CanonicalName = targetName; target.UpdatedAt = now; ResetCandidates(target, [targetName]); AddMeetingReference(target, meetingReference); AddSnippetIfNeeded(target, snippet); await context.SaveChangesAsync(cancellationToken); await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync( target.References, sourceLabel, targetName, cancellationToken); } public async Task DeleteSpeakerIdentityAsync( string identity, CancellationToken cancellationToken) { if (!options.Enabled || string.IsNullOrWhiteSpace(identity)) { return; } await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); var target = await FindIdentityByAcceptedNameAsync(context, identity.Trim(), cancellationToken); if (target is null) { return; } context.SpeakerIdentities.Remove(target); await context.SaveChangesAsync(cancellationToken); } private async Task ProcessTranscriptAsync( SpeakerIdentificationRequest request, SpeakerIdentityProcessingMode mode, CancellationToken cancellationToken) { if (!options.Enabled || request.Segments.Count == 0) { return new SpeakerIdentificationResult(request.Segments, new Dictionary()); } await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees); var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow); var speakerMappings = new Dictionary(StringComparer.OrdinalIgnoreCase); var attendeeMatches = new List(); var knownSpeakerMappings = request.KnownSpeakerMappings ?? new Dictionary(StringComparer.OrdinalIgnoreCase); var knownDiarizedSpeakers = knownSpeakerMappings.Keys .Where(speaker => !string.IsNullOrWhiteSpace(speaker)) .ToHashSet(StringComparer.OrdinalIgnoreCase); var alreadyIdentifiedNames = knownSpeakerMappings.Values .Where(name => !string.IsNullOrWhiteSpace(name)) .Select(name => name.Trim()) .ToHashSet(StringComparer.OrdinalIgnoreCase); foreach (var speaker in request.Segments .Select(segment => segment.Speaker) .Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker))) { alreadyIdentifiedNames.Add(speaker.Trim()); } var matchedAcceptedNames = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var name in alreadyIdentifiedNames) { matchedAcceptedNames.Add(name); } var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>(); var samplesBySpeaker = request.Samples? .Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0) .GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase) .ToDictionary( group => group.Key, group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(), StringComparer.OrdinalIgnoreCase) ?? new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var group in request.Segments .Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker)) .GroupBy(segment => segment.Speaker) .OrderBy(group => group.Min(segment => segment.Start))) { var speaker = group.Key; if (knownDiarizedSpeakers.Contains(speaker) || alreadyIdentifiedNames.Contains(speaker)) { continue; } if (speakerMappings.ContainsKey(speaker)) { continue; } var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet) ? suppliedSnippet : await snippetExtractor.ExtractSnippetAsync( request.AudioPath, group.ToList(), cancellationToken); if (snippet.Length == 0) { unmatchedSpeakers.Add((speaker, snippet)); continue; } var match = await FindMatchAsync( context, attendees, alreadyIdentifiedNames, speaker, snippet, cancellationToken); if (match is null) { unmatchedSpeakers.Add((speaker, snippet)); continue; } var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken); if (identity is null) { unmatchedSpeakers.Add((speaker, snippet)); continue; } if (mode == SpeakerIdentityProcessingMode.Final) { var previousCanonicalName = identity.CanonicalName; AddMeetingReference(identity, meetingReference); UpdateMatchedIdentity(identity, attendees, snippet); if (string.IsNullOrWhiteSpace(previousCanonicalName) && !string.IsNullOrWhiteSpace(identity.CanonicalName)) { await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync( identity.References, speaker, identity.CanonicalName, cancellationToken); } foreach (var acceptedName in GetAcceptedNames(identity)) { matchedAcceptedNames.Add(acceptedName); } } var speakerName = identity.GetDisplayName(); if (!string.IsNullOrWhiteSpace(speakerName)) { speakerMappings[speaker] = speakerName; alreadyIdentifiedNames.Add(speakerName); foreach (var acceptedName in GetAcceptedNames(identity)) { alreadyIdentifiedNames.Add(acceptedName); } attendeeMatches.Add(new SpeakerIdentityAttendeeMatch( speakerName, GetAcceptedNames(identity).ToList())); } } if (mode == SpeakerIdentityProcessingMode.Final) { await LearnUnmatchedSpeakersAsync( context, attendees, matchedAcceptedNames, unmatchedSpeakers, meetingReference, cancellationToken); } await context.SaveChangesAsync(cancellationToken); var relabeledSegments = request.Segments .Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName) ? segment with { Speaker = speakerName } : segment) .ToList(); return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches); } private async Task FindMatchAsync( SpeakerIdentityDbContext context, IReadOnlyList attendees, IReadOnlySet alreadyIdentifiedNames, string speaker, byte[] snippet, CancellationToken cancellationToken) { var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge; var maxCandidates = Math.Max(1, options.MaxMatchCandidates); var identities = await context.SpeakerIdentities .Include(identity => identity.Snippets) .Include(identity => identity.Aliases) .Include(identity => identity.References) .OrderByDescending(identity => identity.References.Count) .ThenBy(identity => identity.Id) .ToListAsync(cancellationToken); identities = identities .Select(identity => new { Identity = identity, IsAttendee = MatchesAttendees(identity, attendees), IsActive = identity.UpdatedAt >= activeCutoff }) .Where(candidate => candidate.IsAttendee || candidate.IsActive) .Where(candidate => !MatchesAcceptedNames(candidate.Identity, alreadyIdentifiedNames)) .OrderByDescending(candidate => candidate.IsAttendee) .ThenByDescending(candidate => candidate.Identity.ReferenceCount) .ThenBy(candidate => candidate.Identity.Id) .Take(maxCandidates) .Select(candidate => candidate.Identity) .ToList(); foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize))) { var request = new SpeakerIdentityMatchRequest( speaker, snippet, batch.Select(identity => new SpeakerIdentityMatchCandidate( identity.Id, identity.CanonicalName, identity.ReferenceCount, identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList())) .ToList()); var match = await matcher.MatchAsync(request, cancellationToken); if (match is not null) { return match; } } return null; } private async Task ResolveOverrideSnippetAsync( SpeakerIdentificationRequest request, string sourceSpeaker, CancellationToken cancellationToken) { var sample = request.Samples? .Where(sample => string.Equals(sample.Speaker, sourceSpeaker, StringComparison.OrdinalIgnoreCase)) .OrderByDescending(sample => sample.Score) .FirstOrDefault(); if (sample is not null) { return sample.WavBytes; } var segments = request.Segments .Where(segment => string.Equals(segment.Speaker, sourceSpeaker, StringComparison.OrdinalIgnoreCase)) .ToList(); return segments.Count == 0 ? [] : await snippetExtractor.ExtractSnippetAsync(request.AudioPath, segments, cancellationToken); } private static async Task FindIdentityByAcceptedNameAsync( SpeakerIdentityDbContext context, string name, CancellationToken cancellationToken) { var identities = await context.SpeakerIdentities .Include(identity => identity.CandidateNames) .Include(identity => identity.Aliases) .Include(identity => identity.Snippets) .Include(identity => identity.References) .ToListAsync(cancellationToken); return identities .Where(identity => GetAcceptedNames(identity).Contains(name)) .OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1) .ThenBy(identity => identity.Id) .FirstOrDefault(); } private static async Task FindCurrentRunCandidateAsync( SpeakerIdentityDbContext context, SpeakerIdentityReference reference, string targetName, CancellationToken cancellationToken) { var candidates = await context.SpeakerIdentities .Include(identity => identity.CandidateNames) .Include(identity => identity.Aliases) .Include(identity => identity.Snippets) .Include(identity => identity.References) .Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName)) .ToListAsync(cancellationToken); return candidates .Where(identity => identity.References.Any(existing => IsSameReference(existing, reference))) .Where(identity => identity.CandidateNames.Any(candidate => string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) || identity.Aliases.Any(alias => string.Equals(alias.Name, targetName, StringComparison.OrdinalIgnoreCase))) .OrderBy(identity => identity.Id) .FirstOrDefault(); } private void MergeOverrideCandidate( SpeakerIdentity target, SpeakerIdentity source) { AddAlias(target, source.CanonicalName); foreach (var alias in source.Aliases) { AddAlias(target, alias.Name); } foreach (var candidate in source.CandidateNames) { AddAlias(target, candidate.Name); } foreach (var reference in source.References) { SpeakerIdentityReferences.AddIfMissing(target, reference); } foreach (var snippet in source.Snippets) { AddSnippetIfNeeded(target, snippet.WavBytes); } } private static void AddAlias(SpeakerIdentity identity, string? alias) { if (string.IsNullOrWhiteSpace(alias) || string.Equals(identity.CanonicalName, alias, StringComparison.OrdinalIgnoreCase) || identity.Aliases.Any(existing => string.Equals(existing.Name, alias, StringComparison.OrdinalIgnoreCase))) { return; } identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() }); } private static bool IsSameReference( SpeakerIdentityReference first, SpeakerIdentityReference second) { return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) && string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase); } private static bool MatchesAcceptedNames( SpeakerIdentity identity, IReadOnlySet names) { return GetAcceptedNames(identity).Any(names.Contains); } private static bool MatchesAttendees( SpeakerIdentity identity, IReadOnlyList attendees) { var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase); return GetAcceptedNames(identity).Any(attendeeSet.Contains); } private static Task LoadIdentityAsync( SpeakerIdentityDbContext context, int identityId, CancellationToken cancellationToken) { return context.SpeakerIdentities .Include(identity => identity.CandidateNames) .Include(identity => identity.Aliases) .Include(identity => identity.Snippets) .Include(identity => identity.References) .SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken); } private void UpdateMatchedIdentity( SpeakerIdentity identity, IReadOnlyList attendees, byte[] snippet) { identity.UpdatedAt = DateTimeOffset.UtcNow; if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0) { var currentCandidates = identity.CandidateNames .Select(candidate => candidate.Name) .ToHashSet(StringComparer.OrdinalIgnoreCase); var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null; var aliasToCandidate = identity.Aliases .Where(alias => !string.IsNullOrWhiteSpace(alias.Name)) .SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate })) .Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) || pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) || pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase)) .ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase); var intersection = attendees .Select(attendee => currentCandidates.Contains(attendee) ? attendee : aliasToCandidate.GetValueOrDefault(attendee) ?? (identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase)) ? fallbackAliasCandidate : null)) .Where(candidate => !string.IsNullOrWhiteSpace(candidate)) .Select(candidate => candidate!) .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); if (intersection.Count == 0) { ResetCandidates(identity, attendees); ReplaceOldestSnippet(identity, snippet); return; } ResetCandidates(identity, intersection); if (intersection.Count == 1) { identity.CanonicalName = intersection[0]; } } AddSnippetIfNeeded(identity, snippet); } private async Task LearnUnmatchedSpeakersAsync( SpeakerIdentityDbContext context, IReadOnlyList attendees, IEnumerable matchedCanonicalNames, IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers, SpeakerIdentityReference meetingReference, CancellationToken cancellationToken) { var remainingCandidates = attendees .Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); if (remainingCandidates.Count == 0) { return; } foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0)) { var now = DateTimeOffset.UtcNow; var canonicalName = remainingCandidates.Count == 1 ? remainingCandidates[0] : null; var identity = new SpeakerIdentity { CanonicalName = canonicalName, CreatedAt = now, UpdatedAt = now, CandidateNames = remainingCandidates .Select(candidate => new SpeakerCandidateName { Name = candidate }) .ToList(), Snippets = [ new SpeakerSnippet { WavBytes = snippet, CreatedAt = now } ], References = [ SpeakerIdentityReferences.Create( meetingReference.MeetingNotePath, meetingReference.TranscriptPath, now) ] }; context.SpeakerIdentities.Add(identity); if (!string.IsNullOrWhiteSpace(canonicalName)) { await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync( identity.References, speaker, canonicalName, cancellationToken); } } await Task.CompletedTask; } private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet) { if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker) { return; } identity.Snippets.Add(new SpeakerSnippet { WavBytes = snippet, CreatedAt = DateTimeOffset.UtcNow }); } private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet) { var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault(); if (oldest is not null) { identity.Snippets.Remove(oldest); } if (snippet.Length > 0) { identity.Snippets.Add(new SpeakerSnippet { WavBytes = snippet, CreatedAt = DateTimeOffset.UtcNow }); } } private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList candidates) { identity.CandidateNames.Clear(); identity.CandidateNames.AddRange(candidates .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .Select(candidate => new SpeakerCandidateName { Name = candidate })); } private static IReadOnlySet GetAcceptedNames(SpeakerIdentity identity) { return new[] { identity.CanonicalName } .Concat(identity.Aliases.Select(alias => alias.Name)) .Concat(identity.CandidateNames.Select(candidate => candidate.Name)) .Where(name => !string.IsNullOrWhiteSpace(name)) .Select(name => name!.Trim()) .ToHashSet(StringComparer.OrdinalIgnoreCase); } private static IReadOnlyList NormalizeAttendees(IEnumerable attendees) { return attendees .Select(NormalizeAttendee) .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); } private static string NormalizeAttendee(string attendee) { return MeetingAttendeeNames.NormalizeDisplayName(attendee); } private static bool IsDiarizedSpeakerLabel(string speaker) { var normalized = speaker.Trim(); return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) || normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) || normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase); } private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp) { return SpeakerIdentityReferences.Create( meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp); } private static void AddMeetingReference( SpeakerIdentity identity, SpeakerIdentityReference reference) { SpeakerIdentityReferences.AddIfMissing(identity, reference, DateTimeOffset.UtcNow); } private enum SpeakerIdentityProcessingMode { LiveReadOnly, Final } }