using MeetingAssistant.Transcription; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; namespace MeetingAssistant.Speakers; public sealed class SpeakerIdentityService : ISpeakerIdentificationService { private readonly IDbContextFactory dbContextFactory; private readonly ISpeakerSnippetExtractor snippetExtractor; private readonly ISpeakerIdentityMatcher matcher; private readonly SpeakerIdentificationOptions options; private readonly ILogger logger; public SpeakerIdentityService( IDbContextFactory dbContextFactory, ISpeakerSnippetExtractor snippetExtractor, ISpeakerIdentityMatcher matcher, IOptions options, ILogger logger) { this.dbContextFactory = dbContextFactory; this.snippetExtractor = snippetExtractor; this.matcher = matcher; this.options = options.Value.SpeakerIdentification; this.logger = logger; } public async Task ProcessFinishedTranscriptAsync( SpeakerIdentificationRequest request, CancellationToken cancellationToken) { return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken); } public async Task IdentifyKnownSpeakersAsync( SpeakerIdentificationRequest request, CancellationToken cancellationToken) { return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken); } private async Task ProcessTranscriptAsync( SpeakerIdentificationRequest request, SpeakerIdentityProcessingMode mode, CancellationToken cancellationToken) { if (!options.Enabled || request.Segments.Count == 0) { return new SpeakerIdentificationResult(request.Segments, new Dictionary()); } await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken); await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken); var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees); var speakerMappings = new Dictionary(StringComparer.OrdinalIgnoreCase); var attendeeMatches = new List(); var matchedAcceptedNames = new HashSet(StringComparer.OrdinalIgnoreCase); var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>(); var samplesBySpeaker = request.Samples? .Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0) .GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase) .ToDictionary( group => group.Key, group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(), StringComparer.OrdinalIgnoreCase) ?? new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var group in request.Segments .Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker)) .GroupBy(segment => segment.Speaker) .OrderBy(group => group.Min(segment => segment.Start))) { var speaker = group.Key; if (speakerMappings.ContainsKey(speaker)) { continue; } var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet) ? suppliedSnippet : await snippetExtractor.ExtractSnippetAsync( request.AudioPath, group.ToList(), cancellationToken); if (snippet.Length == 0) { unmatchedSpeakers.Add((speaker, snippet)); continue; } var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken); if (match is null) { unmatchedSpeakers.Add((speaker, snippet)); continue; } var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken); if (identity is null) { unmatchedSpeakers.Add((speaker, snippet)); continue; } if (mode == SpeakerIdentityProcessingMode.Final) { UpdateMatchedIdentity(identity, attendees, snippet); foreach (var acceptedName in GetAcceptedNames(identity)) { matchedAcceptedNames.Add(acceptedName); } } var speakerName = identity.GetDisplayName(); if (!string.IsNullOrWhiteSpace(speakerName)) { speakerMappings[speaker] = speakerName; attendeeMatches.Add(new SpeakerIdentityAttendeeMatch( speakerName, GetAcceptedNames(identity).ToList())); } } if (mode == SpeakerIdentityProcessingMode.Final) { await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken); } await context.SaveChangesAsync(cancellationToken); var relabeledSegments = request.Segments .Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName) ? segment with { Speaker = speakerName } : segment) .ToList(); return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches); } private async Task FindMatchAsync( SpeakerIdentityDbContext context, IReadOnlyList attendees, string speaker, byte[] snippet, CancellationToken cancellationToken) { var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge; var maxCandidates = Math.Max(1, options.MaxMatchCandidates); var identities = await context.SpeakerIdentities .Include(identity => identity.Snippets) .Include(identity => identity.Aliases) .OrderByDescending(identity => identity.TranscriptionCount) .ThenBy(identity => identity.Id) .ToListAsync(cancellationToken); identities = identities .Select(identity => new { Identity = identity, IsAttendee = MatchesAttendees(identity, attendees), IsActive = identity.UpdatedAt >= activeCutoff }) .Where(candidate => candidate.IsAttendee || candidate.IsActive) .OrderByDescending(candidate => candidate.IsAttendee) .ThenByDescending(candidate => candidate.Identity.TranscriptionCount) .ThenBy(candidate => candidate.Identity.Id) .Take(maxCandidates) .Select(candidate => candidate.Identity) .ToList(); foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize))) { var request = new SpeakerIdentityMatchRequest( speaker, snippet, batch.Select(identity => new SpeakerIdentityMatchCandidate( identity.Id, identity.CanonicalName, identity.TranscriptionCount, identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList())) .ToList()); var match = await matcher.MatchAsync(request, cancellationToken); if (match is not null) { return match; } } return null; } private static bool MatchesAttendees( SpeakerIdentity identity, IReadOnlyList attendees) { var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase); return GetAcceptedNames(identity).Any(attendeeSet.Contains); } private static Task LoadIdentityAsync( SpeakerIdentityDbContext context, int identityId, CancellationToken cancellationToken) { return context.SpeakerIdentities .Include(identity => identity.CandidateNames) .Include(identity => identity.Aliases) .Include(identity => identity.Snippets) .SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken); } private void UpdateMatchedIdentity( SpeakerIdentity identity, IReadOnlyList attendees, byte[] snippet) { identity.TranscriptionCount++; identity.UpdatedAt = DateTimeOffset.UtcNow; if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0) { var currentCandidates = identity.CandidateNames .Select(candidate => candidate.Name) .ToHashSet(StringComparer.OrdinalIgnoreCase); var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null; var aliasToCandidate = identity.Aliases .Where(alias => !string.IsNullOrWhiteSpace(alias.Name)) .SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate })) .Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) || pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) || pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase)) .ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase); var intersection = attendees .Select(attendee => currentCandidates.Contains(attendee) ? attendee : aliasToCandidate.GetValueOrDefault(attendee) ?? (identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase)) ? fallbackAliasCandidate : null)) .Where(candidate => !string.IsNullOrWhiteSpace(candidate)) .Select(candidate => candidate!) .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); if (intersection.Count == 0) { ResetCandidates(identity, attendees); ReplaceOldestSnippet(identity, snippet); return; } ResetCandidates(identity, intersection); if (intersection.Count == 1) { identity.CanonicalName = intersection[0]; } } AddSnippetIfNeeded(identity, snippet); } private async Task LearnUnmatchedSpeakersAsync( SpeakerIdentityDbContext context, IReadOnlyList attendees, IEnumerable matchedCanonicalNames, IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers, CancellationToken cancellationToken) { var remainingCandidates = attendees .Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); if (remainingCandidates.Count == 0) { return; } foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0)) { var now = DateTimeOffset.UtcNow; context.SpeakerIdentities.Add(new SpeakerIdentity { CreatedAt = now, UpdatedAt = now, CandidateNames = remainingCandidates .Select(candidate => new SpeakerCandidateName { Name = candidate }) .ToList(), Snippets = [ new SpeakerSnippet { WavBytes = snippet, CreatedAt = now } ] }); } await Task.CompletedTask; } private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet) { if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker) { return; } identity.Snippets.Add(new SpeakerSnippet { WavBytes = snippet, CreatedAt = DateTimeOffset.UtcNow }); } private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet) { var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault(); if (oldest is not null) { identity.Snippets.Remove(oldest); } if (snippet.Length > 0) { identity.Snippets.Add(new SpeakerSnippet { WavBytes = snippet, CreatedAt = DateTimeOffset.UtcNow }); } } private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList candidates) { identity.CandidateNames.Clear(); identity.CandidateNames.AddRange(candidates .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .Select(candidate => new SpeakerCandidateName { Name = candidate })); } private static IReadOnlySet GetAcceptedNames(SpeakerIdentity identity) { return new[] { identity.CanonicalName } .Concat(identity.Aliases.Select(alias => alias.Name)) .Concat(identity.CandidateNames.Select(candidate => candidate.Name)) .Where(name => !string.IsNullOrWhiteSpace(name)) .Select(name => name!.Trim()) .ToHashSet(StringComparer.OrdinalIgnoreCase); } private static IReadOnlyList NormalizeAttendees(IEnumerable attendees) { return attendees .Select(NormalizeAttendee) .Where(attendee => !string.IsNullOrWhiteSpace(attendee)) .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase) .ToList(); } private static string NormalizeAttendee(string attendee) { var trimmed = attendee.Trim(); var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal); return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed; } private enum SpeakerIdentityProcessingMode { LiveReadOnly, Final } }