Public Access
380 lines
15 KiB
C#
380 lines
15 KiB
C#
using MeetingAssistant.Transcription;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace MeetingAssistant.Speakers;
|
|
|
|
public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|
{
|
|
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
|
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
|
private readonly ISpeakerIdentityMatcher matcher;
|
|
private readonly SpeakerIdentificationOptions options;
|
|
private readonly ILogger<SpeakerIdentityService> logger;
|
|
|
|
public SpeakerIdentityService(
|
|
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
|
ISpeakerSnippetExtractor snippetExtractor,
|
|
ISpeakerIdentityMatcher matcher,
|
|
IOptions<MeetingAssistantOptions> options,
|
|
ILogger<SpeakerIdentityService> logger)
|
|
{
|
|
this.dbContextFactory = dbContextFactory;
|
|
this.snippetExtractor = snippetExtractor;
|
|
this.matcher = matcher;
|
|
this.options = options.Value.SpeakerIdentification;
|
|
this.logger = logger;
|
|
}
|
|
|
|
public async Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
|
SpeakerIdentificationRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken);
|
|
}
|
|
|
|
public async Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
|
SpeakerIdentificationRequest request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken);
|
|
}
|
|
|
|
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
|
SpeakerIdentificationRequest request,
|
|
SpeakerIdentityProcessingMode mode,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (!options.Enabled || request.Segments.Count == 0)
|
|
{
|
|
return new SpeakerIdentificationResult(request.Segments, new Dictionary<string, string>());
|
|
}
|
|
|
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
|
|
|
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
|
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
|
var matchedAcceptedNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
|
|
var samplesBySpeaker = request.Samples?
|
|
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
|
|
.GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(
|
|
group => group.Key,
|
|
group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(),
|
|
StringComparer.OrdinalIgnoreCase)
|
|
?? new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
foreach (var group in request.Segments
|
|
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker))
|
|
.GroupBy(segment => segment.Speaker)
|
|
.OrderBy(group => group.Min(segment => segment.Start)))
|
|
{
|
|
var speaker = group.Key;
|
|
if (speakerMappings.ContainsKey(speaker))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet)
|
|
? suppliedSnippet
|
|
: await snippetExtractor.ExtractSnippetAsync(
|
|
request.AudioPath,
|
|
group.ToList(),
|
|
cancellationToken);
|
|
if (snippet.Length == 0)
|
|
{
|
|
unmatchedSpeakers.Add((speaker, snippet));
|
|
continue;
|
|
}
|
|
|
|
var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken);
|
|
if (match is null)
|
|
{
|
|
unmatchedSpeakers.Add((speaker, snippet));
|
|
continue;
|
|
}
|
|
|
|
var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken);
|
|
if (identity is null)
|
|
{
|
|
unmatchedSpeakers.Add((speaker, snippet));
|
|
continue;
|
|
}
|
|
|
|
if (mode == SpeakerIdentityProcessingMode.Final)
|
|
{
|
|
UpdateMatchedIdentity(identity, attendees, snippet);
|
|
foreach (var acceptedName in GetAcceptedNames(identity))
|
|
{
|
|
matchedAcceptedNames.Add(acceptedName);
|
|
}
|
|
}
|
|
|
|
var speakerName = identity.GetDisplayName();
|
|
if (!string.IsNullOrWhiteSpace(speakerName))
|
|
{
|
|
speakerMappings[speaker] = speakerName;
|
|
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
|
speakerName,
|
|
GetAcceptedNames(identity).ToList()));
|
|
}
|
|
}
|
|
|
|
if (mode == SpeakerIdentityProcessingMode.Final)
|
|
{
|
|
await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken);
|
|
}
|
|
|
|
await context.SaveChangesAsync(cancellationToken);
|
|
|
|
var relabeledSegments = request.Segments
|
|
.Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName)
|
|
? segment with { Speaker = speakerName }
|
|
: segment)
|
|
.ToList();
|
|
return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches);
|
|
}
|
|
|
|
private async Task<SpeakerIdentityMatch?> FindMatchAsync(
|
|
SpeakerIdentityDbContext context,
|
|
IReadOnlyList<string> attendees,
|
|
string speaker,
|
|
byte[] snippet,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
|
var maxCandidates = Math.Max(1, options.MaxMatchCandidates);
|
|
var identities = await context.SpeakerIdentities
|
|
.Include(identity => identity.Snippets)
|
|
.Include(identity => identity.Aliases)
|
|
.OrderByDescending(identity => identity.TranscriptionCount)
|
|
.ThenBy(identity => identity.Id)
|
|
.ToListAsync(cancellationToken);
|
|
identities = identities
|
|
.Select(identity => new
|
|
{
|
|
Identity = identity,
|
|
IsAttendee = MatchesAttendees(identity, attendees),
|
|
IsActive = identity.UpdatedAt >= activeCutoff
|
|
})
|
|
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
|
.OrderByDescending(candidate => candidate.IsAttendee)
|
|
.ThenByDescending(candidate => candidate.Identity.TranscriptionCount)
|
|
.ThenBy(candidate => candidate.Identity.Id)
|
|
.Take(maxCandidates)
|
|
.Select(candidate => candidate.Identity)
|
|
.ToList();
|
|
|
|
foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize)))
|
|
{
|
|
var request = new SpeakerIdentityMatchRequest(
|
|
speaker,
|
|
snippet,
|
|
batch.Select(identity => new SpeakerIdentityMatchCandidate(
|
|
identity.Id,
|
|
identity.CanonicalName,
|
|
identity.TranscriptionCount,
|
|
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
|
|
.ToList());
|
|
var match = await matcher.MatchAsync(request, cancellationToken);
|
|
if (match is not null)
|
|
{
|
|
return match;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static bool MatchesAttendees(
|
|
SpeakerIdentity identity,
|
|
IReadOnlyList<string> attendees)
|
|
{
|
|
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
|
}
|
|
|
|
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
|
SpeakerIdentityDbContext context,
|
|
int identityId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return context.SpeakerIdentities
|
|
.Include(identity => identity.CandidateNames)
|
|
.Include(identity => identity.Aliases)
|
|
.Include(identity => identity.Snippets)
|
|
.SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken);
|
|
}
|
|
|
|
private void UpdateMatchedIdentity(
|
|
SpeakerIdentity identity,
|
|
IReadOnlyList<string> attendees,
|
|
byte[] snippet)
|
|
{
|
|
identity.TranscriptionCount++;
|
|
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
|
|
|
if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0)
|
|
{
|
|
var currentCandidates = identity.CandidateNames
|
|
.Select(candidate => candidate.Name)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
|
var aliasToCandidate = identity.Aliases
|
|
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
|
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
|
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
|
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
|
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
|
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
|
var intersection = attendees
|
|
.Select(attendee => currentCandidates.Contains(attendee)
|
|
? attendee
|
|
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
|
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
|
? fallbackAliasCandidate
|
|
: null))
|
|
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
|
.Select(candidate => candidate!)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
|
|
if (intersection.Count == 0)
|
|
{
|
|
ResetCandidates(identity, attendees);
|
|
ReplaceOldestSnippet(identity, snippet);
|
|
return;
|
|
}
|
|
|
|
ResetCandidates(identity, intersection);
|
|
if (intersection.Count == 1)
|
|
{
|
|
identity.CanonicalName = intersection[0];
|
|
}
|
|
}
|
|
|
|
AddSnippetIfNeeded(identity, snippet);
|
|
}
|
|
|
|
private async Task LearnUnmatchedSpeakersAsync(
|
|
SpeakerIdentityDbContext context,
|
|
IReadOnlyList<string> attendees,
|
|
IEnumerable<string> matchedCanonicalNames,
|
|
IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var remainingCandidates = attendees
|
|
.Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
if (remainingCandidates.Count == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
|
|
{
|
|
var now = DateTimeOffset.UtcNow;
|
|
context.SpeakerIdentities.Add(new SpeakerIdentity
|
|
{
|
|
CreatedAt = now,
|
|
UpdatedAt = now,
|
|
CandidateNames = remainingCandidates
|
|
.Select(candidate => new SpeakerCandidateName { Name = candidate })
|
|
.ToList(),
|
|
Snippets =
|
|
[
|
|
new SpeakerSnippet
|
|
{
|
|
WavBytes = snippet,
|
|
CreatedAt = now
|
|
}
|
|
]
|
|
});
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
|
|
{
|
|
if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker)
|
|
{
|
|
return;
|
|
}
|
|
|
|
identity.Snippets.Add(new SpeakerSnippet
|
|
{
|
|
WavBytes = snippet,
|
|
CreatedAt = DateTimeOffset.UtcNow
|
|
});
|
|
}
|
|
|
|
private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet)
|
|
{
|
|
var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault();
|
|
if (oldest is not null)
|
|
{
|
|
identity.Snippets.Remove(oldest);
|
|
}
|
|
|
|
if (snippet.Length > 0)
|
|
{
|
|
identity.Snippets.Add(new SpeakerSnippet
|
|
{
|
|
WavBytes = snippet,
|
|
CreatedAt = DateTimeOffset.UtcNow
|
|
});
|
|
}
|
|
}
|
|
|
|
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
|
{
|
|
identity.CandidateNames.Clear();
|
|
identity.CandidateNames.AddRange(candidates
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
|
}
|
|
|
|
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
|
{
|
|
return new[]
|
|
{
|
|
identity.CanonicalName
|
|
}
|
|
.Concat(identity.Aliases.Select(alias => alias.Name))
|
|
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
|
.Where(name => !string.IsNullOrWhiteSpace(name))
|
|
.Select(name => name!.Trim())
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
|
{
|
|
return attendees
|
|
.Select(NormalizeAttendee)
|
|
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
}
|
|
|
|
private static string NormalizeAttendee(string attendee)
|
|
{
|
|
var trimmed = attendee.Trim();
|
|
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
|
|
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
|
|
}
|
|
|
|
private enum SpeakerIdentityProcessingMode
|
|
{
|
|
LiveReadOnly,
|
|
Final
|
|
}
|
|
}
|