Public Access
Add meeting assistant speech and summary automation
This commit is contained in:
@@ -0,0 +1,379 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityService> logger;
|
||||
|
||||
public SpeakerIdentityService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.matcher = matcher;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
SpeakerIdentityProcessingMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return new SpeakerIdentificationResult(request.Segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var matchedAcceptedNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
|
||||
var samplesBySpeaker = request.Samples?
|
||||
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
|
||||
.GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(
|
||||
group => group.Key,
|
||||
group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var group in request.Segments
|
||||
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker))
|
||||
.GroupBy(segment => segment.Speaker)
|
||||
.OrderBy(group => group.Min(segment => segment.Start)))
|
||||
{
|
||||
var speaker = group.Key;
|
||||
if (speakerMappings.ContainsKey(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet)
|
||||
? suppliedSnippet
|
||||
: await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
group.ToList(),
|
||||
cancellationToken);
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken);
|
||||
if (match is null)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mode == SpeakerIdentityProcessingMode.Final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees, snippet);
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var speakerName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(speakerName))
|
||||
{
|
||||
speakerMappings[speaker] = speakerName;
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
speakerName,
|
||||
GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (mode == SpeakerIdentityProcessingMode.Final)
|
||||
{
|
||||
await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
|
||||
var relabeledSegments = request.Segments
|
||||
.Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName)
|
||||
? segment with { Speaker = speakerName }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentityMatch?> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
string speaker,
|
||||
byte[] snippet,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var maxCandidates = Math.Max(1, options.MaxMatchCandidates);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.Aliases)
|
||||
.OrderByDescending(identity => identity.TranscriptionCount)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
identities = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.TranscriptionCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(maxCandidates)
|
||||
.Select(candidate => candidate.Identity)
|
||||
.ToList();
|
||||
|
||||
foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize)))
|
||||
{
|
||||
var request = new SpeakerIdentityMatchRequest(
|
||||
speaker,
|
||||
snippet,
|
||||
batch.Select(identity => new SpeakerIdentityMatchCandidate(
|
||||
identity.Id,
|
||||
identity.CanonicalName,
|
||||
identity.TranscriptionCount,
|
||||
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
|
||||
.ToList());
|
||||
var match = await matcher.MatchAsync(request, cancellationToken);
|
||||
if (match is not null)
|
||||
{
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
}
|
||||
|
||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
int identityId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.Snippets)
|
||||
.SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees,
|
||||
byte[] snippet)
|
||||
{
|
||||
identity.TranscriptionCount++;
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0)
|
||||
{
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (intersection.Count == 0)
|
||||
{
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
}
|
||||
}
|
||||
|
||||
AddSnippetIfNeeded(identity, snippet);
|
||||
}
|
||||
|
||||
private async Task LearnUnmatchedSpeakersAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IEnumerable<string> matchedCanonicalNames,
|
||||
IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var remainingCandidates = attendees
|
||||
.Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (remainingCandidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = remainingCandidates
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate })
|
||||
.ToList(),
|
||||
Snippets =
|
||||
[
|
||||
new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = now
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
|
||||
{
|
||||
if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
identity.Snippets.Add(new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet)
|
||||
{
|
||||
var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault();
|
||||
if (oldest is not null)
|
||||
{
|
||||
identity.Snippets.Remove(oldest);
|
||||
}
|
||||
|
||||
if (snippet.Length > 0)
|
||||
{
|
||||
identity.Snippets.Add(new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
|
||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
identity.CanonicalName
|
||||
}
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var trimmed = attendee.Trim();
|
||||
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
|
||||
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
|
||||
}
|
||||
|
||||
private enum SpeakerIdentityProcessingMode
|
||||
{
|
||||
LiveReadOnly,
|
||||
Final
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user