forked from Manuel/meeting-assistant
feat: add local Resemblyzer speaker recognition
This commit is contained in:
@@ -0,0 +1,631 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
ResemblyzerVoiceClusterMatcher clusterMatcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentificationService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.encoder = encoder;
|
||||
this.clusterMatcher = clusterMatcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ApplySpeakerOverrideAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string sourceSpeaker,
|
||||
string targetSpeaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled ||
|
||||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
|
||||
string.IsNullOrWhiteSpace(targetSpeaker) ||
|
||||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = sourceSpeaker.Trim();
|
||||
var targetName = targetSpeaker.Trim();
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var sourceCandidate = identities
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
|
||||
if (target is null && sourceCandidate is null && vectors.Count == 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
|
||||
sourceLabel,
|
||||
targetName);
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (target is null)
|
||||
{
|
||||
target = sourceCandidate ?? new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now
|
||||
};
|
||||
if (target.Id == 0)
|
||||
{
|
||||
context.SpeakerIdentities.Add(target);
|
||||
}
|
||||
}
|
||||
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||
{
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
sourceCandidate,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||
}
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
|
||||
outlierPruner.AddAndPrune(
|
||||
target,
|
||||
vectors,
|
||||
now);
|
||||
target.UpdatedAt = now;
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
target.References,
|
||||
sourceLabel,
|
||||
targetName,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteSpeakerIdentityAsync(
|
||||
string identity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities.FirstOrDefault(candidate =>
|
||||
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
|
||||
if (target is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
context.SpeakerIdentities.Remove(target);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
bool final,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return EmptyResult(request.Segments);
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var knownMappings = request.KnownSpeakerMappings ??
|
||||
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
if (final && knownMappings.Count > 0)
|
||||
{
|
||||
await PersistMappedSpeakerEvidenceAsync(
|
||||
context,
|
||||
request,
|
||||
knownMappings,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
var identifiedNames = knownMappings.Values
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var segmentSpeaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
identifiedNames.Add(segmentSpeaker.Trim());
|
||||
}
|
||||
|
||||
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var pendingAudits = new List<PendingIdentificationAudit>();
|
||||
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAutomaticVectorsAsync(
|
||||
request,
|
||||
speaker,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
|
||||
speaker,
|
||||
vectors.Count,
|
||||
resemblyzerOptions.RequiredVectorsPerSpeaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (identity, decision) = await FindMatchAsync(
|
||||
context,
|
||||
attendees,
|
||||
identifiedNames,
|
||||
vectors,
|
||||
cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, vectors));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousCanonicalName = identity.CanonicalName;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees);
|
||||
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var displayName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(displayName))
|
||||
{
|
||||
mappings[speaker] = displayName;
|
||||
identifiedNames.Add(displayName);
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
identifiedNames.Add(acceptedName);
|
||||
}
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
displayName,
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
LearnUnmatchedSpeakers(
|
||||
context,
|
||||
request.MeetingNote,
|
||||
attendees,
|
||||
matchedAcceptedNames,
|
||||
unmatchedSpeakers,
|
||||
pendingAudits);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await AppendAuditsAsync(pendingAudits, cancellationToken);
|
||||
var relabeled = request.Segments
|
||||
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
|
||||
? segment with { Speaker = name }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> identifiedNames,
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var identities = await LoadIdentities(context)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var candidates = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
|
||||
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
|
||||
candidate.Identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
candidate.Identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
var match = clusterMatcher.Match(queryVectors, candidates);
|
||||
return (
|
||||
match.IdentityId is { } identityId
|
||||
? identities.Single(identity => identity.Id == identityId)
|
||||
: null,
|
||||
match);
|
||||
}
|
||||
|
||||
private void LearnUnmatchedSpeakers(
|
||||
SpeakerIdentityDbContext context,
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> matchedAcceptedNames,
|
||||
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
|
||||
ICollection<PendingIdentificationAudit> pendingAudits)
|
||||
{
|
||||
var candidates = attendees
|
||||
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (speaker, vectors) in unmatchedSpeakers)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates
|
||||
.Select(name => new SpeakerCandidateName { Name = name })
|
||||
.ToList(),
|
||||
References = [CreateReference(meetingNote, now)]
|
||||
};
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
logger.LogInformation(
|
||||
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
|
||||
speaker,
|
||||
identity.VoiceVectors.Count,
|
||||
string.Join(", ", candidates));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AppendAuditsAsync(
|
||||
IEnumerable<PendingIdentificationAudit> pendingAudits,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
audit.Identity.References,
|
||||
audit.Speaker,
|
||||
audit.Name,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (samples.Count < requiredCount)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task PersistMappedSpeakerEvidenceAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
SpeakerIdentificationRequest request,
|
||||
IReadOnlyDictionary<string, string> knownMappings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
foreach (var (speaker, mappedName) in knownMappings)
|
||||
{
|
||||
var identity = identities
|
||||
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
|
||||
.OrderBy(candidate =>
|
||||
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(candidate => candidate.Id)
|
||||
.FirstOrDefault();
|
||||
if (identity is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
|
||||
speaker,
|
||||
mappedName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
|
||||
if (vectors.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
var vectorUpdate = outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
|
||||
vectorUpdate.AddedCount,
|
||||
speaker,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback: true,
|
||||
cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
int maxSamples,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var suppliedSamples = request.Samples?
|
||||
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(sample => sample.WavBytes.Length > 0)
|
||||
.OrderByDescending(sample => sample.Score)
|
||||
.Take(maxSamples)
|
||||
.ToList()
|
||||
?? [];
|
||||
var wavSamples = suppliedSamples
|
||||
.Select(sample => sample.WavBytes)
|
||||
.ToList();
|
||||
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
|
||||
{
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
request.Segments.Count);
|
||||
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
|
||||
{
|
||||
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
span,
|
||||
cancellationToken);
|
||||
if (wavBytes.Length > 0)
|
||||
{
|
||||
wavSamples.Add(wavBytes);
|
||||
}
|
||||
|
||||
if (wavSamples.Count >= maxSamples)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
|
||||
wavSamples.Count,
|
||||
maxSamples,
|
||||
speaker,
|
||||
suppliedSamples.Count,
|
||||
wavSamples.Count - suppliedSamples.Count);
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
private static bool OverlapsSuppliedSample(
|
||||
IReadOnlyList<TranscriptionSegment> span,
|
||||
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
|
||||
{
|
||||
if (span.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var start = span[0].Start;
|
||||
var end = span[^1].End;
|
||||
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
|
||||
}
|
||||
|
||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
|
||||
}
|
||||
|
||||
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
private sealed record PendingIdentificationAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string Speaker,
|
||||
string Name);
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user