Public Access
Add inactivity safeguard and speaker diagnostics
PR and Push Build/Test / build-and-test (push) Failing after 8m31s
PR and Push Build/Test / build-and-test (push) Failing after 8m31s
This commit is contained in:
@@ -63,6 +63,13 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} composite layout: unknown {UnknownStart}-{UnknownEnd}, {KnownSegmentCount} known segment(s) across identity ids {IdentityIds}",
|
||||
request.DiarizedSpeaker,
|
||||
layout.UnknownSegment.Value.Start,
|
||||
layout.UnknownSegment.Value.End,
|
||||
layout.KnownSegments.Count,
|
||||
string.Join(", ", layout.KnownSegments.Select(segment => segment.IdentityId).Distinct().Order()));
|
||||
var segments = await DiarizeWithTimeoutAsync(tempPath, cancellationToken);
|
||||
if (segments.Count == 0)
|
||||
{
|
||||
@@ -76,6 +83,16 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
"Speaker identity matching {DiarizedSpeaker} received {SegmentCount} diarized segment(s)",
|
||||
request.DiarizedSpeaker,
|
||||
segments.Count);
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} diarized turn {Start}-{End} as {Speaker}",
|
||||
request.DiarizedSpeaker,
|
||||
segment.Start,
|
||||
segment.End,
|
||||
segment.Speaker);
|
||||
}
|
||||
|
||||
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
|
||||
if (string.IsNullOrWhiteSpace(unknownSpeaker))
|
||||
{
|
||||
@@ -93,6 +110,14 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
unknownSpeaker,
|
||||
StringComparison.Ordinal));
|
||||
var requiredMatches = known.Count() > 1 ? 2 : 1;
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} compared unknown speaker {UnknownSpeaker} with identity {IdentityId}: {MatchingSnippetCount}/{KnownSnippetCount} matching known snippet(s), required {RequiredMatches}",
|
||||
request.DiarizedSpeaker,
|
||||
unknownSpeaker,
|
||||
known.Key,
|
||||
matchingSnippetCount,
|
||||
known.Count(),
|
||||
requiredMatches);
|
||||
if (matchingSnippetCount >= requiredMatches)
|
||||
{
|
||||
var validationRequest = new SpeakerIdentityMatchValidationRequest(
|
||||
|
||||
@@ -30,14 +30,29 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
}
|
||||
|
||||
var segments = await DiarizeBytesAsync(wavBytes, cancellationToken);
|
||||
var result = HasSingleSpeaker(segments);
|
||||
if (!result)
|
||||
var result = AnalyzeSingleSpeakerCoverage(segments);
|
||||
if (!result.Accepted)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity sample because it did not contain one dominant speaker");
|
||||
"pyannote rejected speaker identity sample because it did not contain one dominant speaker: segments {SegmentCount}, speakers {SpeakerCount}, dominant speaker {DominantSpeaker}, coverage {Coverage:P2}, required coverage {RequiredCoverage:P2}, duration {Duration}",
|
||||
segments.Count,
|
||||
result.SpeakerCount,
|
||||
result.DominantSpeaker,
|
||||
result.Coverage,
|
||||
Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1),
|
||||
result.TotalDuration);
|
||||
return false;
|
||||
}
|
||||
|
||||
return result;
|
||||
logger.LogInformation(
|
||||
"pyannote accepted speaker identity sample: segments {SegmentCount}, speakers {SpeakerCount}, dominant speaker {DominantSpeaker}, coverage {Coverage:P2}, duration {Duration}, sample bytes {SampleBytes}",
|
||||
segments.Count,
|
||||
result.SpeakerCount,
|
||||
result.DominantSpeaker,
|
||||
result.Coverage,
|
||||
result.TotalDuration,
|
||||
wavBytes.Length);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<bool> ValidateMatchAsync(
|
||||
@@ -63,10 +78,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
TimeSpan.FromSeconds(1));
|
||||
if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId} because no readable composite snippets were available",
|
||||
request.DiarizedSpeaker,
|
||||
request.IdentityId);
|
||||
return false;
|
||||
}
|
||||
|
||||
var segments = await DiarizePathAsync(tempPath, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"pyannote diarized speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {SegmentCount} segment(s), {KnownSnippetCount} known snippet(s)",
|
||||
request.DiarizedSpeaker,
|
||||
request.IdentityId,
|
||||
segments.Count,
|
||||
layout.KnownSegments.Count);
|
||||
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
|
||||
if (string.IsNullOrWhiteSpace(unknownSpeaker))
|
||||
{
|
||||
@@ -98,11 +123,22 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
if (!accepted)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched",
|
||||
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched, required ratio {RequiredRatio:P2}",
|
||||
request.DiarizedSpeaker,
|
||||
request.IdentityId,
|
||||
matching,
|
||||
compared);
|
||||
compared,
|
||||
minimumRatio);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote accepted speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched, required ratio {RequiredRatio:P2}",
|
||||
request.DiarizedSpeaker,
|
||||
request.IdentityId,
|
||||
matching,
|
||||
compared,
|
||||
minimumRatio);
|
||||
}
|
||||
|
||||
return accepted;
|
||||
@@ -162,7 +198,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private bool HasSingleSpeaker(IReadOnlyList<TranscriptionSegment> segments)
|
||||
private SingleSpeakerCoverage AnalyzeSingleSpeakerCoverage(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
var durationsBySpeaker = segments
|
||||
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker) && segment.End > segment.Start)
|
||||
@@ -176,17 +212,29 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
.ToList();
|
||||
if (durationsBySpeaker.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (durationsBySpeaker.Count == 1)
|
||||
{
|
||||
return true;
|
||||
return new SingleSpeakerCoverage(false, 0, null, 0, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
var total = durationsBySpeaker.Sum(group => group.Duration);
|
||||
var dominant = durationsBySpeaker.Max(group => group.Duration);
|
||||
return total > 0 && dominant / total >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
|
||||
var dominant = durationsBySpeaker.OrderByDescending(group => group.Duration).First();
|
||||
var coverage = total > 0 ? dominant.Duration / total : 0;
|
||||
if (durationsBySpeaker.Count == 1)
|
||||
{
|
||||
return new SingleSpeakerCoverage(
|
||||
true,
|
||||
durationsBySpeaker.Count,
|
||||
dominant.Speaker,
|
||||
coverage,
|
||||
TimeSpan.FromSeconds(total));
|
||||
}
|
||||
|
||||
var accepted = total > 0 && coverage >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
|
||||
return new SingleSpeakerCoverage(
|
||||
accepted,
|
||||
durationsBySpeaker.Count,
|
||||
dominant.Speaker,
|
||||
coverage,
|
||||
TimeSpan.FromSeconds(total));
|
||||
}
|
||||
|
||||
private CompositeLayout WriteCompositeWav(
|
||||
@@ -234,4 +282,11 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
}
|
||||
|
||||
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<TimeSegment> KnownSegments);
|
||||
|
||||
private sealed record SingleSpeakerCoverage(
|
||||
bool Accepted,
|
||||
int SpeakerCount,
|
||||
string? DominantSpeaker,
|
||||
double Coverage,
|
||||
TimeSpan TotalDuration);
|
||||
}
|
||||
|
||||
@@ -58,53 +58,97 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
.ToHashSet();
|
||||
var mergedPairs = 0;
|
||||
var attempts = 0;
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics started: cutoff {Cutoff}, recent identity ids {RecentIdentityIds}, candidate identity count {CandidateIdentityCount}",
|
||||
cutoff,
|
||||
FormatIds(recentIds),
|
||||
identities.Count);
|
||||
|
||||
foreach (var sourceId in recentIds.ToList())
|
||||
{
|
||||
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||
if (source is null || source.Snippets.Count == 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics skipped source identity {SourceIdentityId} because it was missing or had no snippets",
|
||||
sourceId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var targets = identities
|
||||
.Where(identity => identity.Id != source.Id && identity.Snippets.Count > 0)
|
||||
.ToList();
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics evaluating source identity {SourceIdentityId} ({SourceName}) with {SnippetCount} snippet(s) against target ids {TargetIdentityIds}",
|
||||
source.Id,
|
||||
source.GetDisplayName(),
|
||||
source.Snippets.Count,
|
||||
FormatIds(targets.Select(target => target.Id)));
|
||||
foreach (var batch in targets.Chunk(Math.Max(1, options.MatchBatchSize)))
|
||||
{
|
||||
var firstSnippet = SelectSnippet(source, excludedSnippet: null);
|
||||
if (firstSnippet is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics stopped evaluating source identity {SourceIdentityId} because no first snippet was available",
|
||||
source.Id);
|
||||
break;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics round 1 attempt {Attempt} for source identity {SourceIdentityId}: target ids {TargetIdentityIds}, source snippet {SnippetId}",
|
||||
attempts,
|
||||
source.Id,
|
||||
FormatIds(batch.Select(target => target.Id)),
|
||||
firstSnippet.Id);
|
||||
var firstMatch = await matcher.MatchAsync(
|
||||
CreateRequest(source, firstSnippet, batch),
|
||||
cancellationToken);
|
||||
if (firstMatch is null || firstMatch.IdentityId == source.Id)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics round 1 found no usable target for source identity {SourceIdentityId}",
|
||||
source.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = batch.SingleOrDefault(identity => identity.Id == firstMatch.IdentityId);
|
||||
if (target is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics round 1 matched identity {IdentityId}, but it was not in the current target batch",
|
||||
firstMatch.IdentityId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var secondSnippet = SelectSnippet(source, excludedSnippet: firstSnippet);
|
||||
if (secondSnippet is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics could not validate source identity {SourceIdentityId} against target identity {TargetIdentityId} because no second source snippet was available",
|
||||
source.Id,
|
||||
target.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics round 2 attempt {Attempt} for source identity {SourceIdentityId}: target identity {TargetIdentityId}, source snippet {SnippetId}",
|
||||
attempts,
|
||||
source.Id,
|
||||
target.Id,
|
||||
secondSnippet.Id);
|
||||
var secondMatch = await matcher.MatchAsync(
|
||||
CreateRequest(source, secondSnippet, batch),
|
||||
cancellationToken);
|
||||
if (secondMatch?.IdentityId != target.Id)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics rejected merge source identity {SourceIdentityId} into target identity {TargetIdentityId}: second match was {SecondMatchIdentityId}",
|
||||
source.Id,
|
||||
target.Id,
|
||||
secondMatch?.IdentityId);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -114,6 +158,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker);
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
||||
source.Id,
|
||||
sourceName,
|
||||
target.Id,
|
||||
targetName);
|
||||
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
|
||||
target.References,
|
||||
targetName,
|
||||
@@ -164,4 +214,13 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
.FirstOrDefault(snippet => excludedSnippet is null || snippet.Id != excludedSnippet.Id);
|
||||
}
|
||||
|
||||
private static string FormatIds(IEnumerable<int> ids)
|
||||
{
|
||||
var formatted = ids
|
||||
.Distinct()
|
||||
.Order()
|
||||
.Select(id => id.ToString(System.Globalization.CultureInfo.InvariantCulture))
|
||||
.ToList();
|
||||
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,18 +67,32 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
var sourceLabel = sourceSpeaker.Trim();
|
||||
var targetName = targetSpeaker.Trim();
|
||||
logger.LogInformation(
|
||||
"Applying speaker override from {SourceSpeaker} to {TargetSpeaker} for meeting {MeetingNotePath}",
|
||||
sourceLabel,
|
||||
targetName,
|
||||
request.MeetingNote.Path);
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var meetingReference = CreateReference(request.MeetingNote, now);
|
||||
var snippet = await ResolveOverrideSnippetAsync(request, sourceLabel, cancellationToken);
|
||||
var snippetResolution = await ResolveOverrideSnippetAsync(request, sourceLabel, cancellationToken);
|
||||
var snippet = snippetResolution.WavBytes;
|
||||
var target = await FindIdentityByAcceptedNameAsync(context, targetName, cancellationToken);
|
||||
var sourceCandidate = await FindCurrentRunCandidateAsync(
|
||||
context,
|
||||
meetingReference,
|
||||
targetName,
|
||||
cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Speaker override from {SourceSpeaker} to {TargetSpeaker} resolved sample source {SampleSource}, sample bytes {SampleBytes}, target identity {TargetIdentityId}, source candidate identity {SourceCandidateIdentityId}",
|
||||
sourceLabel,
|
||||
targetName,
|
||||
snippetResolution.Source,
|
||||
snippet.Length,
|
||||
target?.Id,
|
||||
sourceCandidate?.Id);
|
||||
|
||||
if (target is null)
|
||||
{
|
||||
@@ -99,10 +113,19 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
if (target.Id == 0)
|
||||
{
|
||||
context.SpeakerIdentities.Add(target);
|
||||
logger.LogInformation(
|
||||
"Creating speaker identity from override for {TargetSpeaker} using source {SourceSpeaker}",
|
||||
targetName,
|
||||
sourceLabel);
|
||||
}
|
||||
}
|
||||
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Merging override source candidate identity {SourceCandidateIdentityId} into target identity {TargetIdentityId} for {TargetSpeaker}",
|
||||
sourceCandidate.Id,
|
||||
target.Id,
|
||||
targetName);
|
||||
MergeOverrideCandidate(target, sourceCandidate);
|
||||
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||
}
|
||||
@@ -111,8 +134,16 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
target.UpdatedAt = now;
|
||||
ResetCandidates(target, [targetName]);
|
||||
AddMeetingReference(target, meetingReference);
|
||||
AddSnippetIfNeeded(target, snippet);
|
||||
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Applied speaker override from {SourceSpeaker} to {TargetSpeaker}: identity {IdentityId}, snippet added {SnippetAdded}, snippet count {SnippetCount}, reference count {ReferenceCount}",
|
||||
sourceLabel,
|
||||
targetName,
|
||||
target.Id,
|
||||
snippetAdded,
|
||||
target.Snippets.Count,
|
||||
target.References.Count);
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
target.References,
|
||||
sourceLabel,
|
||||
@@ -134,9 +165,20 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var target = await FindIdentityByAcceptedNameAsync(context, identity.Trim(), cancellationToken);
|
||||
if (target is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity deletion skipped because {IdentityName} was not found",
|
||||
identity.Trim());
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Deleting speaker identity {IdentityId} ({IdentityName}) with {SnippetCount} snippet(s), {CandidateCount} candidate(s), {AliasCount} alias(es), {ReferenceCount} reference(s)",
|
||||
target.Id,
|
||||
target.GetDisplayName() ?? identity.Trim(),
|
||||
target.Snippets.Count,
|
||||
target.CandidateNames.Count,
|
||||
target.Aliases.Count,
|
||||
target.References.Count);
|
||||
context.SpeakerIdentities.Remove(target);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
@@ -180,6 +222,15 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
matchedAcceptedNames.Add(name);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing started in {Mode} mode for meeting {MeetingNotePath}: {SegmentCount} segment(s), {SampleCount} supplied sample(s), {KnownMappingCount} known mapping(s), attendees {Attendees}",
|
||||
mode,
|
||||
request.MeetingNote.Path,
|
||||
request.Segments.Count,
|
||||
request.Samples?.Count ?? 0,
|
||||
knownSpeakerMappings.Count,
|
||||
FormatNames(attendees));
|
||||
|
||||
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
|
||||
var samplesBySpeaker = request.Samples?
|
||||
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
|
||||
@@ -189,6 +240,12 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
|
||||
if (samplesBySpeaker.Count > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing has supplied samples for {SampleSpeakers}",
|
||||
string.Join(", ", samplesBySpeaker.Select(pair => $"{pair.Key}:{pair.Value.Length} bytes")));
|
||||
}
|
||||
|
||||
foreach (var group in request.Segments
|
||||
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker))
|
||||
@@ -198,26 +255,55 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var speaker = group.Key;
|
||||
if (knownDiarizedSpeakers.Contains(speaker) || alreadyIdentifiedNames.Contains(speaker))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing skipped {Speaker} because it is already known or identified",
|
||||
speaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (speakerMappings.ContainsKey(speaker))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing skipped {Speaker} because a mapping already exists to {MappedSpeaker}",
|
||||
speaker,
|
||||
speakerMappings[speaker]);
|
||||
continue;
|
||||
}
|
||||
|
||||
var snippetSource = "supplied";
|
||||
var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet)
|
||||
? suppliedSnippet
|
||||
: await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration),
|
||||
cancellationToken);
|
||||
: [];
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
snippetSource = "extracted";
|
||||
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration);
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing extracting sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
||||
speaker,
|
||||
span.Count,
|
||||
SpeakerSampleSpanSelector.SpanDuration(span),
|
||||
options.MinimumSampleSpeechDuration);
|
||||
snippet = await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
span,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing resolved {SampleSource} sample for {Speaker}: {SampleBytes} byte(s)",
|
||||
snippetSource,
|
||||
speaker,
|
||||
snippet.Length);
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing cannot match or learn {Speaker} because no usable sample was available",
|
||||
speaker);
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
@@ -231,6 +317,9 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
cancellationToken);
|
||||
if (match is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing found no existing identity match for {Speaker}",
|
||||
speaker);
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
@@ -238,6 +327,10 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Speaker identity processing matched {Speaker} to identity {IdentityId}, but the identity could not be loaded",
|
||||
speaker,
|
||||
match.IdentityId);
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
@@ -247,6 +340,15 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var previousCanonicalName = identity.CanonicalName;
|
||||
AddMeetingReference(identity, meetingReference);
|
||||
UpdateMatchedIdentity(identity, attendees, snippet);
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing updated matched identity {IdentityId} for {Speaker}: previous canonical {PreviousCanonicalName}, canonical {CanonicalName}, candidates {CandidateNames}, snippets {SnippetCount}, references {ReferenceCount}",
|
||||
identity.Id,
|
||||
speaker,
|
||||
previousCanonicalName,
|
||||
identity.CanonicalName,
|
||||
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)),
|
||||
identity.Snippets.Count,
|
||||
identity.References.Count);
|
||||
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
@@ -279,18 +381,34 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyList<SpeakerIdentity> learnedIdentities = [];
|
||||
if (mode == SpeakerIdentityProcessingMode.Final)
|
||||
{
|
||||
await LearnUnmatchedSpeakersAsync(
|
||||
learnedIdentities = await LearnUnmatchedSpeakersAsync(
|
||||
context,
|
||||
attendees,
|
||||
matchedAcceptedNames,
|
||||
unmatchedSpeakers,
|
||||
meetingReference,
|
||||
cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing queued {LearnedIdentityCount} new unmatched speaker identity candidate(s)",
|
||||
learnedIdentities.Count);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
if (learnedIdentities.Count > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing saved learned identity candidate ids {IdentityIds}",
|
||||
FormatIds(learnedIdentities.Select(identity => identity.Id)));
|
||||
}
|
||||
logger.LogInformation(
|
||||
"Speaker identity processing completed in {Mode} mode for meeting {MeetingNotePath}: {MappingCount} mapping(s), {AttendeeMatchCount} attendee match(es)",
|
||||
mode,
|
||||
request.MeetingNote.Path,
|
||||
speakerMappings.Count,
|
||||
attendeeMatches.Count);
|
||||
|
||||
var relabeledSegments = request.Segments
|
||||
.Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName)
|
||||
@@ -333,8 +451,16 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Select(candidate => candidate.Identity)
|
||||
.ToList();
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity match candidate selection for {Speaker}: {CandidateCount} candidate(s) after active/attendee/name filters, ids {IdentityIds}",
|
||||
speaker,
|
||||
identities.Count,
|
||||
FormatIds(identities.Select(identity => identity.Id)));
|
||||
|
||||
var round = 0;
|
||||
foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize)))
|
||||
{
|
||||
round++;
|
||||
var request = new SpeakerIdentityMatchRequest(
|
||||
speaker,
|
||||
snippet,
|
||||
@@ -344,17 +470,31 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
identity.ReferenceCount,
|
||||
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
|
||||
.ToList());
|
||||
logger.LogInformation(
|
||||
"Speaker identity match round {Round} for {Speaker}: candidate identity ids {IdentityIds}",
|
||||
round,
|
||||
speaker,
|
||||
FormatIds(request.Candidates.Select(candidate => candidate.IdentityId)));
|
||||
var match = await matcher.MatchAsync(request, cancellationToken);
|
||||
if (match is not null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity match round {Round} for {Speaker} matched identity {IdentityId}",
|
||||
round,
|
||||
speaker,
|
||||
match.IdentityId);
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity match candidate selection for {Speaker} completed with no match after {RoundCount} round(s)",
|
||||
speaker,
|
||||
round);
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<byte[]> ResolveOverrideSnippetAsync(
|
||||
private async Task<SpeakerSnippetResolution> ResolveOverrideSnippetAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string sourceSpeaker,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -365,15 +505,29 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.FirstOrDefault();
|
||||
if (sample is not null)
|
||||
{
|
||||
return sample.WavBytes;
|
||||
return new SpeakerSnippetResolution(
|
||||
sample.WavBytes,
|
||||
"supplied-sample",
|
||||
SegmentCount: 1,
|
||||
sample.Score,
|
||||
sample.Segment.End - sample.Segment.Start);
|
||||
}
|
||||
|
||||
var segments = request.Segments
|
||||
.Where(segment => string.Equals(segment.Speaker, sourceSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
return segments.Count == 0
|
||||
? []
|
||||
: await snippetExtractor.ExtractSnippetAsync(request.AudioPath, segments, cancellationToken);
|
||||
if (segments.Count == 0)
|
||||
{
|
||||
return new SpeakerSnippetResolution([], "missing-segments", SegmentCount: 0, Score: null, Duration: TimeSpan.Zero);
|
||||
}
|
||||
|
||||
var snippet = await snippetExtractor.ExtractSnippetAsync(request.AudioPath, segments, cancellationToken);
|
||||
return new SpeakerSnippetResolution(
|
||||
snippet,
|
||||
"extracted-from-recording",
|
||||
segments.Count,
|
||||
Score: null,
|
||||
segments.Max(segment => segment.End) - segments.Min(segment => segment.Start));
|
||||
}
|
||||
|
||||
private static async Task<SpeakerIdentity?> FindIdentityByAcceptedNameAsync(
|
||||
@@ -421,6 +575,10 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker override candidate merge started: source identity {SourceIdentityId} into target identity {TargetIdentityId}",
|
||||
source.Id,
|
||||
target.Id);
|
||||
AddAlias(target, source.CanonicalName);
|
||||
foreach (var alias in source.Aliases)
|
||||
{
|
||||
@@ -441,6 +599,12 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
AddSnippetIfNeeded(target, snippet.WavBytes);
|
||||
}
|
||||
logger.LogInformation(
|
||||
"Speaker override candidate merge completed: target identity {TargetIdentityId} now has {AliasCount} alias(es), {SnippetCount} snippet(s), {ReferenceCount} reference(s)",
|
||||
target.Id,
|
||||
target.Aliases.Count,
|
||||
target.Snippets.Count,
|
||||
target.References.Count);
|
||||
}
|
||||
|
||||
private static void AddAlias(SpeakerIdentity identity, string? alias)
|
||||
@@ -526,22 +690,36 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
if (intersection.Count == 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
||||
identity.Id,
|
||||
FormatNames(attendees));
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
|
||||
identity.Id,
|
||||
FormatNames(currentCandidates),
|
||||
FormatNames(attendees),
|
||||
FormatNames(intersection));
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
||||
identity.Id,
|
||||
identity.CanonicalName);
|
||||
}
|
||||
}
|
||||
|
||||
AddSnippetIfNeeded(identity, snippet);
|
||||
}
|
||||
|
||||
private async Task LearnUnmatchedSpeakersAsync(
|
||||
private async Task<IReadOnlyList<SpeakerIdentity>> LearnUnmatchedSpeakersAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IEnumerable<string> matchedCanonicalNames,
|
||||
@@ -555,16 +733,26 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.ToList();
|
||||
if (remainingCandidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate learning skipped because no attendee candidates remain. Matched names {MatchedNames}",
|
||||
FormatNames(matchedCanonicalNames));
|
||||
return [];
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate learning started for {UnmatchedSpeakerCount} unmatched speaker(s) with remaining candidate names {CandidateNames}",
|
||||
unmatchedSpeakers.Count,
|
||||
FormatNames(remainingCandidates));
|
||||
var learned = new List<SpeakerIdentity>();
|
||||
foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
|
||||
{
|
||||
if (!await matchValidator.ValidateSampleAsync(snippet, cancellationToken))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample",
|
||||
speaker);
|
||||
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample: candidate names {CandidateNames}, sample bytes {SampleBytes}",
|
||||
speaker,
|
||||
FormatNames(remainingCandidates),
|
||||
snippet.Length);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -595,6 +783,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
]
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
learned.Add(identity);
|
||||
logger.LogInformation(
|
||||
"Created speaker identity candidate for {Speaker}: canonical {CanonicalName}, candidate names {CandidateNames}, sample bytes {SampleBytes}, meeting reference {MeetingNotePath}",
|
||||
speaker,
|
||||
canonicalName,
|
||||
FormatNames(remainingCandidates),
|
||||
snippet.Length,
|
||||
meetingReference.MeetingNotePath);
|
||||
if (!string.IsNullOrWhiteSpace(canonicalName))
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
@@ -605,14 +801,22 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
}
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
foreach (var (speaker, _) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length == 0))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping speaker identity candidate for {Speaker} because no sample was available: candidate names {CandidateNames}",
|
||||
speaker,
|
||||
FormatNames(remainingCandidates));
|
||||
}
|
||||
|
||||
return learned;
|
||||
}
|
||||
|
||||
private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
|
||||
private bool AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
|
||||
{
|
||||
if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker)
|
||||
{
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
identity.Snippets.Add(new SpeakerSnippet
|
||||
@@ -620,6 +824,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
WavBytes = snippet,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet)
|
||||
@@ -700,6 +905,34 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
SpeakerIdentityReferences.AddIfMissing(identity, reference, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static string FormatNames(IEnumerable<string?> names)
|
||||
{
|
||||
var formatted = names
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
|
||||
}
|
||||
|
||||
private static string FormatIds(IEnumerable<int> ids)
|
||||
{
|
||||
var formatted = ids
|
||||
.Distinct()
|
||||
.Order()
|
||||
.Select(id => id.ToString(System.Globalization.CultureInfo.InvariantCulture))
|
||||
.ToList();
|
||||
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
|
||||
}
|
||||
|
||||
private sealed record SpeakerSnippetResolution(
|
||||
byte[] WavBytes,
|
||||
string Source,
|
||||
int SegmentCount,
|
||||
double? Score,
|
||||
TimeSpan Duration);
|
||||
|
||||
private enum SpeakerIdentityProcessingMode
|
||||
{
|
||||
LiveReadOnly,
|
||||
|
||||
Reference in New Issue
Block a user