Harden speaker identity samples
PR and Push Build/Test / build-and-test (push) Successful in 7m0s

This commit is contained in:
2026-05-28 12:02:44 +02:00
parent c84f8a984a
commit a72cda0c03
23 changed files with 1251 additions and 123 deletions
@@ -15,15 +15,18 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
private readonly ISpeakerIdentityDiarizationClient diarizationClient;
private readonly SpeakerIdentificationOptions options;
private readonly ILogger<AzureSpeechSpeakerIdentityMatcher> logger;
private readonly ISpeakerIdentityMatchValidator matchValidator;
public AzureSpeechSpeakerIdentityMatcher(
ISpeakerIdentityDiarizationClient diarizationClient,
Microsoft.Extensions.Options.IOptions<MeetingAssistantOptions> options,
ILogger<AzureSpeechSpeakerIdentityMatcher> logger)
ILogger<AzureSpeechSpeakerIdentityMatcher> logger,
ISpeakerIdentityMatchValidator matchValidator)
{
this.diarizationClient = diarizationClient;
this.options = options.Value.SpeakerIdentification;
this.logger = logger;
this.matchValidator = matchValidator;
}
public async Task<SpeakerIdentityMatch?> MatchAsync(
@@ -35,6 +38,14 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
return null;
}
if (!await matchValidator.ValidateSampleAsync(request.UnknownSnippet, cancellationToken))
{
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} skipped because secondary validation rejected the unknown sample",
request.DiarizedSpeaker);
return null;
}
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} against {CandidateCount} candidate(s)",
request.DiarizedSpeaker,
@@ -84,6 +95,20 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
var requiredMatches = known.Count() > 1 ? 2 : 1;
if (matchingSnippetCount >= requiredMatches)
{
var validationRequest = new SpeakerIdentityMatchValidationRequest(
request.DiarizedSpeaker,
known.Key,
request.UnknownSnippet,
known.Select(segment => segment.Snippet).ToList());
if (!await matchValidator.ValidateMatchAsync(validationRequest, cancellationToken))
{
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} rejected identity {IdentityId} after secondary validation",
request.DiarizedSpeaker,
known.Key);
continue;
}
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} matched identity {IdentityId}",
request.DiarizedSpeaker,
@@ -99,7 +124,7 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
}
finally
{
TryDelete(tempPath);
SpeakerCompositeWav.TryDelete(tempPath);
}
}
@@ -130,7 +155,8 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request)
{
using var firstReader = OpenFirstReadableWave(request);
using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(
request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet));
if (firstReader is null)
{
return new CompositeLayout(null, []);
@@ -145,118 +171,33 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
{
foreach (var snippet in candidate.Snippets.Where(storedSnippet => storedSnippet.Length > 0))
{
var segment = AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
if (segment is null)
{
continue;
}
knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value));
knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value, snippet));
current = segment.Value.End + silence;
WriteSilence(writer, firstReader.WaveFormat, silence);
SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
}
}
var unknownSegment = AppendSnippet(writer, firstReader.WaveFormat, request.UnknownSnippet, current);
var unknownSegment = SpeakerCompositeWav.AppendSnippet(
writer,
firstReader.WaveFormat,
request.UnknownSnippet,
current);
return new CompositeLayout(unknownSegment, knownSegments);
}
private static WaveFileReader? OpenFirstReadableWave(SpeakerIdentityMatchRequest request)
{
foreach (var bytes in request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet))
{
if (bytes.Length == 0)
{
continue;
}
try
{
return new WaveFileReader(new MemoryStream(bytes));
}
catch (FormatException)
{
}
}
return null;
}
private static TimeSegment? AppendSnippet(
WaveFileWriter writer,
WaveFormat targetFormat,
byte[] snippet,
TimeSpan start)
{
if (snippet.Length == 0)
{
return null;
}
using var reader = new WaveFileReader(new MemoryStream(snippet));
if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
{
throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
}
reader.CopyTo(writer);
return new TimeSegment(start, start + reader.TotalTime);
}
private static void WriteSilence(WaveFileWriter writer, WaveFormat format, TimeSpan duration)
{
if (duration <= TimeSpan.Zero)
{
return;
}
var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
writer.Write(bytes, 0, bytes.Length);
}
private static string? FindBestSpeaker(TimeSegment segment, IReadOnlyList<TranscriptionSegment> segments)
{
var bestOverlap = 0d;
string? bestSpeaker = null;
foreach (var transcriptSegment in segments)
{
var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
- Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
if (overlap > bestOverlap)
{
bestOverlap = overlap;
bestSpeaker = transcriptSegment.Speaker;
}
}
return bestSpeaker;
}
private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
{
return left.Encoding == right.Encoding
&& left.SampleRate == right.SampleRate
&& left.Channels == right.Channels
&& left.BitsPerSample == right.BitsPerSample;
}
private static void TryDelete(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (IOException)
{
}
return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
}
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<KnownCompositeSegment> KnownSegments);
private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment);
private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment, byte[] Snippet);
private readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
}
@@ -0,0 +1,18 @@
namespace MeetingAssistant.Speakers;
public sealed class NoopSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
{
public Task<bool> ValidateSampleAsync(
byte[] wavBytes,
CancellationToken cancellationToken)
{
return Task.FromResult(true);
}
public Task<bool> ValidateMatchAsync(
SpeakerIdentityMatchValidationRequest request,
CancellationToken cancellationToken)
{
return Task.FromResult(true);
}
}
@@ -0,0 +1,237 @@
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
using NAudio.Wave;
namespace MeetingAssistant.Speakers;
public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
{
private readonly PyannoteTranscriptFinalizer finalizer;
private readonly SpeakerIdentityPyannoteValidationOptions options;
private readonly ILogger<PyannoteSpeakerIdentityMatchValidator> logger;
public PyannoteSpeakerIdentityMatchValidator(
PyannoteTranscriptFinalizer finalizer,
IOptions<MeetingAssistantOptions> options,
ILogger<PyannoteSpeakerIdentityMatchValidator> logger)
{
this.finalizer = finalizer;
this.options = options.Value.SpeakerIdentification.PyannoteValidation;
this.logger = logger;
}
public async Task<bool> ValidateSampleAsync(
byte[] wavBytes,
CancellationToken cancellationToken)
{
if (!options.Enabled)
{
return true;
}
var segments = await DiarizeBytesAsync(wavBytes, cancellationToken);
var result = HasSingleSpeaker(segments);
if (!result)
{
logger.LogInformation(
"pyannote rejected speaker identity sample because it did not contain one dominant speaker");
}
return result;
}
public async Task<bool> ValidateMatchAsync(
SpeakerIdentityMatchValidationRequest request,
CancellationToken cancellationToken)
{
if (!options.Enabled)
{
return true;
}
var tempPath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-speaker-match-pyannote",
$"{Guid.NewGuid():N}.wav");
try
{
Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
var layout = WriteCompositeWav(
tempPath,
request.KnownSnippets,
request.UnknownSnippet,
TimeSpan.FromSeconds(1));
if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
{
return false;
}
var segments = await DiarizePathAsync(tempPath, cancellationToken);
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
if (string.IsNullOrWhiteSpace(unknownSpeaker))
{
logger.LogInformation(
"pyannote rejected speaker identity match for {DiarizedSpeaker} because the unknown sample had no speaker",
request.DiarizedSpeaker);
return false;
}
var compared = 0;
var matching = 0;
foreach (var knownSegment in layout.KnownSegments)
{
var knownSpeaker = FindBestSpeaker(knownSegment, segments);
if (string.IsNullOrWhiteSpace(knownSpeaker))
{
continue;
}
compared++;
if (string.Equals(knownSpeaker, unknownSpeaker, StringComparison.Ordinal))
{
matching++;
}
}
var minimumRatio = Math.Clamp(options.MinimumMatchingKnownSnippetRatio, 0, 1);
var accepted = compared > 0 && (double)matching / compared >= minimumRatio;
if (!accepted)
{
logger.LogInformation(
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched",
request.DiarizedSpeaker,
request.IdentityId,
matching,
compared);
}
return accepted;
}
finally
{
SpeakerCompositeWav.TryDelete(tempPath);
}
}
private async Task<IReadOnlyList<TranscriptionSegment>> DiarizeBytesAsync(
byte[] wavBytes,
CancellationToken cancellationToken)
{
var tempPath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-speaker-sample-pyannote",
$"{Guid.NewGuid():N}.wav");
try
{
Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
await File.WriteAllBytesAsync(tempPath, wavBytes, cancellationToken);
return await DiarizePathAsync(tempPath, cancellationToken);
}
finally
{
SpeakerCompositeWav.TryDelete(tempPath);
}
}
private async Task<IReadOnlyList<TranscriptionSegment>> DiarizePathAsync(
string wavPath,
CancellationToken cancellationToken)
{
TimeSpan duration;
try
{
using var reader = new WaveFileReader(wavPath);
duration = reader.TotalTime;
}
catch (Exception exception) when (exception is IOException or InvalidDataException or FormatException)
{
logger.LogInformation(exception, "pyannote speaker identity validation could not read WAV input");
return [];
}
if (duration <= TimeSpan.Zero)
{
return [];
}
return await finalizer.FinalizeAsync(
wavPath,
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
options.Diarization,
SpeechRecognitionPipelineOptions.Default,
cancellationToken);
}
private bool HasSingleSpeaker(IReadOnlyList<TranscriptionSegment> segments)
{
var durationsBySpeaker = segments
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker) && segment.End > segment.Start)
.GroupBy(segment => segment.Speaker)
.Select(group => new
{
Speaker = group.Key,
Duration = group.Sum(segment => (segment.End - segment.Start).TotalSeconds)
})
.Where(group => group.Duration > 0)
.ToList();
if (durationsBySpeaker.Count == 0)
{
return false;
}
if (durationsBySpeaker.Count == 1)
{
return true;
}
var total = durationsBySpeaker.Sum(group => group.Duration);
var dominant = durationsBySpeaker.Max(group => group.Duration);
return total > 0 && dominant / total >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
}
private CompositeLayout WriteCompositeWav(
string path,
IReadOnlyList<byte[]> knownSnippets,
byte[] unknownSnippet,
TimeSpan silence)
{
using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(knownSnippets.Append(unknownSnippet));
if (firstReader is null)
{
return new CompositeLayout(null, []);
}
using var writer = new WaveFileWriter(path, firstReader.WaveFormat);
var current = TimeSpan.Zero;
var knownSegments = new List<TimeSegment>();
foreach (var snippet in knownSnippets.Where(storedSnippet => storedSnippet.Length > 0))
{
var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
if (segment is null)
{
continue;
}
knownSegments.Add(segment.Value);
current = segment.Value.End + silence;
SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
}
var unknownSegment = SpeakerCompositeWav.AppendSnippet(
writer,
firstReader.WaveFormat,
unknownSnippet,
current);
return new CompositeLayout(unknownSegment, knownSegments);
}
private static string? FindBestSpeaker(
TimeSegment segment,
IReadOnlyList<TranscriptionSegment> segments)
{
return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
}
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<TimeSegment> KnownSegments);
}
@@ -0,0 +1,107 @@
using MeetingAssistant.Transcription;
using NAudio.Wave;
namespace MeetingAssistant.Speakers;
internal static class SpeakerCompositeWav
{
public static WaveFileReader? OpenFirstReadableWave(IEnumerable<byte[]> snippets)
{
foreach (var bytes in snippets)
{
if (bytes.Length == 0)
{
continue;
}
try
{
return new WaveFileReader(new MemoryStream(bytes));
}
catch (FormatException)
{
}
}
return null;
}
public static TimeSegment? AppendSnippet(
WaveFileWriter writer,
WaveFormat targetFormat,
byte[] snippet,
TimeSpan start)
{
if (snippet.Length == 0)
{
return null;
}
using var reader = new WaveFileReader(new MemoryStream(snippet));
if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
{
throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
}
reader.CopyTo(writer);
return new TimeSegment(start, start + reader.TotalTime);
}
public static void WriteSilence(
WaveFileWriter writer,
WaveFormat format,
TimeSpan duration)
{
if (duration <= TimeSpan.Zero)
{
return;
}
var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
writer.Write(bytes, 0, bytes.Length);
}
public static string? FindBestSpeaker(
TimeSegment segment,
IReadOnlyList<TranscriptionSegment> segments)
{
var bestOverlap = 0d;
string? bestSpeaker = null;
foreach (var transcriptSegment in segments)
{
var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
- Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
if (overlap > bestOverlap)
{
bestOverlap = overlap;
bestSpeaker = transcriptSegment.Speaker;
}
}
return bestSpeaker;
}
public static void TryDelete(string path)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (IOException)
{
}
}
private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
{
return left.Encoding == right.Encoding
&& left.SampleRate == right.SampleRate
&& left.Channels == right.Channels
&& left.BitsPerSample == right.BitsPerSample;
}
}
internal readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
@@ -38,6 +38,12 @@ public sealed record SpeakerIdentityMatchCandidate(
public sealed record SpeakerIdentityMatch(int IdentityId);
public sealed record SpeakerIdentityMatchValidationRequest(
string DiarizedSpeaker,
int IdentityId,
byte[] UnknownSnippet,
IReadOnlyList<byte[]> KnownSnippets);
public interface ISpeakerIdentityMatcher
{
Task<SpeakerIdentityMatch?> MatchAsync(
@@ -45,6 +51,17 @@ public interface ISpeakerIdentityMatcher
CancellationToken cancellationToken);
}
public interface ISpeakerIdentityMatchValidator
{
Task<bool> ValidateSampleAsync(
byte[] wavBytes,
CancellationToken cancellationToken);
Task<bool> ValidateMatchAsync(
SpeakerIdentityMatchValidationRequest request,
CancellationToken cancellationToken);
}
public interface ISpeakerSnippetExtractor
{
Task<byte[]> ExtractSnippetAsync(
+5 -2
View File
@@ -13,6 +13,8 @@ public sealed class SpeakerIdentity
public DateTimeOffset UpdatedAt { get; set; }
public int TranscriptionCount { get; set; }
public List<SpeakerCandidateName> CandidateNames { get; set; } = [];
public List<SpeakerAlias> Aliases { get; set; } = [];
@@ -154,8 +156,9 @@ public sealed class SpeakerIdentityDbContext : DbContext
{
modelBuilder.Entity<SpeakerIdentity>(entity =>
{
entity.Property<int>("TranscriptionCount")
.HasDefaultValue(0);
entity.Property(identity => identity.TranscriptionCount)
.IsRequired()
.ValueGeneratedNever();
entity.HasMany(identity => identity.CandidateNames)
.WithOne(candidate => candidate.SpeakerIdentity)
@@ -10,6 +10,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerSnippetExtractor snippetExtractor;
private readonly ISpeakerIdentityMatcher matcher;
private readonly ISpeakerIdentityMatchValidator matchValidator;
private readonly SpeakerIdentificationOptions options;
private readonly ILogger<SpeakerIdentityService> logger;
@@ -18,11 +19,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
ISpeakerSnippetExtractor snippetExtractor,
ISpeakerIdentityMatcher matcher,
IOptions<MeetingAssistantOptions> options,
ILogger<SpeakerIdentityService> logger)
ILogger<SpeakerIdentityService> logger,
ISpeakerIdentityMatchValidator matchValidator)
{
this.dbContextFactory = dbContextFactory;
this.snippetExtractor = snippetExtractor;
this.matcher = matcher;
this.matchValidator = matchValidator;
this.options = options.Value.SpeakerIdentification;
this.logger = logger;
}
@@ -198,7 +201,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
? suppliedSnippet
: await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
group.ToList(),
SpeakerSampleSpanSelector.SelectBestContinuousSpan(
request.Segments,
speaker,
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration),
cancellationToken);
if (snippet.Length == 0)
{
@@ -544,6 +551,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
{
if (!await matchValidator.ValidateSampleAsync(snippet, cancellationToken))
{
logger.LogInformation(
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample",
speaker);
continue;
}
var now = DateTimeOffset.UtcNow;
var canonicalName = remainingCandidates.Count == 1 ? remainingCandidates[0] : null;
var identity = new SpeakerIdentity
@@ -0,0 +1,79 @@
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Speakers;
internal static class SpeakerSampleSpanSelector
{
public static bool CanExtend(
string currentSpeaker,
TimeSpan currentEnd,
TranscriptionSegment nextSegment,
TimeSpan maximumSegmentGap)
{
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
nextSegment.Start - currentEnd <= maximumSegmentGap;
}
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan maximumSegmentGap,
TimeSpan minimumDuration)
{
var best = new List<TranscriptionSegment>();
var current = new List<TranscriptionSegment>();
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (current.Count == 0)
{
if (IsSpeaker(segment, speaker))
{
current.Add(segment);
best = LongerSpan(current, best);
}
continue;
}
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
{
current.Clear();
if (!IsSpeaker(segment, speaker))
{
continue;
}
}
current.Add(segment);
best = LongerSpan(current, best);
}
return SpanDuration(best) >= minimumDuration
? best
: [];
}
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
{
return segments.Count == 0
? TimeSpan.Zero
: segments[^1].End - segments[0].Start;
}
private static bool IsSpeaker(
TranscriptionSegment segment,
string speaker)
{
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
}
private static List<TranscriptionSegment> LongerSpan(
List<TranscriptionSegment> current,
List<TranscriptionSegment> best)
{
return SpanDuration(current) > SpanDuration(best)
? current.ToList()
: best;
}
}