Public Access
This commit is contained in:
@@ -15,15 +15,18 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
private readonly ISpeakerIdentityDiarizationClient diarizationClient;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<AzureSpeechSpeakerIdentityMatcher> logger;
|
||||
private readonly ISpeakerIdentityMatchValidator matchValidator;
|
||||
|
||||
public AzureSpeechSpeakerIdentityMatcher(
|
||||
ISpeakerIdentityDiarizationClient diarizationClient,
|
||||
Microsoft.Extensions.Options.IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<AzureSpeechSpeakerIdentityMatcher> logger)
|
||||
ILogger<AzureSpeechSpeakerIdentityMatcher> logger,
|
||||
ISpeakerIdentityMatchValidator matchValidator)
|
||||
{
|
||||
this.diarizationClient = diarizationClient;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
this.matchValidator = matchValidator;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMatch?> MatchAsync(
|
||||
@@ -35,6 +38,14 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!await matchValidator.ValidateSampleAsync(request.UnknownSnippet, cancellationToken))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} skipped because secondary validation rejected the unknown sample",
|
||||
request.DiarizedSpeaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} against {CandidateCount} candidate(s)",
|
||||
request.DiarizedSpeaker,
|
||||
@@ -84,6 +95,20 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
var requiredMatches = known.Count() > 1 ? 2 : 1;
|
||||
if (matchingSnippetCount >= requiredMatches)
|
||||
{
|
||||
var validationRequest = new SpeakerIdentityMatchValidationRequest(
|
||||
request.DiarizedSpeaker,
|
||||
known.Key,
|
||||
request.UnknownSnippet,
|
||||
known.Select(segment => segment.Snippet).ToList());
|
||||
if (!await matchValidator.ValidateMatchAsync(validationRequest, cancellationToken))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} rejected identity {IdentityId} after secondary validation",
|
||||
request.DiarizedSpeaker,
|
||||
known.Key);
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} matched identity {IdentityId}",
|
||||
request.DiarizedSpeaker,
|
||||
@@ -99,7 +124,7 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(tempPath);
|
||||
SpeakerCompositeWav.TryDelete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,7 +155,8 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
|
||||
private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request)
|
||||
{
|
||||
using var firstReader = OpenFirstReadableWave(request);
|
||||
using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(
|
||||
request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet));
|
||||
if (firstReader is null)
|
||||
{
|
||||
return new CompositeLayout(null, []);
|
||||
@@ -145,118 +171,33 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
{
|
||||
foreach (var snippet in candidate.Snippets.Where(storedSnippet => storedSnippet.Length > 0))
|
||||
{
|
||||
var segment = AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
|
||||
var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
|
||||
if (segment is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value));
|
||||
knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value, snippet));
|
||||
current = segment.Value.End + silence;
|
||||
WriteSilence(writer, firstReader.WaveFormat, silence);
|
||||
SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
|
||||
}
|
||||
}
|
||||
|
||||
var unknownSegment = AppendSnippet(writer, firstReader.WaveFormat, request.UnknownSnippet, current);
|
||||
var unknownSegment = SpeakerCompositeWav.AppendSnippet(
|
||||
writer,
|
||||
firstReader.WaveFormat,
|
||||
request.UnknownSnippet,
|
||||
current);
|
||||
return new CompositeLayout(unknownSegment, knownSegments);
|
||||
}
|
||||
|
||||
private static WaveFileReader? OpenFirstReadableWave(SpeakerIdentityMatchRequest request)
|
||||
{
|
||||
foreach (var bytes in request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet))
|
||||
{
|
||||
if (bytes.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return new WaveFileReader(new MemoryStream(bytes));
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static TimeSegment? AppendSnippet(
|
||||
WaveFileWriter writer,
|
||||
WaveFormat targetFormat,
|
||||
byte[] snippet,
|
||||
TimeSpan start)
|
||||
{
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
using var reader = new WaveFileReader(new MemoryStream(snippet));
|
||||
if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
|
||||
{
|
||||
throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
|
||||
}
|
||||
|
||||
reader.CopyTo(writer);
|
||||
return new TimeSegment(start, start + reader.TotalTime);
|
||||
}
|
||||
|
||||
private static void WriteSilence(WaveFileWriter writer, WaveFormat format, TimeSpan duration)
|
||||
{
|
||||
if (duration <= TimeSpan.Zero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
|
||||
writer.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
private static string? FindBestSpeaker(TimeSegment segment, IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
var bestOverlap = 0d;
|
||||
string? bestSpeaker = null;
|
||||
foreach (var transcriptSegment in segments)
|
||||
{
|
||||
var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
|
||||
- Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
|
||||
if (overlap > bestOverlap)
|
||||
{
|
||||
bestOverlap = overlap;
|
||||
bestSpeaker = transcriptSegment.Speaker;
|
||||
}
|
||||
}
|
||||
|
||||
return bestSpeaker;
|
||||
}
|
||||
|
||||
private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
|
||||
{
|
||||
return left.Encoding == right.Encoding
|
||||
&& left.SampleRate == right.SampleRate
|
||||
&& left.Channels == right.Channels
|
||||
&& left.BitsPerSample == right.BitsPerSample;
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
}
|
||||
return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
|
||||
}
|
||||
|
||||
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<KnownCompositeSegment> KnownSegments);
|
||||
|
||||
private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment);
|
||||
private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment, byte[] Snippet);
|
||||
|
||||
private readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class NoopSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
|
||||
{
|
||||
public Task<bool> ValidateSampleAsync(
|
||||
byte[] wavBytes,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
public Task<bool> ValidateMatchAsync(
|
||||
SpeakerIdentityMatchValidationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatchValidator
|
||||
{
|
||||
private readonly PyannoteTranscriptFinalizer finalizer;
|
||||
private readonly SpeakerIdentityPyannoteValidationOptions options;
|
||||
private readonly ILogger<PyannoteSpeakerIdentityMatchValidator> logger;
|
||||
|
||||
public PyannoteSpeakerIdentityMatchValidator(
|
||||
PyannoteTranscriptFinalizer finalizer,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<PyannoteSpeakerIdentityMatchValidator> logger)
|
||||
{
|
||||
this.finalizer = finalizer;
|
||||
this.options = options.Value.SpeakerIdentification.PyannoteValidation;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<bool> ValidateSampleAsync(
|
||||
byte[] wavBytes,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var segments = await DiarizeBytesAsync(wavBytes, cancellationToken);
|
||||
var result = HasSingleSpeaker(segments);
|
||||
if (!result)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity sample because it did not contain one dominant speaker");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<bool> ValidateMatchAsync(
|
||||
SpeakerIdentityMatchValidationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var tempPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-speaker-match-pyannote",
|
||||
$"{Guid.NewGuid():N}.wav");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
|
||||
var layout = WriteCompositeWav(
|
||||
tempPath,
|
||||
request.KnownSnippets,
|
||||
request.UnknownSnippet,
|
||||
TimeSpan.FromSeconds(1));
|
||||
if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var segments = await DiarizePathAsync(tempPath, cancellationToken);
|
||||
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
|
||||
if (string.IsNullOrWhiteSpace(unknownSpeaker))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity match for {DiarizedSpeaker} because the unknown sample had no speaker",
|
||||
request.DiarizedSpeaker);
|
||||
return false;
|
||||
}
|
||||
|
||||
var compared = 0;
|
||||
var matching = 0;
|
||||
foreach (var knownSegment in layout.KnownSegments)
|
||||
{
|
||||
var knownSpeaker = FindBestSpeaker(knownSegment, segments);
|
||||
if (string.IsNullOrWhiteSpace(knownSpeaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
compared++;
|
||||
if (string.Equals(knownSpeaker, unknownSpeaker, StringComparison.Ordinal))
|
||||
{
|
||||
matching++;
|
||||
}
|
||||
}
|
||||
|
||||
var minimumRatio = Math.Clamp(options.MinimumMatchingKnownSnippetRatio, 0, 1);
|
||||
var accepted = compared > 0 && (double)matching / compared >= minimumRatio;
|
||||
if (!accepted)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched",
|
||||
request.DiarizedSpeaker,
|
||||
request.IdentityId,
|
||||
matching,
|
||||
compared);
|
||||
}
|
||||
|
||||
return accepted;
|
||||
}
|
||||
finally
|
||||
{
|
||||
SpeakerCompositeWav.TryDelete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<TranscriptionSegment>> DiarizeBytesAsync(
|
||||
byte[] wavBytes,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var tempPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-speaker-sample-pyannote",
|
||||
$"{Guid.NewGuid():N}.wav");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
|
||||
await File.WriteAllBytesAsync(tempPath, wavBytes, cancellationToken);
|
||||
return await DiarizePathAsync(tempPath, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SpeakerCompositeWav.TryDelete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<TranscriptionSegment>> DiarizePathAsync(
|
||||
string wavPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
TimeSpan duration;
|
||||
try
|
||||
{
|
||||
using var reader = new WaveFileReader(wavPath);
|
||||
duration = reader.TotalTime;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or InvalidDataException or FormatException)
|
||||
{
|
||||
logger.LogInformation(exception, "pyannote speaker identity validation could not read WAV input");
|
||||
return [];
|
||||
}
|
||||
|
||||
if (duration <= TimeSpan.Zero)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await finalizer.FinalizeAsync(
|
||||
wavPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
||||
options.Diarization,
|
||||
SpeechRecognitionPipelineOptions.Default,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private bool HasSingleSpeaker(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
var durationsBySpeaker = segments
|
||||
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker) && segment.End > segment.Start)
|
||||
.GroupBy(segment => segment.Speaker)
|
||||
.Select(group => new
|
||||
{
|
||||
Speaker = group.Key,
|
||||
Duration = group.Sum(segment => (segment.End - segment.Start).TotalSeconds)
|
||||
})
|
||||
.Where(group => group.Duration > 0)
|
||||
.ToList();
|
||||
if (durationsBySpeaker.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (durationsBySpeaker.Count == 1)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var total = durationsBySpeaker.Sum(group => group.Duration);
|
||||
var dominant = durationsBySpeaker.Max(group => group.Duration);
|
||||
return total > 0 && dominant / total >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
|
||||
}
|
||||
|
||||
private CompositeLayout WriteCompositeWav(
|
||||
string path,
|
||||
IReadOnlyList<byte[]> knownSnippets,
|
||||
byte[] unknownSnippet,
|
||||
TimeSpan silence)
|
||||
{
|
||||
using var firstReader = SpeakerCompositeWav.OpenFirstReadableWave(knownSnippets.Append(unknownSnippet));
|
||||
if (firstReader is null)
|
||||
{
|
||||
return new CompositeLayout(null, []);
|
||||
}
|
||||
|
||||
using var writer = new WaveFileWriter(path, firstReader.WaveFormat);
|
||||
var current = TimeSpan.Zero;
|
||||
var knownSegments = new List<TimeSegment>();
|
||||
|
||||
foreach (var snippet in knownSnippets.Where(storedSnippet => storedSnippet.Length > 0))
|
||||
{
|
||||
var segment = SpeakerCompositeWav.AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
|
||||
if (segment is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
knownSegments.Add(segment.Value);
|
||||
current = segment.Value.End + silence;
|
||||
SpeakerCompositeWav.WriteSilence(writer, firstReader.WaveFormat, silence);
|
||||
}
|
||||
|
||||
var unknownSegment = SpeakerCompositeWav.AppendSnippet(
|
||||
writer,
|
||||
firstReader.WaveFormat,
|
||||
unknownSnippet,
|
||||
current);
|
||||
return new CompositeLayout(unknownSegment, knownSegments);
|
||||
}
|
||||
|
||||
private static string? FindBestSpeaker(
|
||||
TimeSegment segment,
|
||||
IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return SpeakerCompositeWav.FindBestSpeaker(segment, segments);
|
||||
}
|
||||
|
||||
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<TimeSegment> KnownSegments);
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerCompositeWav
|
||||
{
|
||||
public static WaveFileReader? OpenFirstReadableWave(IEnumerable<byte[]> snippets)
|
||||
{
|
||||
foreach (var bytes in snippets)
|
||||
{
|
||||
if (bytes.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return new WaveFileReader(new MemoryStream(bytes));
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static TimeSegment? AppendSnippet(
|
||||
WaveFileWriter writer,
|
||||
WaveFormat targetFormat,
|
||||
byte[] snippet,
|
||||
TimeSpan start)
|
||||
{
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
using var reader = new WaveFileReader(new MemoryStream(snippet));
|
||||
if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
|
||||
{
|
||||
throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
|
||||
}
|
||||
|
||||
reader.CopyTo(writer);
|
||||
return new TimeSegment(start, start + reader.TotalTime);
|
||||
}
|
||||
|
||||
public static void WriteSilence(
|
||||
WaveFileWriter writer,
|
||||
WaveFormat format,
|
||||
TimeSpan duration)
|
||||
{
|
||||
if (duration <= TimeSpan.Zero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
|
||||
writer.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
public static string? FindBestSpeaker(
|
||||
TimeSegment segment,
|
||||
IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
var bestOverlap = 0d;
|
||||
string? bestSpeaker = null;
|
||||
foreach (var transcriptSegment in segments)
|
||||
{
|
||||
var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
|
||||
- Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
|
||||
if (overlap > bestOverlap)
|
||||
{
|
||||
bestOverlap = overlap;
|
||||
bestSpeaker = transcriptSegment.Speaker;
|
||||
}
|
||||
}
|
||||
|
||||
return bestSpeaker;
|
||||
}
|
||||
|
||||
public static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
|
||||
{
|
||||
return left.Encoding == right.Encoding
|
||||
&& left.SampleRate == right.SampleRate
|
||||
&& left.Channels == right.Channels
|
||||
&& left.BitsPerSample == right.BitsPerSample;
|
||||
}
|
||||
}
|
||||
|
||||
internal readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
|
||||
@@ -38,6 +38,12 @@ public sealed record SpeakerIdentityMatchCandidate(
|
||||
|
||||
public sealed record SpeakerIdentityMatch(int IdentityId);
|
||||
|
||||
public sealed record SpeakerIdentityMatchValidationRequest(
|
||||
string DiarizedSpeaker,
|
||||
int IdentityId,
|
||||
byte[] UnknownSnippet,
|
||||
IReadOnlyList<byte[]> KnownSnippets);
|
||||
|
||||
public interface ISpeakerIdentityMatcher
|
||||
{
|
||||
Task<SpeakerIdentityMatch?> MatchAsync(
|
||||
@@ -45,6 +51,17 @@ public interface ISpeakerIdentityMatcher
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISpeakerIdentityMatchValidator
|
||||
{
|
||||
Task<bool> ValidateSampleAsync(
|
||||
byte[] wavBytes,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<bool> ValidateMatchAsync(
|
||||
SpeakerIdentityMatchValidationRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISpeakerSnippetExtractor
|
||||
{
|
||||
Task<byte[]> ExtractSnippetAsync(
|
||||
|
||||
@@ -13,6 +13,8 @@ public sealed class SpeakerIdentity
|
||||
|
||||
public DateTimeOffset UpdatedAt { get; set; }
|
||||
|
||||
public int TranscriptionCount { get; set; }
|
||||
|
||||
public List<SpeakerCandidateName> CandidateNames { get; set; } = [];
|
||||
|
||||
public List<SpeakerAlias> Aliases { get; set; } = [];
|
||||
@@ -154,8 +156,9 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
{
|
||||
modelBuilder.Entity<SpeakerIdentity>(entity =>
|
||||
{
|
||||
entity.Property<int>("TranscriptionCount")
|
||||
.HasDefaultValue(0);
|
||||
entity.Property(identity => identity.TranscriptionCount)
|
||||
.IsRequired()
|
||||
.ValueGeneratedNever();
|
||||
|
||||
entity.HasMany(identity => identity.CandidateNames)
|
||||
.WithOne(candidate => candidate.SpeakerIdentity)
|
||||
|
||||
@@ -10,6 +10,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly ISpeakerIdentityMatchValidator matchValidator;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityService> logger;
|
||||
|
||||
@@ -18,11 +19,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityService> logger)
|
||||
ILogger<SpeakerIdentityService> logger,
|
||||
ISpeakerIdentityMatchValidator matchValidator)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.matcher = matcher;
|
||||
this.matchValidator = matchValidator;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -198,7 +201,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
? suppliedSnippet
|
||||
: await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
group.ToList(),
|
||||
SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration),
|
||||
cancellationToken);
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
@@ -544,6 +551,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
|
||||
{
|
||||
if (!await matchValidator.ValidateSampleAsync(snippet, cancellationToken))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample",
|
||||
speaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var canonicalName = remainingCandidates.Count == 1 ? remainingCandidates[0] : null;
|
||||
var identity = new SpeakerIdentity
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerSampleSpanSelector
|
||||
{
|
||||
public static bool CanExtend(
|
||||
string currentSpeaker,
|
||||
TimeSpan currentEnd,
|
||||
TranscriptionSegment nextSegment,
|
||||
TimeSpan maximumSegmentGap)
|
||||
{
|
||||
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
|
||||
nextSegment.Start - currentEnd <= maximumSegmentGap;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan maximumSegmentGap,
|
||||
TimeSpan minimumDuration)
|
||||
{
|
||||
var best = new List<TranscriptionSegment>();
|
||||
var current = new List<TranscriptionSegment>();
|
||||
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (current.Count == 0)
|
||||
{
|
||||
if (IsSpeaker(segment, speaker))
|
||||
{
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
|
||||
{
|
||||
current.Clear();
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
return SpanDuration(best) >= minimumDuration
|
||||
? best
|
||||
: [];
|
||||
}
|
||||
|
||||
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return segments.Count == 0
|
||||
? TimeSpan.Zero
|
||||
: segments[^1].End - segments[0].Start;
|
||||
}
|
||||
|
||||
private static bool IsSpeaker(
|
||||
TranscriptionSegment segment,
|
||||
string speaker)
|
||||
{
|
||||
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> LongerSpan(
|
||||
List<TranscriptionSegment> current,
|
||||
List<TranscriptionSegment> best)
|
||||
{
|
||||
return SpanDuration(current) > SpanDuration(best)
|
||||
? current.ToList()
|
||||
: best;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user