Public Access
Add meeting assistant speech and summary automation
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.CognitiveServices.Speech;
|
||||
using Microsoft.CognitiveServices.Speech.Audio;
|
||||
using Microsoft.CognitiveServices.Speech.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class AzureSpeechSpeakerIdentityDiarizationClient : ISpeakerIdentityDiarizationClient
|
||||
{
|
||||
private readonly AzureSpeechOptions options;
|
||||
private readonly ILogger<AzureSpeechSpeakerIdentityDiarizationClient> logger;
|
||||
|
||||
public AzureSpeechSpeakerIdentityDiarizationClient(
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<AzureSpeechSpeakerIdentityDiarizationClient> logger)
|
||||
{
|
||||
this.options = options.Value.SpeakerIdentification.AzureSpeech;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> DiarizeAsync(
|
||||
string wavPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var key = AzureSpeechStreamingTranscriptionProvider.ResolveKey(options);
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Speaker identity Azure Speech matching is enabled but no key is configured directly or via {KeyEnv}",
|
||||
options.KeyEnv);
|
||||
return [];
|
||||
}
|
||||
|
||||
var speechConfig = string.IsNullOrWhiteSpace(options.Region)
|
||||
? SpeechConfig.FromEndpoint(new Uri(options.Endpoint), key)
|
||||
: SpeechConfig.FromSubscription(key, options.Region);
|
||||
speechConfig.SpeechRecognitionLanguage = ResolveRecognitionLanguage(options);
|
||||
speechConfig.OutputFormat = OutputFormat.Detailed;
|
||||
speechConfig.SetProperty(
|
||||
PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
|
||||
options.DiarizeIntermediateResults ? "true" : "false");
|
||||
|
||||
using var audioConfig = AudioConfig.FromWavFileInput(wavPath);
|
||||
using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
|
||||
var channel = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
|
||||
recognizer.Transcribed += (_, args) =>
|
||||
{
|
||||
if (args.Result.Reason != ResultReason.RecognizedSpeech)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
|
||||
channel.Writer.TryWrite(new TranscriptionSegment(
|
||||
start,
|
||||
start + args.Result.Duration,
|
||||
NormalizeSpeakerId(args.Result.SpeakerId),
|
||||
args.Result.Text.Trim()));
|
||||
};
|
||||
recognizer.Canceled += (_, args) =>
|
||||
{
|
||||
if (args.Reason == CancellationReason.Error)
|
||||
{
|
||||
channel.Writer.TryComplete(new InvalidOperationException(
|
||||
$"Azure Speech identity matching canceled: {args.ErrorCode} {args.ErrorDetails}"));
|
||||
return;
|
||||
}
|
||||
|
||||
channel.Writer.TryComplete();
|
||||
};
|
||||
recognizer.SessionStopped += (_, _) => channel.Writer.TryComplete();
|
||||
|
||||
await recognizer.StartTranscribingAsync();
|
||||
var segments = new List<TranscriptionSegment>();
|
||||
try
|
||||
{
|
||||
await foreach (var segment in channel.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
segments.Add(segment);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
var stopTask = recognizer.StopTranscribingAsync();
|
||||
if (options.RecognitionStopTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await stopTask.WaitAsync(options.RecognitionStopTimeout, CancellationToken.None);
|
||||
}
|
||||
else
|
||||
{
|
||||
await stopTask.WaitAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning("Timed out while stopping Azure Speech identity diarization recognizer");
|
||||
}
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
internal static string ResolveRecognitionLanguage(AzureSpeechOptions options)
|
||||
{
|
||||
if (!options.Language.Equals("auto", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return options.Language;
|
||||
}
|
||||
|
||||
return options.AutoDetectLanguages
|
||||
.Select(language => language.Trim())
|
||||
.FirstOrDefault(language => !string.IsNullOrWhiteSpace(language))
|
||||
?? "de-DE";
|
||||
}
|
||||
|
||||
private static string NormalizeSpeakerId(string? speakerId)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase)
|
||||
? "Unknown"
|
||||
: speakerId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public interface ISpeakerIdentityDiarizationClient
|
||||
{
|
||||
Task<IReadOnlyList<TranscriptionSegment>> DiarizeAsync(
|
||||
string wavPath,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
{
|
||||
private readonly ISpeakerIdentityDiarizationClient diarizationClient;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<AzureSpeechSpeakerIdentityMatcher> logger;
|
||||
|
||||
public AzureSpeechSpeakerIdentityMatcher(
|
||||
ISpeakerIdentityDiarizationClient diarizationClient,
|
||||
Microsoft.Extensions.Options.IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<AzureSpeechSpeakerIdentityMatcher> logger)
|
||||
{
|
||||
this.diarizationClient = diarizationClient;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMatch?> MatchAsync(
|
||||
SpeakerIdentityMatchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Candidates.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} against {CandidateCount} candidate(s)",
|
||||
request.DiarizedSpeaker,
|
||||
request.Candidates.Count);
|
||||
var tempPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-speaker-match", $"{Guid.NewGuid():N}.wav");
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(tempPath)!);
|
||||
var layout = WriteCompositeWav(tempPath, request);
|
||||
if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} skipped because no readable snippets were available",
|
||||
request.DiarizedSpeaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
var segments = await diarizationClient.DiarizeAsync(tempPath, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} received {SegmentCount} diarized segment(s)",
|
||||
request.DiarizedSpeaker,
|
||||
segments.Count);
|
||||
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
|
||||
if (string.IsNullOrWhiteSpace(unknownSpeaker))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} found no speaker for the unknown snippet",
|
||||
request.DiarizedSpeaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var known in layout.KnownSegments.GroupBy(segment => segment.IdentityId))
|
||||
{
|
||||
var matchingSnippetCount = known.Count(segment =>
|
||||
string.Equals(
|
||||
FindBestSpeaker(segment.Segment, segments),
|
||||
unknownSpeaker,
|
||||
StringComparison.Ordinal));
|
||||
var requiredMatches = known.Count() > 1 ? 2 : 1;
|
||||
if (matchingSnippetCount >= requiredMatches)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} matched identity {IdentityId}",
|
||||
request.DiarizedSpeaker,
|
||||
known.Key);
|
||||
return new SpeakerIdentityMatch(known.Key);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity matching {DiarizedSpeaker} found no matching identity",
|
||||
request.DiarizedSpeaker);
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request)
|
||||
{
|
||||
using var firstReader = OpenFirstReadableWave(request);
|
||||
if (firstReader is null)
|
||||
{
|
||||
return new CompositeLayout(null, []);
|
||||
}
|
||||
|
||||
var silence = TimeSpan.FromSeconds(Math.Max(0, options.SilenceBetweenSnippetsSeconds));
|
||||
using var writer = new WaveFileWriter(path, firstReader.WaveFormat);
|
||||
var current = TimeSpan.Zero;
|
||||
var knownSegments = new List<KnownCompositeSegment>();
|
||||
|
||||
foreach (var candidate in request.Candidates)
|
||||
{
|
||||
foreach (var snippet in candidate.Snippets.Where(storedSnippet => storedSnippet.Length > 0))
|
||||
{
|
||||
var segment = AppendSnippet(writer, firstReader.WaveFormat, snippet, current);
|
||||
if (segment is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
knownSegments.Add(new KnownCompositeSegment(candidate.IdentityId, segment.Value));
|
||||
current = segment.Value.End + silence;
|
||||
WriteSilence(writer, firstReader.WaveFormat, silence);
|
||||
}
|
||||
}
|
||||
|
||||
var unknownSegment = AppendSnippet(writer, firstReader.WaveFormat, request.UnknownSnippet, current);
|
||||
return new CompositeLayout(unknownSegment, knownSegments);
|
||||
}
|
||||
|
||||
private static WaveFileReader? OpenFirstReadableWave(SpeakerIdentityMatchRequest request)
|
||||
{
|
||||
foreach (var bytes in request.Candidates.SelectMany(candidate => candidate.Snippets).Append(request.UnknownSnippet))
|
||||
{
|
||||
if (bytes.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return new WaveFileReader(new MemoryStream(bytes));
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static TimeSegment? AppendSnippet(
|
||||
WaveFileWriter writer,
|
||||
WaveFormat targetFormat,
|
||||
byte[] snippet,
|
||||
TimeSpan start)
|
||||
{
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
using var reader = new WaveFileReader(new MemoryStream(snippet));
|
||||
if (!WaveFormatsMatch(reader.WaveFormat, targetFormat))
|
||||
{
|
||||
throw new InvalidDataException("Speaker identity snippets must use the same WAV format.");
|
||||
}
|
||||
|
||||
reader.CopyTo(writer);
|
||||
return new TimeSegment(start, start + reader.TotalTime);
|
||||
}
|
||||
|
||||
private static void WriteSilence(WaveFileWriter writer, WaveFormat format, TimeSpan duration)
|
||||
{
|
||||
if (duration <= TimeSpan.Zero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var bytes = new byte[(int)(format.AverageBytesPerSecond * duration.TotalSeconds)];
|
||||
writer.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
private static string? FindBestSpeaker(TimeSegment segment, IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
var bestOverlap = 0d;
|
||||
string? bestSpeaker = null;
|
||||
foreach (var transcriptSegment in segments)
|
||||
{
|
||||
var overlap = Math.Min(segment.End.TotalSeconds, transcriptSegment.End.TotalSeconds)
|
||||
- Math.Max(segment.Start.TotalSeconds, transcriptSegment.Start.TotalSeconds);
|
||||
if (overlap > bestOverlap)
|
||||
{
|
||||
bestOverlap = overlap;
|
||||
bestSpeaker = transcriptSegment.Speaker;
|
||||
}
|
||||
}
|
||||
|
||||
return bestSpeaker;
|
||||
}
|
||||
|
||||
private static bool WaveFormatsMatch(WaveFormat left, WaveFormat right)
|
||||
{
|
||||
return left.Encoding == right.Encoding
|
||||
&& left.SampleRate == right.SampleRate
|
||||
&& left.Channels == right.Channels
|
||||
&& left.BitsPerSample == right.BitsPerSample;
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<KnownCompositeSegment> KnownSegments);
|
||||
|
||||
private sealed record KnownCompositeSegment(int IdentityId, TimeSegment Segment);
|
||||
|
||||
private readonly record struct TimeSegment(TimeSpan Start, TimeSpan End);
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class NoopSpeakerIdentityMatcher : ISpeakerIdentityMatcher
|
||||
{
|
||||
public Task<SpeakerIdentityMatch?> MatchAsync(
|
||||
SpeakerIdentityMatchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<SpeakerIdentityMatch?>(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record SpeakerIdentificationRequest(
|
||||
string AudioPath,
|
||||
MeetingNote MeetingNote,
|
||||
IReadOnlyList<TranscriptionSegment> Segments,
|
||||
IReadOnlyList<SpeakerAudioSample>? Samples = null);
|
||||
|
||||
public sealed record SpeakerAudioSample(
|
||||
string Speaker,
|
||||
TranscriptionSegment Segment,
|
||||
byte[] WavBytes,
|
||||
double Score);
|
||||
|
||||
public sealed record SpeakerIdentificationResult(
|
||||
IReadOnlyList<TranscriptionSegment> Segments,
|
||||
IReadOnlyDictionary<string, string> SpeakerMappings,
|
||||
IReadOnlyList<SpeakerIdentityAttendeeMatch>? AttendeeMatches = null);
|
||||
|
||||
public sealed record SpeakerIdentityAttendeeMatch(
|
||||
string DisplayName,
|
||||
IReadOnlyList<string> AcceptedNames);
|
||||
|
||||
public sealed record SpeakerIdentityMatchRequest(
|
||||
string DiarizedSpeaker,
|
||||
byte[] UnknownSnippet,
|
||||
IReadOnlyList<SpeakerIdentityMatchCandidate> Candidates);
|
||||
|
||||
public sealed record SpeakerIdentityMatchCandidate(
|
||||
int IdentityId,
|
||||
string? CanonicalName,
|
||||
int TranscriptionCount,
|
||||
IReadOnlyList<byte[]> Snippets);
|
||||
|
||||
public sealed record SpeakerIdentityMatch(int IdentityId);
|
||||
|
||||
public interface ISpeakerIdentityMatcher
|
||||
{
|
||||
Task<SpeakerIdentityMatch?> MatchAsync(
|
||||
SpeakerIdentityMatchRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISpeakerSnippetExtractor
|
||||
{
|
||||
Task<byte[]> ExtractSnippetAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISpeakerIdentificationService
|
||||
{
|
||||
Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class SpeakerIdentity
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public string? CanonicalName { get; set; }
|
||||
|
||||
public int TranscriptionCount { get; set; }
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
public DateTimeOffset UpdatedAt { get; set; }
|
||||
|
||||
public List<SpeakerCandidateName> CandidateNames { get; set; } = [];
|
||||
|
||||
public List<SpeakerAlias> Aliases { get; set; } = [];
|
||||
|
||||
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
||||
|
||||
public string? GetDisplayName()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(CanonicalName))
|
||||
{
|
||||
return CanonicalName;
|
||||
}
|
||||
|
||||
return Aliases
|
||||
.Select(alias => alias.Name)
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SpeakerCandidateName
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public string Name { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SpeakerSnippet
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public byte[] WavBytes { get; set; } = [];
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerAlias
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public string Name { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class SpeakerIdentityDbContext : DbContext
|
||||
{
|
||||
public SpeakerIdentityDbContext(DbContextOptions<SpeakerIdentityDbContext> options)
|
||||
: base(options)
|
||||
{
|
||||
}
|
||||
|
||||
public DbSet<SpeakerIdentity> SpeakerIdentities => Set<SpeakerIdentity>();
|
||||
|
||||
public DbSet<SpeakerCandidateName> SpeakerCandidateNames => Set<SpeakerCandidateName>();
|
||||
|
||||
public DbSet<SpeakerAlias> SpeakerAliases => Set<SpeakerAlias>();
|
||||
|
||||
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
modelBuilder.Entity<SpeakerIdentity>(entity =>
|
||||
{
|
||||
entity.HasMany(identity => identity.CandidateNames)
|
||||
.WithOne(candidate => candidate.SpeakerIdentity)
|
||||
.HasForeignKey(candidate => candidate.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.Aliases)
|
||||
.WithOne(alias => alias.SpeakerIdentity)
|
||||
.HasForeignKey(alias => alias.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.Snippets)
|
||||
.WithOne(snippet => snippet.SpeakerIdentity)
|
||||
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity<SpeakerCandidateName>()
|
||||
.HasIndex(candidate => new { candidate.SpeakerIdentityId, candidate.Name })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerAlias>()
|
||||
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
||||
.IsUnique();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class SpeakerIdentityDatabaseInitializer : IHostedService
|
||||
{
|
||||
private static readonly SemaphoreSlim InitializeLock = new(1, 1);
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
|
||||
public SpeakerIdentityDatabaseInitializer(IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await InitializeLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
InitializeLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public interface ISpeakerIdentityMergeService
|
||||
{
|
||||
Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||
TimeSpan? recentIdentityAge,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record SpeakerIdentityMergeResult(
|
||||
int RecentIdentityCount,
|
||||
int CandidateIdentityCount,
|
||||
int MatchAttempts,
|
||||
int MergedPairs);
|
||||
|
||||
public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
||||
|
||||
public SpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||
TimeSpan? recentIdentityAge,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.OrderByDescending(identity => identity.TranscriptionCount)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
var recentIds = identities
|
||||
.Where(identity => identity.CreatedAt >= cutoff)
|
||||
.Select(identity => identity.Id)
|
||||
.ToHashSet();
|
||||
var mergedPairs = 0;
|
||||
var attempts = 0;
|
||||
|
||||
foreach (var sourceId in recentIds.ToList())
|
||||
{
|
||||
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||
if (source is null || source.Snippets.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var targets = identities
|
||||
.Where(identity => identity.Id != source.Id && identity.Snippets.Count > 0)
|
||||
.ToList();
|
||||
foreach (var batch in targets.Chunk(Math.Max(1, options.MatchBatchSize)))
|
||||
{
|
||||
var firstSnippet = SelectSnippet(source, excludedSnippet: null);
|
||||
if (firstSnippet is null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var firstMatch = await matcher.MatchAsync(
|
||||
CreateRequest(source, firstSnippet, batch),
|
||||
cancellationToken);
|
||||
if (firstMatch is null || firstMatch.IdentityId == source.Id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = batch.SingleOrDefault(identity => identity.Id == firstMatch.IdentityId);
|
||||
if (target is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var secondSnippet = SelectSnippet(source, excludedSnippet: firstSnippet);
|
||||
if (secondSnippet is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var secondMatch = await matcher.MatchAsync(
|
||||
CreateRequest(source, secondSnippet, batch),
|
||||
cancellationToken);
|
||||
if (secondMatch?.IdentityId != target.Id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
MergeIdentities(target, source);
|
||||
context.SpeakerIdentities.Remove(source);
|
||||
identities.Remove(source);
|
||||
mergedPairs++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics completed: {MergedPairs} merge(s), {Attempts} match attempt(s)",
|
||||
mergedPairs,
|
||||
attempts);
|
||||
return new SpeakerIdentityMergeResult(
|
||||
recentIds.Count,
|
||||
identities.Count,
|
||||
attempts,
|
||||
mergedPairs);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityMatchRequest CreateRequest(
|
||||
SpeakerIdentity source,
|
||||
SpeakerSnippet unknownSnippet,
|
||||
IReadOnlyList<SpeakerIdentity> candidates)
|
||||
{
|
||||
return new SpeakerIdentityMatchRequest(
|
||||
source.GetDisplayName() ?? $"identity-{source.Id}",
|
||||
unknownSnippet.WavBytes,
|
||||
candidates.Select(identity => new SpeakerIdentityMatchCandidate(
|
||||
identity.Id,
|
||||
identity.GetDisplayName(),
|
||||
identity.TranscriptionCount,
|
||||
identity.Snippets
|
||||
.OrderBy(snippet => snippet.CreatedAt)
|
||||
.Select(snippet => snippet.WavBytes)
|
||||
.ToList()))
|
||||
.ToList());
|
||||
}
|
||||
|
||||
private static SpeakerSnippet? SelectSnippet(SpeakerIdentity identity, SpeakerSnippet? excludedSnippet)
|
||||
{
|
||||
return identity.Snippets
|
||||
.OrderBy(snippet => snippet.CreatedAt)
|
||||
.FirstOrDefault(snippet => excludedSnippet is null || snippet.Id != excludedSnippet.Id);
|
||||
}
|
||||
|
||||
private void MergeIdentities(SpeakerIdentity target, SpeakerIdentity source)
|
||||
{
|
||||
AddAlias(target, source.CanonicalName);
|
||||
foreach (var alias in source.Aliases)
|
||||
{
|
||||
AddAlias(target, alias.Name);
|
||||
}
|
||||
|
||||
foreach (var candidate in source.CandidateNames)
|
||||
{
|
||||
AddAlias(target, candidate.Name);
|
||||
}
|
||||
|
||||
target.TranscriptionCount += source.TranscriptionCount;
|
||||
target.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
var retainedSnippets = target.Snippets
|
||||
.Concat(source.Snippets)
|
||||
.OrderBy(snippet => StableSnippetKey(snippet.WavBytes))
|
||||
.Take(Math.Max(1, options.MaxSnippetsPerSpeaker))
|
||||
.Select(snippet => new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet.WavBytes,
|
||||
CreatedAt = snippet.CreatedAt
|
||||
})
|
||||
.ToList();
|
||||
target.Snippets.Clear();
|
||||
target.Snippets.AddRange(retainedSnippets);
|
||||
}
|
||||
|
||||
private static void AddAlias(SpeakerIdentity identity, string? alias)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(alias) ||
|
||||
string.Equals(identity.CanonicalName, alias, StringComparison.OrdinalIgnoreCase) ||
|
||||
identity.Aliases.Any(existing => string.Equals(existing.Name, alias, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
||||
}
|
||||
|
||||
private static int StableSnippetKey(byte[] snippet)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
foreach (var value in snippet)
|
||||
{
|
||||
hash.Add(value);
|
||||
}
|
||||
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentitySchema
|
||||
{
|
||||
public static async Task EnsureCreatedOrUpdatedAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await context.Database.EnsureCreatedAsync(cancellationToken);
|
||||
await EnsureSpeakerIdentityTimestampColumnsAsync(context, cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS "SpeakerAliases" (
|
||||
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerAliases" PRIMARY KEY AUTOINCREMENT,
|
||||
"SpeakerIdentityId" INTEGER NOT NULL,
|
||||
"Name" TEXT NOT NULL,
|
||||
CONSTRAINT "FK_SpeakerAliases_SpeakerIdentities_SpeakerIdentityId"
|
||||
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
|
||||
);
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerAliases_SpeakerIdentityId_Name"
|
||||
ON "SpeakerAliases" ("SpeakerIdentityId", "Name");
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
const string defaultTimestamp = "1970-01-01T00:00:00.0000000+00:00";
|
||||
var hasCreatedAt = await HasColumnAsync(context, "SpeakerIdentities", "CreatedAt", cancellationToken);
|
||||
if (!hasCreatedAt)
|
||||
{
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
$"""
|
||||
ALTER TABLE "SpeakerIdentities"
|
||||
ADD COLUMN "CreatedAt" TEXT NOT NULL DEFAULT '{defaultTimestamp}';
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
var hasUpdatedAt = await HasColumnAsync(context, "SpeakerIdentities", "UpdatedAt", cancellationToken);
|
||||
if (!hasUpdatedAt)
|
||||
{
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
$"""
|
||||
ALTER TABLE "SpeakerIdentities"
|
||||
ADD COLUMN "UpdatedAt" TEXT NOT NULL DEFAULT '{defaultTimestamp}';
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
$"""
|
||||
UPDATE "SpeakerIdentities"
|
||||
SET "UpdatedAt" = "CreatedAt"
|
||||
WHERE "UpdatedAt" = '{defaultTimestamp}';
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<bool> HasColumnAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
string tableName,
|
||||
string columnName,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = context.Database.GetDbConnection();
|
||||
var shouldClose = connection.State == System.Data.ConnectionState.Closed;
|
||||
if (shouldClose)
|
||||
{
|
||||
await connection.OpenAsync(cancellationToken);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = $"""PRAGMA table_info("{tableName}")""";
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (shouldClose)
|
||||
{
|
||||
await connection.CloseAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityService> logger;
|
||||
|
||||
public SpeakerIdentityService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.matcher = matcher;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.Final, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await ProcessTranscriptAsync(request, mode: SpeakerIdentityProcessingMode.LiveReadOnly, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
SpeakerIdentityProcessingMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return new SpeakerIdentificationResult(request.Segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var matchedAcceptedNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
|
||||
var samplesBySpeaker = request.Samples?
|
||||
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
|
||||
.GroupBy(sample => sample.Speaker, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(
|
||||
group => group.Key,
|
||||
group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var group in request.Segments
|
||||
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker))
|
||||
.GroupBy(segment => segment.Speaker)
|
||||
.OrderBy(group => group.Min(segment => segment.Start)))
|
||||
{
|
||||
var speaker = group.Key;
|
||||
if (speakerMappings.ContainsKey(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet)
|
||||
? suppliedSnippet
|
||||
: await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
group.ToList(),
|
||||
cancellationToken);
|
||||
if (snippet.Length == 0)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken);
|
||||
if (match is null)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, snippet));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mode == SpeakerIdentityProcessingMode.Final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees, snippet);
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var speakerName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(speakerName))
|
||||
{
|
||||
speakerMappings[speaker] = speakerName;
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
speakerName,
|
||||
GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (mode == SpeakerIdentityProcessingMode.Final)
|
||||
{
|
||||
await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
|
||||
var relabeledSegments = request.Segments
|
||||
.Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName)
|
||||
? segment with { Speaker = speakerName }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeledSegments, speakerMappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentityMatch?> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
string speaker,
|
||||
byte[] snippet,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var maxCandidates = Math.Max(1, options.MaxMatchCandidates);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.Aliases)
|
||||
.OrderByDescending(identity => identity.TranscriptionCount)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
identities = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.TranscriptionCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(maxCandidates)
|
||||
.Select(candidate => candidate.Identity)
|
||||
.ToList();
|
||||
|
||||
foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize)))
|
||||
{
|
||||
var request = new SpeakerIdentityMatchRequest(
|
||||
speaker,
|
||||
snippet,
|
||||
batch.Select(identity => new SpeakerIdentityMatchCandidate(
|
||||
identity.Id,
|
||||
identity.CanonicalName,
|
||||
identity.TranscriptionCount,
|
||||
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
|
||||
.ToList());
|
||||
var match = await matcher.MatchAsync(request, cancellationToken);
|
||||
if (match is not null)
|
||||
{
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
}
|
||||
|
||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
int identityId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.Snippets)
|
||||
.SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees,
|
||||
byte[] snippet)
|
||||
{
|
||||
identity.TranscriptionCount++;
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0)
|
||||
{
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (intersection.Count == 0)
|
||||
{
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
}
|
||||
}
|
||||
|
||||
AddSnippetIfNeeded(identity, snippet);
|
||||
}
|
||||
|
||||
private async Task LearnUnmatchedSpeakersAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IEnumerable<string> matchedCanonicalNames,
|
||||
IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var remainingCandidates = attendees
|
||||
.Except(matchedCanonicalNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (remainingCandidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = remainingCandidates
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate })
|
||||
.ToList(),
|
||||
Snippets =
|
||||
[
|
||||
new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = now
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
|
||||
{
|
||||
if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
identity.Snippets.Add(new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet)
|
||||
{
|
||||
var oldest = identity.Snippets.OrderBy(storedSnippet => storedSnippet.CreatedAt).FirstOrDefault();
|
||||
if (oldest is not null)
|
||||
{
|
||||
identity.Snippets.Remove(oldest);
|
||||
}
|
||||
|
||||
if (snippet.Length > 0)
|
||||
{
|
||||
identity.Snippets.Add(new SpeakerSnippet
|
||||
{
|
||||
WavBytes = snippet,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
|
||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
identity.CanonicalName
|
||||
}
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var trimmed = attendee.Trim();
|
||||
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
|
||||
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
|
||||
}
|
||||
|
||||
private enum SpeakerIdentityProcessingMode
|
||||
{
|
||||
LiveReadOnly,
|
||||
Final
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class WavSpeakerSnippetExtractor : ISpeakerSnippetExtractor
|
||||
{
|
||||
public Task<byte[]> ExtractSnippetAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (speakerSegments.Count == 0 || !File.Exists(audioPath))
|
||||
{
|
||||
return Task.FromResult<byte[]>([]);
|
||||
}
|
||||
|
||||
var start = speakerSegments.Min(segment => segment.Start);
|
||||
var end = speakerSegments.Max(segment => segment.End);
|
||||
if (end <= start)
|
||||
{
|
||||
return Task.FromResult<byte[]>([]);
|
||||
}
|
||||
|
||||
using var reader = new WaveFileReader(audioPath);
|
||||
using var output = new MemoryStream();
|
||||
using (var writer = new WaveFileWriter(output, reader.WaveFormat))
|
||||
{
|
||||
reader.CurrentTime = start;
|
||||
var bytesRemaining = reader.WaveFormat.AverageBytesPerSecond * (end - start).TotalSeconds;
|
||||
var buffer = new byte[reader.WaveFormat.AverageBytesPerSecond];
|
||||
while (bytesRemaining > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var bytesToRead = (int)Math.Min(buffer.Length, bytesRemaining);
|
||||
var bytesRead = reader.Read(buffer, 0, bytesToRead);
|
||||
if (bytesRead == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
writer.Write(buffer, 0, bytesRead);
|
||||
bytesRemaining -= bytesRead;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(output.ToArray());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user