Public Access
Implement meeting assistant v1
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class AsrDiagnosticService
|
||||
{
|
||||
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
|
||||
|
||||
public AsrDiagnosticService(ISpeechRecognitionPipelineFactory pipelineFactory)
|
||||
{
|
||||
this.pipelineFactory = pipelineFactory;
|
||||
}
|
||||
|
||||
public async Task<AsrDiagnosticResult> TranscribeWavAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
throw new ArgumentException("A WAV file path is required.", nameof(path));
|
||||
}
|
||||
|
||||
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
|
||||
}
|
||||
|
||||
await using var pipeline = pipelineFactory.Create();
|
||||
await pipeline.InitializeAsync(cancellationToken);
|
||||
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
|
||||
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
|
||||
await pipeline.CompleteAsync(cancellationToken);
|
||||
return new AsrDiagnosticResult(fullPath, await liveTranscriptTask);
|
||||
}
|
||||
|
||||
public async Task<AsrDiagnosticResult> DiarizeWavAsync(
|
||||
string path,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fullPath = ResolveExistingWavPath(path);
|
||||
await using var pipeline = pipelineFactory.Create();
|
||||
await pipeline.InitializeAsync(cancellationToken);
|
||||
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
|
||||
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
|
||||
await pipeline.CompleteAsync(cancellationToken);
|
||||
await liveTranscriptTask;
|
||||
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(fullPath, options, cancellationToken);
|
||||
|
||||
return new AsrDiagnosticResult(fullPath, finishedSegments);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<TranscriptionSegment>> CollectLiveTranscriptAsync(
|
||||
ISpeechRecognitionPipeline pipeline,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var segments = new List<TranscriptionSegment>();
|
||||
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
|
||||
{
|
||||
segments.Add(segment);
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
private static async Task WriteWavToPipelineAsync(
|
||||
string fullPath,
|
||||
ISpeechRecognitionPipeline pipeline,
|
||||
bool paceAudio,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken))
|
||||
{
|
||||
await pipeline.WriteAsync(chunk, cancellationToken);
|
||||
if (paceAudio && chunk.Duration > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(chunk.Duration, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ResolveExistingWavPath(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
throw new ArgumentException("A WAV file path is required.", nameof(path));
|
||||
}
|
||||
|
||||
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
|
||||
}
|
||||
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<AudioChunk> ReadPcmWavChunks(
|
||||
string path,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var reader = new WaveFileReader(path);
|
||||
var format = reader.WaveFormat;
|
||||
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
|
||||
}
|
||||
|
||||
var buffer = new byte[format.AverageBytesPerSecond];
|
||||
int read;
|
||||
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
|
||||
{
|
||||
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record AsrDiagnosticResult(string Path, IReadOnlyList<TranscriptionSegment> Segments);
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class AzureSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
|
||||
{
|
||||
public AzureSpeechRecognitionPipeline(AzureSpeechStreamingTranscriptionProvider transcriptionProvider)
|
||||
: base(transcriptionProvider)
|
||||
{
|
||||
}
|
||||
|
||||
internal AzureSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
|
||||
: base(transcriptionProvider)
|
||||
{
|
||||
}
|
||||
|
||||
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(liveSegments);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.CognitiveServices.Speech;
|
||||
using Microsoft.CognitiveServices.Speech.Audio;
|
||||
using Microsoft.CognitiveServices.Speech.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTranscriptionProvider
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<AzureSpeechStreamingTranscriptionProvider> logger;
|
||||
|
||||
public AzureSpeechStreamingTranscriptionProvider(
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<AzureSpeechStreamingTranscriptionProvider> logger)
|
||||
{
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
|
||||
IAsyncEnumerable<AudioChunk> audio,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
|
||||
if (!await enumerator.MoveNextAsync())
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var firstChunk = enumerator.Current;
|
||||
using var pushStream = AudioInputStream.CreatePushStream(
|
||||
AudioStreamFormat.GetWaveFormatPCM(
|
||||
(uint)firstChunk.SampleRate,
|
||||
16,
|
||||
(byte)firstChunk.Channels));
|
||||
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
|
||||
var speechConfig = CreateSpeechConfig();
|
||||
using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
|
||||
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
|
||||
recognizer.Transcribed += (_, args) =>
|
||||
{
|
||||
if (args.Result.Reason != ResultReason.RecognizedSpeech
|
||||
|| string.IsNullOrWhiteSpace(args.Result.Text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
|
||||
var segment = new TranscriptionSegment(
|
||||
start,
|
||||
start + args.Result.Duration,
|
||||
NormalizeSpeakerId(args.Result.SpeakerId),
|
||||
args.Result.Text.Trim());
|
||||
logger.LogInformation(
|
||||
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
|
||||
segment.Start,
|
||||
segment.Speaker,
|
||||
segment.Text);
|
||||
segments.Writer.TryWrite(segment);
|
||||
};
|
||||
recognizer.Canceled += (_, args) =>
|
||||
{
|
||||
if (args.Reason == CancellationReason.Error)
|
||||
{
|
||||
segments.Writer.TryComplete(new InvalidOperationException(
|
||||
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
|
||||
return;
|
||||
}
|
||||
|
||||
segments.Writer.TryComplete();
|
||||
};
|
||||
|
||||
await recognizer.StartTranscribingAsync();
|
||||
var pumpTask = Task.Run(
|
||||
() => PumpAudioAsync(
|
||||
firstChunk,
|
||||
enumerator,
|
||||
pushStream,
|
||||
recognizer,
|
||||
segments.Writer,
|
||||
options.AzureSpeech.RecognitionStopTimeout,
|
||||
cancellationToken),
|
||||
CancellationToken.None);
|
||||
|
||||
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
yield return segment;
|
||||
}
|
||||
|
||||
await pumpTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private SpeechConfig CreateSpeechConfig()
|
||||
{
|
||||
var azure = options.AzureSpeech;
|
||||
var key = ResolveKey(azure);
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
|
||||
}
|
||||
|
||||
var speechConfig = string.IsNullOrWhiteSpace(azure.Region)
|
||||
? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key)
|
||||
: SpeechConfig.FromSubscription(key, azure.Region);
|
||||
speechConfig.SpeechRecognitionLanguage = azure.Language;
|
||||
speechConfig.OutputFormat = OutputFormat.Detailed;
|
||||
speechConfig.SetProperty(
|
||||
PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
|
||||
azure.DiarizeIntermediateResults ? "true" : "false");
|
||||
return speechConfig;
|
||||
}
|
||||
|
||||
private static async Task PumpAudioAsync(
|
||||
AudioChunk firstChunk,
|
||||
IAsyncEnumerator<AudioChunk> audio,
|
||||
PushAudioInputStream pushStream,
|
||||
ConversationTranscriber recognizer,
|
||||
ChannelWriter<TranscriptionSegment> segments,
|
||||
TimeSpan recognitionStopTimeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
WriteChunk(pushStream, firstChunk, firstChunk);
|
||||
while (await audio.MoveNextAsync())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
WriteChunk(pushStream, firstChunk, audio.Current);
|
||||
}
|
||||
|
||||
pushStream.Close();
|
||||
try
|
||||
{
|
||||
var stopTask = recognizer.StopTranscribingAsync();
|
||||
if (recognitionStopTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await stopTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// The SDK can outlive short diagnostic streams while waiting for final service events.
|
||||
}
|
||||
|
||||
segments.TryComplete();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
segments.TryComplete(exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteChunk(PushAudioInputStream pushStream, AudioChunk expectedFormat, AudioChunk chunk)
|
||||
{
|
||||
if (chunk.SampleRate != expectedFormat.SampleRate || chunk.Channels != expectedFormat.Channels)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
pushStream.Write(chunk.Pcm);
|
||||
}
|
||||
|
||||
private static string NormalizeSpeakerId(string? speakerId)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase)
|
||||
? "Unknown"
|
||||
: speakerId;
|
||||
}
|
||||
|
||||
private static string ResolveKey(AzureSpeechOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Key))
|
||||
{
|
||||
return options.Key;
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(options.KeyEnv)
|
||||
? ""
|
||||
: Environment.GetEnvironmentVariable(options.KeyEnv) ?? "";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System.Net.WebSockets;
|
||||
using System.Text;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class ClientFunAsrWebSocketConnectionFactory : IFunAsrWebSocketConnectionFactory
|
||||
{
|
||||
public async Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken)
|
||||
{
|
||||
var socket = new ClientWebSocket();
|
||||
await socket.ConnectAsync(endpoint, cancellationToken);
|
||||
return new ClientFunAsrWebSocketConnection(socket);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ClientFunAsrWebSocketConnection : IFunAsrWebSocketConnection
|
||||
{
|
||||
private readonly ClientWebSocket socket;
|
||||
|
||||
public ClientFunAsrWebSocketConnection(ClientWebSocket socket)
|
||||
{
|
||||
this.socket = socket;
|
||||
}
|
||||
|
||||
public Task SendTextAsync(string message, CancellationToken cancellationToken)
|
||||
{
|
||||
return socket.SendAsync(
|
||||
Encoding.UTF8.GetBytes(message),
|
||||
WebSocketMessageType.Text,
|
||||
endOfMessage: true,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken)
|
||||
{
|
||||
await socket.SendAsync(message, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<string?> ReceiveTextAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var buffer = new byte[8192];
|
||||
using var message = new MemoryStream();
|
||||
|
||||
while (true)
|
||||
{
|
||||
var result = await socket.ReceiveAsync(buffer, cancellationToken);
|
||||
if (result.MessageType == WebSocketMessageType.Close)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (result.MessageType != WebSocketMessageType.Text)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
message.Write(buffer, 0, result.Count);
|
||||
if (result.EndOfMessage)
|
||||
{
|
||||
return Encoding.UTF8.GetString(message.ToArray());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CloseOutputAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (socket.State is WebSocketState.Open or WebSocketState.CloseReceived)
|
||||
{
|
||||
await socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "meeting assistant finished sending audio", cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
socket.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface ICommandRunner
|
||||
{
|
||||
Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null);
|
||||
}
|
||||
|
||||
public sealed class ProcessCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly ILogger<ProcessCommandRunner> logger;
|
||||
|
||||
public ProcessCommandRunner(ILogger<ProcessCommandRunner> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = fileName,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
|
||||
foreach (var argument in arguments)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
if (environment is not null)
|
||||
{
|
||||
foreach (var (key, value) in environment)
|
||||
{
|
||||
startInfo.Environment[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation("Running command {Command} {Arguments}", fileName, string.Join(' ', arguments));
|
||||
using var process = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException($"Failed to start command '{fileName}'.");
|
||||
var outputTask = ReadLinesAsync(process.StandardOutput, line => logger.LogInformation("{Command}: {Line}", fileName, line), cancellationToken);
|
||||
var errorTask = ReadLinesAsync(process.StandardError, line => logger.LogWarning("{Command}: {Line}", fileName, line), cancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
logger.LogWarning("Cancelling command {Command}; killing process tree.", fileName);
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
var result = new CommandResult(process.ExitCode, await outputTask, await errorTask);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Command {Command} exited with {ExitCode}: {Error}",
|
||||
fileName,
|
||||
result.ExitCode,
|
||||
result.StandardError);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static async Task<string> ReadLinesAsync(
|
||||
StreamReader reader,
|
||||
Action<string> log,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var output = new StringBuilder();
|
||||
while (await reader.ReadLineAsync(cancellationToken) is { } line)
|
||||
{
|
||||
output.AppendLine(line);
|
||||
log(line);
|
||||
}
|
||||
|
||||
return output.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record CommandResult(int ExitCode, string StandardOutput, string StandardError);
|
||||
@@ -0,0 +1,29 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class ConfiguredSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
|
||||
{
|
||||
private readonly IServiceProvider services;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
|
||||
public ConfiguredSpeechRecognitionPipelineFactory(
|
||||
IServiceProvider services,
|
||||
IOptions<MeetingAssistantOptions> options)
|
||||
{
|
||||
this.services = services;
|
||||
this.options = options.Value;
|
||||
}
|
||||
|
||||
public ISpeechRecognitionPipeline Create()
|
||||
{
|
||||
return options.Recording.TranscriptionProvider switch
|
||||
{
|
||||
"funasr" => ActivatorUtilities.CreateInstance<FunAsrSpeechRecognitionPipeline>(services),
|
||||
"whisper-local" => ActivatorUtilities.CreateInstance<WhisperLocalSpeechRecognitionPipeline>(services),
|
||||
"azure-speech" => ActivatorUtilities.CreateInstance<AzureSpeechRecognitionPipeline>(services),
|
||||
_ => throw new InvalidOperationException(
|
||||
$"Unsupported speech recognition pipeline '{options.Recording.TranscriptionProvider}'.")
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface IFunAsrBackendLifecycle
|
||||
{
|
||||
Task EnsureStartedAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class FunAsrDockerBackendLifecycle : IFunAsrBackendLifecycle, IAsyncDisposable
|
||||
{
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly IFunAsrBackendReadinessProbe readinessProbe;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<FunAsrDockerBackendLifecycle> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
private bool containerStarted;
|
||||
private bool started;
|
||||
private bool disposed;
|
||||
|
||||
public FunAsrDockerBackendLifecycle(
|
||||
ICommandRunner commandRunner,
|
||||
IFunAsrBackendReadinessProbe readinessProbe,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<FunAsrDockerBackendLifecycle> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.readinessProbe = readinessProbe;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task EnsureStartedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(disposed, this);
|
||||
var backend = options.FunAsr.Backend;
|
||||
if (!backend.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (started)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PrepareModelsFolder(backend);
|
||||
await RunRequiredAsync(backend.DockerCommand, ["pull", backend.Image], cancellationToken);
|
||||
await commandRunner.RunAsync(
|
||||
backend.DockerCommand,
|
||||
["rm", "-f", backend.ContainerName],
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(backend.DockerCommand, BuildDockerRunArguments(), cancellationToken);
|
||||
containerStarted = true;
|
||||
await readinessProbe.WaitUntilReadyAsync(
|
||||
new Uri(options.FunAsr.Endpoint),
|
||||
backend.StartupTimeout,
|
||||
cancellationToken);
|
||||
|
||||
started = true;
|
||||
logger.LogInformation(
|
||||
"Started managed FunASR backend container {ContainerName}; endpoint {Endpoint} is ready",
|
||||
backend.ContainerName,
|
||||
options.FunAsr.Endpoint);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var backend = options.FunAsr.Backend;
|
||||
if (!backend.Enabled || !backend.StopOnDispose)
|
||||
{
|
||||
disposed = true;
|
||||
gate.Dispose();
|
||||
return;
|
||||
}
|
||||
|
||||
await gate.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (containerStarted)
|
||||
{
|
||||
await commandRunner.RunAsync(
|
||||
backend.DockerCommand,
|
||||
["stop", backend.ContainerName],
|
||||
CancellationToken.None);
|
||||
containerStarted = false;
|
||||
started = false;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
disposed = true;
|
||||
gate.Release();
|
||||
gate.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunRequiredAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var backend = options.FunAsr.Backend;
|
||||
using var timeoutSource = backend.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(backend.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
var effectiveToken = linkedSource?.Token ?? cancellationToken;
|
||||
CommandResult result;
|
||||
try
|
||||
{
|
||||
result = await commandRunner.RunAsync(fileName, arguments, effectiveToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Command '{fileName} {string.Join(' ', arguments)}' timed out after {backend.CommandTimeout}.");
|
||||
}
|
||||
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Command '{fileName} {string.Join(' ', arguments)}' failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
}
|
||||
|
||||
private string[] BuildDockerRunArguments()
|
||||
{
|
||||
var backend = options.FunAsr.Backend;
|
||||
List<string> arguments =
|
||||
[
|
||||
"run",
|
||||
"--rm",
|
||||
"-d",
|
||||
];
|
||||
|
||||
if (backend.Privileged)
|
||||
{
|
||||
arguments.Add("--privileged=true");
|
||||
}
|
||||
|
||||
arguments.AddRange(
|
||||
[
|
||||
"--name",
|
||||
backend.ContainerName,
|
||||
"-p",
|
||||
$"{GetEndpointPort()}:{backend.ContainerPort}",
|
||||
"-v",
|
||||
$"{VaultPath.Resolve(backend.ModelsFolder)}:/workspace/models",
|
||||
backend.Image,
|
||||
"bash",
|
||||
"-lc",
|
||||
backend.ServerCommand
|
||||
]);
|
||||
|
||||
return [.. arguments];
|
||||
}
|
||||
|
||||
private static void PrepareModelsFolder(FunAsrBackendOptions backend)
|
||||
{
|
||||
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
|
||||
Directory.CreateDirectory(modelsFolder);
|
||||
|
||||
var hotwordsPath = Path.Combine(modelsFolder, "hotwords.txt");
|
||||
if (!File.Exists(hotwordsPath))
|
||||
{
|
||||
File.WriteAllText(hotwordsPath, "");
|
||||
}
|
||||
}
|
||||
|
||||
private int GetEndpointPort()
|
||||
{
|
||||
return new Uri(options.FunAsr.Endpoint).Port;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class FunAsrMessageParser
|
||||
{
|
||||
private readonly FunAsrOptions options;
|
||||
|
||||
public FunAsrMessageParser(FunAsrOptions options)
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public FunAsrParsedMessage Parse(string message)
|
||||
{
|
||||
using var document = JsonDocument.Parse(message);
|
||||
var root = document.RootElement;
|
||||
var mode = GetString(root, "mode");
|
||||
var isFinal = GetBoolean(root, "is_final");
|
||||
var shouldEmit = options.EmitOnlinePartials || !IsOnlineMode(mode);
|
||||
var segments = shouldEmit ? ReadSegments(root) : [];
|
||||
|
||||
return new FunAsrParsedMessage(IsTerminal: isFinal && !IsOnlineMode(mode), segments);
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> ReadSegments(JsonElement root)
|
||||
{
|
||||
var sentenceSegments = ReadSentenceSegments(root, "sentence_info");
|
||||
if (sentenceSegments.Count > 0)
|
||||
{
|
||||
return sentenceSegments;
|
||||
}
|
||||
|
||||
var stampSegments = ReadSentenceSegments(root, "stamp_sents");
|
||||
if (stampSegments.Count > 0)
|
||||
{
|
||||
return stampSegments;
|
||||
}
|
||||
|
||||
var text = GetString(root, "text").Trim();
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var (start, end) = ReadTimestampRange(root);
|
||||
return [new TranscriptionSegment(start, end, ReadSpeaker(root), text)];
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> ReadSentenceSegments(JsonElement root, string propertyName)
|
||||
{
|
||||
if (!root.TryGetProperty(propertyName, out var sentences) || sentences.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var segments = new List<TranscriptionSegment>();
|
||||
foreach (var sentence in sentences.EnumerateArray())
|
||||
{
|
||||
var text = (GetString(sentence, "text") + GetString(sentence, "text_seg") + GetString(sentence, "punc")).Trim();
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
segments.Add(new TranscriptionSegment(
|
||||
TimeSpan.FromMilliseconds(GetDouble(sentence, "start")),
|
||||
TimeSpan.FromMilliseconds(GetDouble(sentence, "end")),
|
||||
ReadSpeaker(sentence),
|
||||
text));
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
private static (TimeSpan Start, TimeSpan End) ReadTimestampRange(JsonElement root)
|
||||
{
|
||||
if (!root.TryGetProperty("timestamp", out var timestamp))
|
||||
{
|
||||
return (TimeSpan.Zero, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
if (timestamp.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var value = timestamp.GetString();
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return (TimeSpan.Zero, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(value);
|
||||
return ReadTimestampArray(document.RootElement);
|
||||
}
|
||||
|
||||
return ReadTimestampArray(timestamp);
|
||||
}
|
||||
|
||||
private static (TimeSpan Start, TimeSpan End) ReadTimestampArray(JsonElement timestamp)
|
||||
{
|
||||
if (timestamp.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return (TimeSpan.Zero, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
double? start = null;
|
||||
double? end = null;
|
||||
foreach (var item in timestamp.EnumerateArray())
|
||||
{
|
||||
if (item.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var values = item.EnumerateArray().ToArray();
|
||||
if (values.Length < 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
start ??= ReadNumber(values[0]);
|
||||
end = ReadNumber(values[1]);
|
||||
}
|
||||
|
||||
return (
|
||||
TimeSpan.FromMilliseconds(start ?? 0),
|
||||
TimeSpan.FromMilliseconds(end ?? start ?? 0));
|
||||
}
|
||||
|
||||
private static string ReadSpeaker(JsonElement element)
|
||||
{
|
||||
var speaker = GetString(element, "spk_name");
|
||||
if (!string.IsNullOrWhiteSpace(speaker))
|
||||
{
|
||||
return speaker;
|
||||
}
|
||||
|
||||
speaker = GetString(element, "speaker");
|
||||
if (!string.IsNullOrWhiteSpace(speaker))
|
||||
{
|
||||
return speaker;
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("spk", out var spk))
|
||||
{
|
||||
return $"Speaker {spk}";
|
||||
}
|
||||
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
private static bool IsOnlineMode(string mode)
|
||||
{
|
||||
return mode.Equals("online", StringComparison.OrdinalIgnoreCase)
|
||||
|| mode.Equals("2pass-online", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string GetString(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
|
||||
? property.GetString() ?? ""
|
||||
: "";
|
||||
}
|
||||
|
||||
private static bool GetBoolean(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind is JsonValueKind.True or JsonValueKind.False
|
||||
&& property.GetBoolean();
|
||||
}
|
||||
|
||||
private static double GetDouble(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property) ? ReadNumber(property) : 0;
|
||||
}
|
||||
|
||||
private static double ReadNumber(JsonElement element)
|
||||
{
|
||||
return element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record FunAsrParsedMessage(bool IsTerminal, IReadOnlyList<TranscriptionSegment> Segments);
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class FunAsrSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
|
||||
{
|
||||
private readonly IFunAsrBackendLifecycle backendLifecycle;
|
||||
private readonly FunAsrTranscriptFinalizer transcriptFinalizer;
|
||||
|
||||
public FunAsrSpeechRecognitionPipeline(
|
||||
FunAsrStreamingTranscriptionProvider transcriptionProvider,
|
||||
IFunAsrBackendLifecycle backendLifecycle,
|
||||
FunAsrTranscriptFinalizer transcriptFinalizer)
|
||||
: base(transcriptionProvider)
|
||||
{
|
||||
this.backendLifecycle = backendLifecycle;
|
||||
this.transcriptFinalizer = transcriptFinalizer;
|
||||
}
|
||||
|
||||
public override Task WaitUntilReadyAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return backendLifecycle.EnsureStartedAsync(cancellationToken);
|
||||
}
|
||||
|
||||
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
using System.Text.Json;
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscriptionProvider, IAsyncDisposable
|
||||
{
|
||||
private readonly IFunAsrWebSocketConnectionFactory connectionFactory;
|
||||
private readonly IFunAsrBackendLifecycle backendLifecycle;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<FunAsrStreamingTranscriptionProvider> logger;
|
||||
|
||||
public FunAsrStreamingTranscriptionProvider(
|
||||
IFunAsrWebSocketConnectionFactory connectionFactory,
|
||||
IFunAsrBackendLifecycle backendLifecycle,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<FunAsrStreamingTranscriptionProvider> logger)
|
||||
{
|
||||
this.connectionFactory = connectionFactory;
|
||||
this.backendLifecycle = backendLifecycle;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
|
||||
IAsyncEnumerable<AudioChunk> audio,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None);
|
||||
|
||||
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
yield return segment;
|
||||
}
|
||||
|
||||
await session;
|
||||
}
|
||||
|
||||
private async Task RunSessionAsync(
|
||||
IAsyncEnumerable<AudioChunk> audio,
|
||||
ChannelWriter<TranscriptionSegment> segments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await backendLifecycle.EnsureStartedAsync(cancellationToken);
|
||||
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
|
||||
if (!await enumerator.MoveNextAsync())
|
||||
{
|
||||
segments.TryComplete();
|
||||
return;
|
||||
}
|
||||
|
||||
var firstChunk = enumerator.Current;
|
||||
var endpoint = new Uri(options.FunAsr.Endpoint);
|
||||
await using var connection = await connectionFactory.ConnectAsync(endpoint, cancellationToken);
|
||||
var parser = new FunAsrMessageParser(options.FunAsr);
|
||||
var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken);
|
||||
|
||||
await SendStartAsync(connection, firstChunk, cancellationToken);
|
||||
await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken);
|
||||
|
||||
while (await enumerator.MoveNextAsync())
|
||||
{
|
||||
await connection.SendBinaryAsync(enumerator.Current.Pcm, cancellationToken);
|
||||
}
|
||||
|
||||
await connection.SendTextAsync("""{"is_speaking":false}""", cancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
await finalReceived.Task.WaitAsync(options.FunAsr.FinalResultTimeout, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Timed out waiting {Timeout} for the final FunASR result from {Endpoint}",
|
||||
options.FunAsr.FinalResultTimeout,
|
||||
endpoint);
|
||||
}
|
||||
|
||||
await connection.CloseOutputAsync(cancellationToken);
|
||||
await receiveTask;
|
||||
segments.TryComplete();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
segments.TryComplete(exception);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (backendLifecycle is IAsyncDisposable disposable)
|
||||
{
|
||||
await disposable.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveAsync(
|
||||
IFunAsrWebSocketConnection connection,
|
||||
FunAsrMessageParser parser,
|
||||
ChannelWriter<TranscriptionSegment> segments,
|
||||
TaskCompletionSource finalReceived,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var message = await connection.ReceiveTextAsync(cancellationToken);
|
||||
if (message is null)
|
||||
{
|
||||
finalReceived.TrySetResult();
|
||||
return;
|
||||
}
|
||||
|
||||
var parsed = parser.Parse(message);
|
||||
foreach (var segment in parsed.Segments)
|
||||
{
|
||||
await segments.WriteAsync(segment, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"FunASR emitted segment at {Start} for {Speaker}: {Text}",
|
||||
segment.Start,
|
||||
segment.Speaker,
|
||||
segment.Text);
|
||||
}
|
||||
|
||||
if (parsed.IsTerminal)
|
||||
{
|
||||
finalReceived.TrySetResult();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendStartAsync(
|
||||
IFunAsrWebSocketConnection connection,
|
||||
AudioChunk firstChunk,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var message = new Dictionary<string, object?>
|
||||
{
|
||||
["mode"] = options.FunAsr.Mode,
|
||||
["chunk_size"] = NormalizeChunkSize(options.FunAsr.ChunkSize),
|
||||
["chunk_interval"] = options.FunAsr.ChunkInterval,
|
||||
["encoder_chunk_look_back"] = options.FunAsr.EncoderChunkLookBack,
|
||||
["decoder_chunk_look_back"] = options.FunAsr.DecoderChunkLookBack,
|
||||
["audio_fs"] = firstChunk.SampleRate,
|
||||
["wav_name"] = options.FunAsr.WavName,
|
||||
["wav_format"] = "pcm",
|
||||
["is_speaking"] = true,
|
||||
["hotwords"] = await LoadHotwordsAsync(cancellationToken),
|
||||
["itn"] = options.FunAsr.Itn
|
||||
};
|
||||
|
||||
await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<string> LoadHotwordsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var path = VaultPath.Resolve(options.Vault.DictationWordsPath);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var words = await File.ReadAllLinesAsync(path, cancellationToken);
|
||||
var hotwords = words
|
||||
.Select(line => line.Trim().TrimStart('-', '*').Trim())
|
||||
.Where(line => !string.IsNullOrWhiteSpace(line))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(word => word, _ => 20);
|
||||
|
||||
return hotwords.Count == 0 ? "" : JsonSerializer.Serialize(hotwords);
|
||||
}
|
||||
|
||||
private static int[] NormalizeChunkSize(int[] configuredChunkSize)
|
||||
{
|
||||
return configuredChunkSize.Length == 3 ? configuredChunkSize : [5, 10, 5];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class FunAsrTranscriptFinalizer
|
||||
{
|
||||
private const string JsonStart = "__MEETING_ASSISTANT_DIARIZATION_JSON_START__";
|
||||
private const string JsonEnd = "__MEETING_ASSISTANT_DIARIZATION_JSON_END__";
|
||||
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<FunAsrTranscriptFinalizer> logger;
|
||||
|
||||
public FunAsrTranscriptFinalizer(
|
||||
ICommandRunner commandRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<FunAsrTranscriptFinalizer> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var diarization = options.FunAsr.Diarization;
|
||||
if (!diarization.Enabled)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
|
||||
if (!File.Exists(fullAudioPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
|
||||
}
|
||||
|
||||
var result = await RunDiarizationAsync(fullAudioPath, cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"FunASR diarization failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
|
||||
var json = ExtractJson(result.StandardOutput);
|
||||
var segments = ParseSegments(json);
|
||||
logger.LogInformation("FunASR final diarization produced {SegmentCount} sentence segments", segments.Count);
|
||||
return segments;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunDiarizationAsync(
|
||||
string fullAudioPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var backend = options.FunAsr.Backend;
|
||||
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
|
||||
Directory.CreateDirectory(modelsFolder);
|
||||
using var timeoutSource = options.FunAsr.Diarization.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(options.FunAsr.Diarization.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
|
||||
try
|
||||
{
|
||||
return await commandRunner.RunAsync(
|
||||
backend.DockerCommand,
|
||||
BuildDockerArguments(fullAudioPath, modelsFolder),
|
||||
linkedSource?.Token ?? cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"FunASR diarization timed out after {options.FunAsr.Diarization.CommandTimeout}.");
|
||||
}
|
||||
}
|
||||
|
||||
private string[] BuildDockerArguments(string fullAudioPath, string modelsFolder)
|
||||
{
|
||||
var backend = options.FunAsr.Backend;
|
||||
return
|
||||
[
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
$"{fullAudioPath}:/workspace/input.wav:ro",
|
||||
"-v",
|
||||
$"{modelsFolder}:/workspace/models",
|
||||
backend.Image,
|
||||
"bash",
|
||||
"-lc",
|
||||
BuildPythonCommand()
|
||||
];
|
||||
}
|
||||
|
||||
private string BuildPythonCommand()
|
||||
{
|
||||
var diarization = options.FunAsr.Diarization;
|
||||
var presetSpeakerCount = diarization.PresetSpeakerCount is int value
|
||||
? $", preset_spk_num={value}"
|
||||
: "";
|
||||
return
|
||||
"cat > /tmp/meeting_assistant_diarize.py <<'PY'\n"
|
||||
+ "import json\n"
|
||||
+ "from funasr import AutoModel\n"
|
||||
+ "model = AutoModel(\n"
|
||||
+ $" model={JsonSerializer.Serialize(diarization.AsrModel)},\n"
|
||||
+ $" vad_model={JsonSerializer.Serialize(diarization.VadModel)},\n"
|
||||
+ $" punc_model={JsonSerializer.Serialize(diarization.PunctuationModel)},\n"
|
||||
+ $" spk_model={JsonSerializer.Serialize(diarization.SpeakerModel)},\n"
|
||||
+ " device='cpu',\n"
|
||||
+ " disable_update=True,\n"
|
||||
+ ")\n"
|
||||
+ "res = model.generate(\n"
|
||||
+ " input='/workspace/input.wav',\n"
|
||||
+ $" batch_size_s={diarization.BatchSizeSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)},\n"
|
||||
+ $" batch_size_threshold_s={diarization.BatchSizeThresholdSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)}"
|
||||
+ presetSpeakerCount
|
||||
+ "\n)\n"
|
||||
+ $"print('{JsonStart}')\n"
|
||||
+ "print(json.dumps(res, ensure_ascii=False, default=str))\n"
|
||||
+ $"print('{JsonEnd}')\n"
|
||||
+ "PY\n"
|
||||
+ "MODELSCOPE_CACHE=/workspace/models/modelscope python /tmp/meeting_assistant_diarize.py";
|
||||
}
|
||||
|
||||
private static string ExtractJson(string output)
|
||||
{
|
||||
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||
if (start < 0)
|
||||
{
|
||||
throw new InvalidOperationException("FunASR diarization output did not contain the JSON start marker.");
|
||||
}
|
||||
|
||||
start += JsonStart.Length;
|
||||
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||
if (end < 0)
|
||||
{
|
||||
throw new InvalidOperationException("FunASR diarization output did not contain the JSON end marker.");
|
||||
}
|
||||
|
||||
return output[start..end].Trim();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<TranscriptionSegment> ParseSegments(string json)
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var segments = new List<TranscriptionSegment>();
|
||||
|
||||
foreach (var result in document.RootElement.EnumerateArray())
|
||||
{
|
||||
if (!result.TryGetProperty("sentence_info", out var sentences)
|
||||
|| sentences.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var sentence in sentences.EnumerateArray())
|
||||
{
|
||||
var text = ReadString(sentence, "text").Trim();
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
segments.Add(new TranscriptionSegment(
|
||||
TimeSpan.FromMilliseconds(ReadDouble(sentence, "start")),
|
||||
TimeSpan.FromMilliseconds(ReadDouble(sentence, "end")),
|
||||
$"Speaker {ReadInt(sentence, "spk")}",
|
||||
text));
|
||||
}
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
private static string ReadString(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind == JsonValueKind.String
|
||||
? property.GetString() ?? ""
|
||||
: "";
|
||||
}
|
||||
|
||||
private static double ReadDouble(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind == JsonValueKind.Number
|
||||
&& property.TryGetDouble(out var value)
|
||||
? value
|
||||
: 0;
|
||||
}
|
||||
|
||||
private static int ReadInt(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind == JsonValueKind.Number
|
||||
&& property.TryGetInt32(out var value)
|
||||
? value
|
||||
: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Net.WebSockets;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface IFunAsrBackendReadinessProbe
|
||||
{
|
||||
Task WaitUntilReadyAsync(Uri endpoint, TimeSpan timeout, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class FunAsrWebSocketBackendReadinessProbe : IFunAsrBackendReadinessProbe
|
||||
{
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(1);
|
||||
|
||||
public async Task WaitUntilReadyAsync(
|
||||
Uri endpoint,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (timeout <= TimeSpan.Zero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var timeoutSource = new CancellationTokenSource(timeout);
|
||||
using var linkedSource = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
cancellationToken,
|
||||
timeoutSource.Token);
|
||||
|
||||
while (!linkedSource.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var client = new ClientWebSocket();
|
||||
await client.ConnectAsync(endpoint, linkedSource.Token).ConfigureAwait(false);
|
||||
await CloseQuietlyAsync(client, linkedSource.Token).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception) when (IsTransientReadinessFailure(exception))
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(RetryDelay, linkedSource.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new TimeoutException(
|
||||
$"Timed out waiting {timeout} for managed FunASR backend WebSocket readiness at {endpoint}.");
|
||||
}
|
||||
|
||||
private static async Task CloseQuietlyAsync(ClientWebSocket client, CancellationToken cancellationToken)
|
||||
{
|
||||
if (client.State != WebSocketState.Open)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await client.CloseOutputAsync(
|
||||
WebSocketCloseStatus.NormalClosure,
|
||||
"readiness probe complete",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (WebSocketException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransientReadinessFailure(Exception exception)
|
||||
{
|
||||
return exception is WebSocketException
|
||||
or HttpRequestException
|
||||
or IOException;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface IFunAsrWebSocketConnectionFactory
|
||||
{
|
||||
Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IFunAsrWebSocketConnection : IAsyncDisposable
|
||||
{
|
||||
Task SendTextAsync(string message, CancellationToken cancellationToken);
|
||||
|
||||
Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken);
|
||||
|
||||
Task<string?> ReceiveTextAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task CloseOutputAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface ISpeechRecognitionPipeline : IAsyncDisposable
|
||||
{
|
||||
Task InitializeAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task WaitUntilReadyAsync(CancellationToken cancellationToken);
|
||||
|
||||
ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken);
|
||||
|
||||
Task CompleteAsync(CancellationToken cancellationToken);
|
||||
|
||||
IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface ISpeechRecognitionPipelineFactory
|
||||
{
|
||||
ISpeechRecognitionPipeline Create();
|
||||
}
|
||||
|
||||
public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null)
|
||||
{
|
||||
public static SpeechRecognitionPipelineOptions Default { get; } = new();
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public interface IStreamingTranscriptionProvider
|
||||
{
|
||||
IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
|
||||
IAsyncEnumerable<AudioChunk> audio,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class PyannoteTranscriptFinalizer
|
||||
{
|
||||
private const string JsonStart = "__MEETING_ASSISTANT_PYANNOTE_JSON_START__";
|
||||
private const string JsonEnd = "__MEETING_ASSISTANT_PYANNOTE_JSON_END__";
|
||||
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<PyannoteTranscriptFinalizer> logger;
|
||||
|
||||
public PyannoteTranscriptFinalizer(
|
||||
ICommandRunner commandRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<PyannoteTranscriptFinalizer> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await FinalizeAsync(
|
||||
audioPath,
|
||||
liveSegments,
|
||||
options.WhisperLocal.Diarization,
|
||||
pipelineOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
PyannoteDiarizationOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!diarization.Enabled || liveSegments.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
|
||||
if (!File.Exists(fullAudioPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
|
||||
}
|
||||
|
||||
var token = ResolveToken(diarization);
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Pyannote diarization is enabled but no Hugging Face token is configured directly or via {TokenEnv}",
|
||||
diarization.TokenEnv);
|
||||
return [];
|
||||
}
|
||||
|
||||
var result = await RunDiarizationAsync(fullAudioPath, token, diarization, pipelineOptions, cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"pyannote diarization failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
|
||||
var turns = ParseTurns(ExtractJson(result.StandardOutput));
|
||||
var segments = diarization.AlignmentMode == PyannoteAlignmentMode.PyannoteTurns
|
||||
? BuildTurnSegments(liveSegments, turns)
|
||||
: AssignSpeakers(liveSegments, turns);
|
||||
logger.LogInformation(
|
||||
"pyannote diarization produced {TurnCount} speaker turns and built {SegmentCount} transcript segments using {AlignmentMode}",
|
||||
turns.Count,
|
||||
segments.Count,
|
||||
diarization.AlignmentMode);
|
||||
return segments;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunDiarizationAsync(
|
||||
string fullAudioPath,
|
||||
string token,
|
||||
PyannoteDiarizationOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var modelsFolder = VaultPath.Resolve(diarization.ModelsFolder);
|
||||
Directory.CreateDirectory(modelsFolder);
|
||||
using var timeoutSource = diarization.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(diarization.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
|
||||
try
|
||||
{
|
||||
if (diarization.BuildImage)
|
||||
{
|
||||
await EnsureDockerImageAsync(diarization, modelsFolder, linkedSource?.Token ?? cancellationToken);
|
||||
}
|
||||
|
||||
return await commandRunner.RunAsync(
|
||||
diarization.DockerCommand,
|
||||
BuildDockerArguments(diarization, fullAudioPath, modelsFolder, pipelineOptions),
|
||||
linkedSource?.Token ?? cancellationToken,
|
||||
new Dictionary<string, string> { ["HF_TOKEN"] = token });
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"pyannote diarization timed out after {diarization.CommandTimeout}.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EnsureDockerImageAsync(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
string modelsFolder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var inspectResult = await commandRunner.RunAsync(
|
||||
diarization.DockerCommand,
|
||||
["image", "inspect", "--format", "{{.Id}}", diarization.Image],
|
||||
cancellationToken);
|
||||
if (inspectResult.ExitCode == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var dockerfilePath = Path.Combine(modelsFolder, "docker", $"{SanitizeFileName(diarization.Image)}.Dockerfile");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(dockerfilePath)!);
|
||||
await File.WriteAllTextAsync(dockerfilePath, BuildDockerfile(diarization), cancellationToken);
|
||||
var result = await commandRunner.RunAsync(
|
||||
diarization.DockerCommand,
|
||||
["build", "-t", diarization.Image, "-f", dockerfilePath, Path.GetDirectoryName(dockerfilePath)!],
|
||||
cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"pyannote Docker image build failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
|
||||
{
|
||||
return
|
||||
$"FROM {diarization.BaseImage}\n"
|
||||
+ "ENV DEBIAN_FRONTEND=noninteractive\n"
|
||||
+ "RUN apt-get update \\\n"
|
||||
+ " && apt-get install -y --no-install-recommends ffmpeg libsndfile1 \\\n"
|
||||
+ " && rm -rf /var/lib/apt/lists/*\n"
|
||||
+ "RUN python -m pip install --upgrade pip \\\n"
|
||||
+ " && python -m pip install pyannote.audio soundfile\n";
|
||||
}
|
||||
|
||||
private string[] BuildDockerArguments(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
string fullAudioPath,
|
||||
string modelsFolder,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||
{
|
||||
return
|
||||
[
|
||||
"run",
|
||||
"--rm",
|
||||
"-e",
|
||||
"HF_TOKEN",
|
||||
"-v",
|
||||
$"{fullAudioPath}:/workspace/input.wav:ro",
|
||||
"-v",
|
||||
$"{modelsFolder}:/workspace/cache",
|
||||
"-e",
|
||||
"HF_HOME=/workspace/cache/huggingface",
|
||||
"-e",
|
||||
"XDG_CACHE_HOME=/workspace/cache",
|
||||
"-e",
|
||||
"PIP_CACHE_DIR=/workspace/cache/pip",
|
||||
"-e",
|
||||
"TORCH_HOME=/workspace/cache/torch",
|
||||
diarization.Image,
|
||||
"sh",
|
||||
"-lc",
|
||||
BuildPythonCommand(diarization, pipelineOptions)
|
||||
];
|
||||
}
|
||||
|
||||
private static string BuildPythonCommand(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||
{
|
||||
var model = diarization.Model;
|
||||
var annotationProperty = diarization.AnnotationSource == PyannoteAnnotationSource.ExclusiveSpeakerDiarization
|
||||
? "exclusive_speaker_diarization"
|
||||
: "speaker_diarization";
|
||||
return
|
||||
"cat > /tmp/meeting_assistant_pyannote.py <<'PY'\n"
|
||||
+ "import json\n"
|
||||
+ "import os\n"
|
||||
+ "import torch\n"
|
||||
+ "from pyannote.audio import Pipeline\n"
|
||||
+ $"pipeline = Pipeline.from_pretrained({JsonSerializer.Serialize(model)}, token=os.environ.get('HF_TOKEN'))\n"
|
||||
+ "pipeline.to(torch.device('cpu'))\n"
|
||||
+ BuildPipelineInvocation(pipelineOptions)
|
||||
+ $"diarization = getattr(result, {JsonSerializer.Serialize(annotationProperty)}, result)\n"
|
||||
+ "turns = []\n"
|
||||
+ "for turn, _, speaker in diarization.itertracks(yield_label=True):\n"
|
||||
+ " turns.append({'start': float(turn.start), 'end': float(turn.end), 'speaker': str(speaker)})\n"
|
||||
+ $"print('{JsonStart}')\n"
|
||||
+ "print(json.dumps(turns, ensure_ascii=False))\n"
|
||||
+ $"print('{JsonEnd}')\n"
|
||||
+ "PY\n"
|
||||
+ "python /tmp/meeting_assistant_pyannote.py";
|
||||
}
|
||||
|
||||
private static string BuildPipelineInvocation(SpeechRecognitionPipelineOptions pipelineOptions)
|
||||
{
|
||||
return pipelineOptions.NumSpeakers is > 0
|
||||
? $"result = pipeline('/workspace/input.wav', num_speakers={pipelineOptions.NumSpeakers.Value})\n"
|
||||
: "result = pipeline('/workspace/input.wav')\n";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<TranscriptionSegment> BuildTurnSegments(
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
IReadOnlyList<SpeakerTurn> turns)
|
||||
{
|
||||
if (turns.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var segments = new List<TranscriptionSegment>();
|
||||
foreach (var turn in turns)
|
||||
{
|
||||
var text = BuildTurnText(liveSegments, turn);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
segments.Add(new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(turn.Start),
|
||||
TimeSpan.FromSeconds(turn.End),
|
||||
turn.Speaker,
|
||||
text));
|
||||
}
|
||||
|
||||
return segments.Count == 0
|
||||
? AssignSpeakers(liveSegments, turns)
|
||||
: segments;
|
||||
}
|
||||
|
||||
private static string BuildTurnText(IReadOnlyList<TranscriptionSegment> liveSegments, SpeakerTurn turn)
|
||||
{
|
||||
var parts = new List<string>();
|
||||
foreach (var segment in liveSegments)
|
||||
{
|
||||
var segmentStart = segment.Start.TotalSeconds;
|
||||
var segmentEnd = segment.End > segment.Start
|
||||
? segment.End.TotalSeconds
|
||||
: segmentStart;
|
||||
var overlapStart = Math.Max(segmentStart, turn.Start);
|
||||
var overlapEnd = Math.Min(segmentEnd, turn.End);
|
||||
if (overlapEnd <= overlapStart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var part = SliceTextByTime(segment.Text, segmentStart, segmentEnd, overlapStart, overlapEnd);
|
||||
if (!string.IsNullOrWhiteSpace(part))
|
||||
{
|
||||
parts.Add(part);
|
||||
}
|
||||
}
|
||||
|
||||
return string.Join(' ', parts);
|
||||
}
|
||||
|
||||
private static string SliceTextByTime(
|
||||
string text,
|
||||
double segmentStart,
|
||||
double segmentEnd,
|
||||
double overlapStart,
|
||||
double overlapEnd)
|
||||
{
|
||||
var words = text
|
||||
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (words.Length == 0)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var duration = Math.Max(0.001, segmentEnd - segmentStart);
|
||||
var startFraction = Math.Clamp((overlapStart - segmentStart) / duration, 0, 1);
|
||||
var endFraction = Math.Clamp((overlapEnd - segmentStart) / duration, 0, 1);
|
||||
var startIndex = Math.Clamp((int)Math.Floor(startFraction * words.Length), 0, words.Length - 1);
|
||||
var endIndex = Math.Clamp((int)Math.Ceiling(endFraction * words.Length), startIndex + 1, words.Length);
|
||||
return string.Join(' ', words[startIndex..endIndex]);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<TranscriptionSegment> AssignSpeakers(
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
IReadOnlyList<SpeakerTurn> turns)
|
||||
{
|
||||
if (turns.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return liveSegments
|
||||
.Select(segment =>
|
||||
{
|
||||
var speaker = FindBestSpeaker(segment, turns) ?? segment.Speaker;
|
||||
return segment with { Speaker = speaker };
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? FindBestSpeaker(TranscriptionSegment segment, IReadOnlyList<SpeakerTurn> turns)
|
||||
{
|
||||
var segmentStart = segment.Start.TotalSeconds;
|
||||
var segmentEnd = segment.End > segment.Start
|
||||
? segment.End.TotalSeconds
|
||||
: segmentStart;
|
||||
var bestOverlap = 0d;
|
||||
string? bestSpeaker = null;
|
||||
|
||||
foreach (var turn in turns)
|
||||
{
|
||||
var overlap = Math.Min(segmentEnd, turn.End) - Math.Max(segmentStart, turn.Start);
|
||||
if (overlap > bestOverlap)
|
||||
{
|
||||
bestOverlap = overlap;
|
||||
bestSpeaker = turn.Speaker;
|
||||
}
|
||||
}
|
||||
|
||||
return bestSpeaker;
|
||||
}
|
||||
|
||||
private static string ExtractJson(string output)
|
||||
{
|
||||
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||
if (start < 0)
|
||||
{
|
||||
throw new InvalidOperationException("pyannote output did not contain the JSON start marker.");
|
||||
}
|
||||
|
||||
start += JsonStart.Length;
|
||||
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||
if (end < 0)
|
||||
{
|
||||
throw new InvalidOperationException("pyannote output did not contain the JSON end marker.");
|
||||
}
|
||||
|
||||
return output[start..end].Trim();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<SpeakerTurn> ParseTurns(string json)
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var turns = new List<SpeakerTurn>();
|
||||
foreach (var item in document.RootElement.EnumerateArray())
|
||||
{
|
||||
var speaker = ReadString(item, "speaker");
|
||||
if (string.IsNullOrWhiteSpace(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
turns.Add(new SpeakerTurn(
|
||||
ReadDouble(item, "start"),
|
||||
ReadDouble(item, "end"),
|
||||
speaker));
|
||||
}
|
||||
|
||||
return turns;
|
||||
}
|
||||
|
||||
private static string ResolveToken(PyannoteDiarizationOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Token))
|
||||
{
|
||||
return options.Token;
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(options.TokenEnv)
|
||||
? ""
|
||||
: Environment.GetEnvironmentVariable(options.TokenEnv) ?? "";
|
||||
}
|
||||
|
||||
private static string ReadString(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind == JsonValueKind.String
|
||||
? property.GetString() ?? ""
|
||||
: "";
|
||||
}
|
||||
|
||||
private static double ReadDouble(JsonElement element, string propertyName)
|
||||
{
|
||||
return element.TryGetProperty(propertyName, out var property)
|
||||
&& property.ValueKind == JsonValueKind.Number
|
||||
&& property.TryGetDouble(out var value)
|
||||
? value
|
||||
: 0;
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
return string.Concat(value.Select(character => invalid.Contains(character) ? '_' : character));
|
||||
}
|
||||
|
||||
private sealed record SpeakerTurn(double Start, double End, string Speaker);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class SpeechRecognitionPipelineHostedService : IHostedService
|
||||
{
|
||||
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
|
||||
private readonly ILogger<SpeechRecognitionPipelineHostedService> logger;
|
||||
private CancellationTokenSource? startupCancellation;
|
||||
private Task? startupTask;
|
||||
private ISpeechRecognitionPipeline? startupPipeline;
|
||||
|
||||
public SpeechRecognitionPipelineHostedService(
|
||||
ISpeechRecognitionPipelineFactory pipelineFactory,
|
||||
ILogger<SpeechRecognitionPipelineHostedService> logger)
|
||||
{
|
||||
this.pipelineFactory = pipelineFactory;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
startupCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
startupPipeline = pipelineFactory.Create();
|
||||
startupTask = Task.Run(
|
||||
() => WarmUpPipelineAsync(startupPipeline, startupCancellation.Token),
|
||||
CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var cancellation = startupCancellation;
|
||||
var task = startupTask;
|
||||
var pipeline = startupPipeline;
|
||||
if (cancellation is null || task is null || pipeline is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
startupCancellation = null;
|
||||
startupTask = null;
|
||||
startupPipeline = null;
|
||||
await cancellation.CancelAsync();
|
||||
try
|
||||
{
|
||||
await task.WaitAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
await pipeline.DisposeAsync();
|
||||
cancellation.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WarmUpPipelineAsync(
|
||||
ISpeechRecognitionPipeline pipeline,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await pipeline.InitializeAsync(cancellationToken);
|
||||
await pipeline.WaitUntilReadyAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Speech recognition pipeline warm-up was cancelled during application shutdown");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Speech recognition pipeline warm-up failed; recording can still start and recognition will retry when audio is processed");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
|
||||
{
|
||||
private readonly IStreamingTranscriptionProvider transcriptionProvider;
|
||||
private readonly Channel<AudioChunk> audio = Channel.CreateUnbounded<AudioChunk>();
|
||||
private readonly Channel<TranscriptionSegment> liveTranscript = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
private readonly List<TranscriptionSegment> liveSegments = [];
|
||||
private Task? transcriptionTask;
|
||||
private bool initialized;
|
||||
|
||||
protected StreamingSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
|
||||
{
|
||||
this.transcriptionProvider = transcriptionProvider;
|
||||
}
|
||||
|
||||
public virtual Task InitializeAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (initialized)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public virtual Task WaitUntilReadyAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
EnsureInitialized();
|
||||
return audio.Writer.WriteAsync(chunk, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
EnsureInitialized();
|
||||
audio.Writer.TryComplete();
|
||||
if (transcriptionTask is not null)
|
||||
{
|
||||
await transcriptionTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
EnsureInitialized();
|
||||
await foreach (var segment in liveTranscript.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
yield return segment;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
EnsureInitialized();
|
||||
await CompleteAsync(cancellationToken);
|
||||
var finalSegments = await BuildFinishedTranscriptAsync(audioPath, liveSegments, options, cancellationToken);
|
||||
return finalSegments.Count == 0 ? liveSegments : finalSegments;
|
||||
}
|
||||
|
||||
public virtual ValueTask DisposeAsync()
|
||||
{
|
||||
audio.Writer.TryComplete();
|
||||
liveTranscript.Writer.TryComplete();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
protected abstract Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
private async Task TranscribeAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (var segment in transcriptionProvider.TranscribeAsync(
|
||||
audio.Reader.ReadAllAsync(cancellationToken),
|
||||
cancellationToken))
|
||||
{
|
||||
liveSegments.Add(segment);
|
||||
await liveTranscript.Writer.WriteAsync(segment, cancellationToken);
|
||||
}
|
||||
|
||||
liveTranscript.Writer.TryComplete();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
liveTranscript.Writer.TryComplete(exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void EnsureInitialized()
|
||||
{
|
||||
if (!initialized)
|
||||
{
|
||||
throw new InvalidOperationException("Speech recognition pipeline must be initialized before use.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text);
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class WhisperLocalSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
|
||||
{
|
||||
private readonly PyannoteTranscriptFinalizer transcriptFinalizer;
|
||||
|
||||
public WhisperLocalSpeechRecognitionPipeline(
|
||||
WhisperLocalStreamingTranscriptionProvider transcriptionProvider,
|
||||
PyannoteTranscriptFinalizer transcriptFinalizer)
|
||||
: base(transcriptionProvider)
|
||||
{
|
||||
this.transcriptFinalizer = transcriptFinalizer;
|
||||
}
|
||||
|
||||
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, options, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
using Whisper.net;
|
||||
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTranscriptionProvider
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<WhisperLocalStreamingTranscriptionProvider> logger;
|
||||
|
||||
public WhisperLocalStreamingTranscriptionProvider(
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<WhisperLocalStreamingTranscriptionProvider> logger)
|
||||
{
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
|
||||
IAsyncEnumerable<AudioChunk> audio,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath));
|
||||
if (!File.Exists(modelPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Whisper model was not found at '{modelPath}'.", modelPath);
|
||||
}
|
||||
|
||||
using var factory = WhisperFactory.FromPath(modelPath);
|
||||
using var processor = factory.CreateBuilder()
|
||||
.WithLanguage(options.WhisperLocal.Language)
|
||||
.Build();
|
||||
|
||||
var window = new List<AudioChunk>();
|
||||
var windowDuration = TimeSpan.Zero;
|
||||
var windowTarget = TimeSpan.FromSeconds(Math.Max(1, options.WhisperLocal.WindowSeconds));
|
||||
var offset = TimeSpan.Zero;
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting local Whisper streaming transcription with model {ModelPath} and {WindowSeconds}s windows",
|
||||
modelPath,
|
||||
windowTarget.TotalSeconds);
|
||||
|
||||
await foreach (var chunk in audio.WithCancellation(cancellationToken))
|
||||
{
|
||||
window.Add(chunk);
|
||||
windowDuration += chunk.Duration;
|
||||
|
||||
if (windowDuration >= windowTarget)
|
||||
{
|
||||
var segments = 0;
|
||||
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
|
||||
{
|
||||
segments++;
|
||||
logger.LogInformation(
|
||||
"Whisper emitted segment at {Start}: {Text}",
|
||||
segment.Start,
|
||||
segment.Text);
|
||||
yield return segment;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Processed Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
|
||||
offset,
|
||||
windowDuration,
|
||||
segments);
|
||||
offset += windowDuration;
|
||||
window.Clear();
|
||||
windowDuration = TimeSpan.Zero;
|
||||
}
|
||||
}
|
||||
|
||||
if (window.Count > 0)
|
||||
{
|
||||
var segments = 0;
|
||||
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
|
||||
{
|
||||
segments++;
|
||||
logger.LogInformation(
|
||||
"Whisper emitted segment at {Start}: {Text}",
|
||||
segment.Start,
|
||||
segment.Text);
|
||||
yield return segment;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Processed final Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
|
||||
offset,
|
||||
windowDuration,
|
||||
segments);
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<TranscriptionSegment> ProcessWindow(
|
||||
WhisperProcessor processor,
|
||||
IReadOnlyList<AudioChunk> chunks,
|
||||
TimeSpan offset,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var stream = CreateWaveStream(chunks);
|
||||
|
||||
await foreach (var result in processor.ProcessAsync(stream, cancellationToken))
|
||||
{
|
||||
var text = result.Text?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsControlMarker(text))
|
||||
{
|
||||
logger.LogInformation("Whisper emitted control marker {Marker}; skipping transcript output", text);
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new TranscriptionSegment(offset + result.Start, offset + result.End, "Unknown", text);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsControlMarker(string text)
|
||||
{
|
||||
return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]";
|
||||
}
|
||||
|
||||
private MemoryStream CreateWaveStream(IReadOnlyList<AudioChunk> chunks)
|
||||
{
|
||||
var first = chunks[0];
|
||||
var stream = new MemoryStream();
|
||||
var pcm = new MemoryStream();
|
||||
|
||||
foreach (var chunk in chunks)
|
||||
{
|
||||
if (chunk.SampleRate != first.SampleRate || chunk.Channels != first.Channels)
|
||||
{
|
||||
logger.LogWarning("Dropping audio chunk with mismatched format while creating Whisper window");
|
||||
continue;
|
||||
}
|
||||
|
||||
pcm.Write(chunk.Pcm, 0, chunk.Pcm.Length);
|
||||
}
|
||||
|
||||
WriteWaveHeader(stream, first.SampleRate, first.Channels, (int)pcm.Length);
|
||||
pcm.Position = 0;
|
||||
pcm.CopyTo(stream);
|
||||
stream.Position = 0;
|
||||
return stream;
|
||||
}
|
||||
|
||||
private static void WriteWaveHeader(Stream stream, int sampleRate, int channels, int dataLength)
|
||||
{
|
||||
using var writer = new BinaryWriter(stream, System.Text.Encoding.ASCII, leaveOpen: true);
|
||||
writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"));
|
||||
writer.Write(36 + dataLength);
|
||||
writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE"));
|
||||
writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt "));
|
||||
writer.Write(16);
|
||||
writer.Write((short)1);
|
||||
writer.Write((short)channels);
|
||||
writer.Write(sampleRate);
|
||||
writer.Write(sampleRate * channels * sizeof(short));
|
||||
writer.Write((short)(channels * sizeof(short)));
|
||||
writer.Write((short)16);
|
||||
writer.Write(System.Text.Encoding.ASCII.GetBytes("data"));
|
||||
writer.Write(dataLength);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user