Implement meeting assistant v1

This commit is contained in:
2026-05-20 02:06:16 +02:00
parent 90df1edc03
commit 0297bcc0f6
120 changed files with 11883 additions and 180 deletions
@@ -0,0 +1,119 @@
using MeetingAssistant.Recording;
using NAudio.Wave;
namespace MeetingAssistant.Transcription;
public sealed class AsrDiagnosticService
{
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
public AsrDiagnosticService(ISpeechRecognitionPipelineFactory pipelineFactory)
{
this.pipelineFactory = pipelineFactory;
}
public async Task<AsrDiagnosticResult> TranscribeWavAsync(string path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("A WAV file path is required.", nameof(path));
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
}
await using var pipeline = pipelineFactory.Create();
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
await pipeline.CompleteAsync(cancellationToken);
return new AsrDiagnosticResult(fullPath, await liveTranscriptTask);
}
public async Task<AsrDiagnosticResult> DiarizeWavAsync(
string path,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
var fullPath = ResolveExistingWavPath(path);
await using var pipeline = pipelineFactory.Create();
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
await pipeline.CompleteAsync(cancellationToken);
await liveTranscriptTask;
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(fullPath, options, cancellationToken);
return new AsrDiagnosticResult(fullPath, finishedSegments);
}
private static async Task<IReadOnlyList<TranscriptionSegment>> CollectLiveTranscriptAsync(
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
var segments = new List<TranscriptionSegment>();
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
{
segments.Add(segment);
}
return segments;
}
private static async Task WriteWavToPipelineAsync(
string fullPath,
ISpeechRecognitionPipeline pipeline,
bool paceAudio,
CancellationToken cancellationToken)
{
await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken))
{
await pipeline.WriteAsync(chunk, cancellationToken);
if (paceAudio && chunk.Duration > TimeSpan.Zero)
{
await Task.Delay(chunk.Duration, cancellationToken);
}
}
}
private static string ResolveExistingWavPath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("A WAV file path is required.", nameof(path));
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
}
return fullPath;
}
private static async IAsyncEnumerable<AudioChunk> ReadPcmWavChunks(
string path,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
using var reader = new WaveFileReader(path);
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
throw new InvalidDataException(
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
}
var buffer = new byte[format.AverageBytesPerSecond];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
}
public sealed record AsrDiagnosticResult(string Path, IReadOnlyList<TranscriptionSegment> Segments);
@@ -0,0 +1,23 @@
namespace MeetingAssistant.Transcription;
public sealed class AzureSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
public AzureSpeechRecognitionPipeline(AzureSpeechStreamingTranscriptionProvider transcriptionProvider)
: base(transcriptionProvider)
{
}
internal AzureSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
: base(transcriptionProvider)
{
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(liveSegments);
}
}
@@ -0,0 +1,190 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.CognitiveServices.Speech;
using Microsoft.CognitiveServices.Speech.Audio;
using Microsoft.CognitiveServices.Speech.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<AzureSpeechStreamingTranscriptionProvider> logger;
public AzureSpeechStreamingTranscriptionProvider(
IOptions<MeetingAssistantOptions> options,
ILogger<AzureSpeechStreamingTranscriptionProvider> logger)
{
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
if (!await enumerator.MoveNextAsync())
{
yield break;
}
var firstChunk = enumerator.Current;
using var pushStream = AudioInputStream.CreatePushStream(
AudioStreamFormat.GetWaveFormatPCM(
(uint)firstChunk.SampleRate,
16,
(byte)firstChunk.Channels));
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
var speechConfig = CreateSpeechConfig();
using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
recognizer.Transcribed += (_, args) =>
{
if (args.Result.Reason != ResultReason.RecognizedSpeech
|| string.IsNullOrWhiteSpace(args.Result.Text))
{
return;
}
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
var segment = new TranscriptionSegment(
start,
start + args.Result.Duration,
NormalizeSpeakerId(args.Result.SpeakerId),
args.Result.Text.Trim());
logger.LogInformation(
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
segments.Writer.TryWrite(segment);
};
recognizer.Canceled += (_, args) =>
{
if (args.Reason == CancellationReason.Error)
{
segments.Writer.TryComplete(new InvalidOperationException(
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
return;
}
segments.Writer.TryComplete();
};
await recognizer.StartTranscribingAsync();
var pumpTask = Task.Run(
() => PumpAudioAsync(
firstChunk,
enumerator,
pushStream,
recognizer,
segments.Writer,
options.AzureSpeech.RecognitionStopTimeout,
cancellationToken),
CancellationToken.None);
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
await pumpTask.WaitAsync(cancellationToken);
}
private SpeechConfig CreateSpeechConfig()
{
var azure = options.AzureSpeech;
var key = ResolveKey(azure);
if (string.IsNullOrWhiteSpace(key))
{
throw new InvalidOperationException(
$"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
}
var speechConfig = string.IsNullOrWhiteSpace(azure.Region)
? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key)
: SpeechConfig.FromSubscription(key, azure.Region);
speechConfig.SpeechRecognitionLanguage = azure.Language;
speechConfig.OutputFormat = OutputFormat.Detailed;
speechConfig.SetProperty(
PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
azure.DiarizeIntermediateResults ? "true" : "false");
return speechConfig;
}
private static async Task PumpAudioAsync(
AudioChunk firstChunk,
IAsyncEnumerator<AudioChunk> audio,
PushAudioInputStream pushStream,
ConversationTranscriber recognizer,
ChannelWriter<TranscriptionSegment> segments,
TimeSpan recognitionStopTimeout,
CancellationToken cancellationToken)
{
try
{
WriteChunk(pushStream, firstChunk, firstChunk);
while (await audio.MoveNextAsync())
{
cancellationToken.ThrowIfCancellationRequested();
WriteChunk(pushStream, firstChunk, audio.Current);
}
pushStream.Close();
try
{
var stopTask = recognizer.StopTranscribingAsync();
if (recognitionStopTimeout > TimeSpan.Zero)
{
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
}
else
{
await stopTask.WaitAsync(cancellationToken);
}
}
catch (TimeoutException)
{
// The SDK can outlive short diagnostic streams while waiting for final service events.
}
segments.TryComplete();
}
catch (Exception exception)
{
segments.TryComplete(exception);
}
}
private static void WriteChunk(PushAudioInputStream pushStream, AudioChunk expectedFormat, AudioChunk chunk)
{
if (chunk.SampleRate != expectedFormat.SampleRate || chunk.Channels != expectedFormat.Channels)
{
return;
}
pushStream.Write(chunk.Pcm);
}
private static string NormalizeSpeakerId(string? speakerId)
{
return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase)
? "Unknown"
: speakerId;
}
private static string ResolveKey(AzureSpeechOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Key))
{
return options.Key;
}
return string.IsNullOrWhiteSpace(options.KeyEnv)
? ""
: Environment.GetEnvironmentVariable(options.KeyEnv) ?? "";
}
}
@@ -0,0 +1,78 @@
using System.Net.WebSockets;
using System.Text;
namespace MeetingAssistant.Transcription;
public sealed class ClientFunAsrWebSocketConnectionFactory : IFunAsrWebSocketConnectionFactory
{
public async Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken)
{
var socket = new ClientWebSocket();
await socket.ConnectAsync(endpoint, cancellationToken);
return new ClientFunAsrWebSocketConnection(socket);
}
}
public sealed class ClientFunAsrWebSocketConnection : IFunAsrWebSocketConnection
{
private readonly ClientWebSocket socket;
public ClientFunAsrWebSocketConnection(ClientWebSocket socket)
{
this.socket = socket;
}
public Task SendTextAsync(string message, CancellationToken cancellationToken)
{
return socket.SendAsync(
Encoding.UTF8.GetBytes(message),
WebSocketMessageType.Text,
endOfMessage: true,
cancellationToken);
}
public async Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken)
{
await socket.SendAsync(message, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken);
}
public async Task<string?> ReceiveTextAsync(CancellationToken cancellationToken)
{
var buffer = new byte[8192];
using var message = new MemoryStream();
while (true)
{
var result = await socket.ReceiveAsync(buffer, cancellationToken);
if (result.MessageType == WebSocketMessageType.Close)
{
return null;
}
if (result.MessageType != WebSocketMessageType.Text)
{
continue;
}
message.Write(buffer, 0, result.Count);
if (result.EndOfMessage)
{
return Encoding.UTF8.GetString(message.ToArray());
}
}
}
public async Task CloseOutputAsync(CancellationToken cancellationToken)
{
if (socket.State is WebSocketState.Open or WebSocketState.CloseReceived)
{
await socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "meeting assistant finished sending audio", cancellationToken);
}
}
public ValueTask DisposeAsync()
{
socket.Dispose();
return ValueTask.CompletedTask;
}
}
@@ -0,0 +1,101 @@
using System.Diagnostics;
using System.Text;
namespace MeetingAssistant.Transcription;
public interface ICommandRunner
{
Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null);
}
public sealed class ProcessCommandRunner : ICommandRunner
{
private readonly ILogger<ProcessCommandRunner> logger;
public ProcessCommandRunner(ILogger<ProcessCommandRunner> logger)
{
this.logger = logger;
}
public async Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
var startInfo = new ProcessStartInfo
{
FileName = fileName,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
foreach (var argument in arguments)
{
startInfo.ArgumentList.Add(argument);
}
if (environment is not null)
{
foreach (var (key, value) in environment)
{
startInfo.Environment[key] = value;
}
}
logger.LogInformation("Running command {Command} {Arguments}", fileName, string.Join(' ', arguments));
using var process = Process.Start(startInfo)
?? throw new InvalidOperationException($"Failed to start command '{fileName}'.");
var outputTask = ReadLinesAsync(process.StandardOutput, line => logger.LogInformation("{Command}: {Line}", fileName, line), cancellationToken);
var errorTask = ReadLinesAsync(process.StandardError, line => logger.LogWarning("{Command}: {Line}", fileName, line), cancellationToken);
try
{
await process.WaitForExitAsync(cancellationToken);
}
catch (OperationCanceledException)
{
if (!process.HasExited)
{
logger.LogWarning("Cancelling command {Command}; killing process tree.", fileName);
process.Kill(entireProcessTree: true);
}
throw;
}
var result = new CommandResult(process.ExitCode, await outputTask, await errorTask);
if (result.ExitCode != 0)
{
logger.LogWarning(
"Command {Command} exited with {ExitCode}: {Error}",
fileName,
result.ExitCode,
result.StandardError);
}
return result;
}
private static async Task<string> ReadLinesAsync(
StreamReader reader,
Action<string> log,
CancellationToken cancellationToken)
{
var output = new StringBuilder();
while (await reader.ReadLineAsync(cancellationToken) is { } line)
{
output.AppendLine(line);
log(line);
}
return output.ToString();
}
}
public sealed record CommandResult(int ExitCode, string StandardOutput, string StandardError);
@@ -0,0 +1,29 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class ConfiguredSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
{
private readonly IServiceProvider services;
private readonly MeetingAssistantOptions options;
public ConfiguredSpeechRecognitionPipelineFactory(
IServiceProvider services,
IOptions<MeetingAssistantOptions> options)
{
this.services = services;
this.options = options.Value;
}
public ISpeechRecognitionPipeline Create()
{
return options.Recording.TranscriptionProvider switch
{
"funasr" => ActivatorUtilities.CreateInstance<FunAsrSpeechRecognitionPipeline>(services),
"whisper-local" => ActivatorUtilities.CreateInstance<WhisperLocalSpeechRecognitionPipeline>(services),
"azure-speech" => ActivatorUtilities.CreateInstance<AzureSpeechRecognitionPipeline>(services),
_ => throw new InvalidOperationException(
$"Unsupported speech recognition pipeline '{options.Recording.TranscriptionProvider}'.")
};
}
}
@@ -0,0 +1,190 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public interface IFunAsrBackendLifecycle
{
Task EnsureStartedAsync(CancellationToken cancellationToken);
}
public sealed class FunAsrDockerBackendLifecycle : IFunAsrBackendLifecycle, IAsyncDisposable
{
private readonly ICommandRunner commandRunner;
private readonly IFunAsrBackendReadinessProbe readinessProbe;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrDockerBackendLifecycle> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private bool containerStarted;
private bool started;
private bool disposed;
public FunAsrDockerBackendLifecycle(
ICommandRunner commandRunner,
IFunAsrBackendReadinessProbe readinessProbe,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrDockerBackendLifecycle> logger)
{
this.commandRunner = commandRunner;
this.readinessProbe = readinessProbe;
this.options = options.Value;
this.logger = logger;
}
public async Task EnsureStartedAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(disposed, this);
var backend = options.FunAsr.Backend;
if (!backend.Enabled)
{
return;
}
await gate.WaitAsync(cancellationToken);
try
{
if (started)
{
return;
}
PrepareModelsFolder(backend);
await RunRequiredAsync(backend.DockerCommand, ["pull", backend.Image], cancellationToken);
await commandRunner.RunAsync(
backend.DockerCommand,
["rm", "-f", backend.ContainerName],
cancellationToken);
await RunRequiredAsync(backend.DockerCommand, BuildDockerRunArguments(), cancellationToken);
containerStarted = true;
await readinessProbe.WaitUntilReadyAsync(
new Uri(options.FunAsr.Endpoint),
backend.StartupTimeout,
cancellationToken);
started = true;
logger.LogInformation(
"Started managed FunASR backend container {ContainerName}; endpoint {Endpoint} is ready",
backend.ContainerName,
options.FunAsr.Endpoint);
}
finally
{
gate.Release();
}
}
public async ValueTask DisposeAsync()
{
if (disposed)
{
return;
}
var backend = options.FunAsr.Backend;
if (!backend.Enabled || !backend.StopOnDispose)
{
disposed = true;
gate.Dispose();
return;
}
await gate.WaitAsync();
try
{
if (containerStarted)
{
await commandRunner.RunAsync(
backend.DockerCommand,
["stop", backend.ContainerName],
CancellationToken.None);
containerStarted = false;
started = false;
}
}
finally
{
disposed = true;
gate.Release();
gate.Dispose();
}
}
private async Task RunRequiredAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken)
{
var backend = options.FunAsr.Backend;
using var timeoutSource = backend.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(backend.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
var effectiveToken = linkedSource?.Token ?? cancellationToken;
CommandResult result;
try
{
result = await commandRunner.RunAsync(fileName, arguments, effectiveToken);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"Command '{fileName} {string.Join(' ', arguments)}' timed out after {backend.CommandTimeout}.");
}
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"Command '{fileName} {string.Join(' ', arguments)}' failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
private string[] BuildDockerRunArguments()
{
var backend = options.FunAsr.Backend;
List<string> arguments =
[
"run",
"--rm",
"-d",
];
if (backend.Privileged)
{
arguments.Add("--privileged=true");
}
arguments.AddRange(
[
"--name",
backend.ContainerName,
"-p",
$"{GetEndpointPort()}:{backend.ContainerPort}",
"-v",
$"{VaultPath.Resolve(backend.ModelsFolder)}:/workspace/models",
backend.Image,
"bash",
"-lc",
backend.ServerCommand
]);
return [.. arguments];
}
private static void PrepareModelsFolder(FunAsrBackendOptions backend)
{
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
var hotwordsPath = Path.Combine(modelsFolder, "hotwords.txt");
if (!File.Exists(hotwordsPath))
{
File.WriteAllText(hotwordsPath, "");
}
}
private int GetEndpointPort()
{
return new Uri(options.FunAsr.Endpoint).Port;
}
}
@@ -0,0 +1,182 @@
using System.Text.Json;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrMessageParser
{
private readonly FunAsrOptions options;
public FunAsrMessageParser(FunAsrOptions options)
{
this.options = options;
}
public FunAsrParsedMessage Parse(string message)
{
using var document = JsonDocument.Parse(message);
var root = document.RootElement;
var mode = GetString(root, "mode");
var isFinal = GetBoolean(root, "is_final");
var shouldEmit = options.EmitOnlinePartials || !IsOnlineMode(mode);
var segments = shouldEmit ? ReadSegments(root) : [];
return new FunAsrParsedMessage(IsTerminal: isFinal && !IsOnlineMode(mode), segments);
}
private static List<TranscriptionSegment> ReadSegments(JsonElement root)
{
var sentenceSegments = ReadSentenceSegments(root, "sentence_info");
if (sentenceSegments.Count > 0)
{
return sentenceSegments;
}
var stampSegments = ReadSentenceSegments(root, "stamp_sents");
if (stampSegments.Count > 0)
{
return stampSegments;
}
var text = GetString(root, "text").Trim();
if (string.IsNullOrWhiteSpace(text))
{
return [];
}
var (start, end) = ReadTimestampRange(root);
return [new TranscriptionSegment(start, end, ReadSpeaker(root), text)];
}
private static List<TranscriptionSegment> ReadSentenceSegments(JsonElement root, string propertyName)
{
if (!root.TryGetProperty(propertyName, out var sentences) || sentences.ValueKind != JsonValueKind.Array)
{
return [];
}
var segments = new List<TranscriptionSegment>();
foreach (var sentence in sentences.EnumerateArray())
{
var text = (GetString(sentence, "text") + GetString(sentence, "text_seg") + GetString(sentence, "punc")).Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromMilliseconds(GetDouble(sentence, "start")),
TimeSpan.FromMilliseconds(GetDouble(sentence, "end")),
ReadSpeaker(sentence),
text));
}
return segments;
}
private static (TimeSpan Start, TimeSpan End) ReadTimestampRange(JsonElement root)
{
if (!root.TryGetProperty("timestamp", out var timestamp))
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
if (timestamp.ValueKind == JsonValueKind.String)
{
var value = timestamp.GetString();
if (string.IsNullOrWhiteSpace(value))
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
using var document = JsonDocument.Parse(value);
return ReadTimestampArray(document.RootElement);
}
return ReadTimestampArray(timestamp);
}
private static (TimeSpan Start, TimeSpan End) ReadTimestampArray(JsonElement timestamp)
{
if (timestamp.ValueKind != JsonValueKind.Array)
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
double? start = null;
double? end = null;
foreach (var item in timestamp.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Array)
{
continue;
}
var values = item.EnumerateArray().ToArray();
if (values.Length < 2)
{
continue;
}
start ??= ReadNumber(values[0]);
end = ReadNumber(values[1]);
}
return (
TimeSpan.FromMilliseconds(start ?? 0),
TimeSpan.FromMilliseconds(end ?? start ?? 0));
}
private static string ReadSpeaker(JsonElement element)
{
var speaker = GetString(element, "spk_name");
if (!string.IsNullOrWhiteSpace(speaker))
{
return speaker;
}
speaker = GetString(element, "speaker");
if (!string.IsNullOrWhiteSpace(speaker))
{
return speaker;
}
if (element.TryGetProperty("spk", out var spk))
{
return $"Speaker {spk}";
}
return "Unknown";
}
private static bool IsOnlineMode(string mode)
{
return mode.Equals("online", StringComparison.OrdinalIgnoreCase)
|| mode.Equals("2pass-online", StringComparison.OrdinalIgnoreCase);
}
private static string GetString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static bool GetBoolean(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind is JsonValueKind.True or JsonValueKind.False
&& property.GetBoolean();
}
private static double GetDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) ? ReadNumber(property) : 0;
}
private static double ReadNumber(JsonElement element)
{
return element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0;
}
}
public sealed record FunAsrParsedMessage(bool IsTerminal, IReadOnlyList<TranscriptionSegment> Segments);
@@ -0,0 +1,31 @@
namespace MeetingAssistant.Transcription;
public sealed class FunAsrSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
private readonly IFunAsrBackendLifecycle backendLifecycle;
private readonly FunAsrTranscriptFinalizer transcriptFinalizer;
public FunAsrSpeechRecognitionPipeline(
FunAsrStreamingTranscriptionProvider transcriptionProvider,
IFunAsrBackendLifecycle backendLifecycle,
FunAsrTranscriptFinalizer transcriptFinalizer)
: base(transcriptionProvider)
{
this.backendLifecycle = backendLifecycle;
this.transcriptFinalizer = transcriptFinalizer;
}
public override Task WaitUntilReadyAsync(CancellationToken cancellationToken)
{
return backendLifecycle.EnsureStartedAsync(cancellationToken);
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, cancellationToken);
}
}
@@ -0,0 +1,184 @@
using System.Text.Json;
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscriptionProvider, IAsyncDisposable
{
private readonly IFunAsrWebSocketConnectionFactory connectionFactory;
private readonly IFunAsrBackendLifecycle backendLifecycle;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrStreamingTranscriptionProvider> logger;
public FunAsrStreamingTranscriptionProvider(
IFunAsrWebSocketConnectionFactory connectionFactory,
IFunAsrBackendLifecycle backendLifecycle,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrStreamingTranscriptionProvider> logger)
{
this.connectionFactory = connectionFactory;
this.backendLifecycle = backendLifecycle;
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None);
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
await session;
}
private async Task RunSessionAsync(
IAsyncEnumerable<AudioChunk> audio,
ChannelWriter<TranscriptionSegment> segments,
CancellationToken cancellationToken)
{
try
{
await backendLifecycle.EnsureStartedAsync(cancellationToken);
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
if (!await enumerator.MoveNextAsync())
{
segments.TryComplete();
return;
}
var firstChunk = enumerator.Current;
var endpoint = new Uri(options.FunAsr.Endpoint);
await using var connection = await connectionFactory.ConnectAsync(endpoint, cancellationToken);
var parser = new FunAsrMessageParser(options.FunAsr);
var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken);
await SendStartAsync(connection, firstChunk, cancellationToken);
await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken);
while (await enumerator.MoveNextAsync())
{
await connection.SendBinaryAsync(enumerator.Current.Pcm, cancellationToken);
}
await connection.SendTextAsync("""{"is_speaking":false}""", cancellationToken);
try
{
await finalReceived.Task.WaitAsync(options.FunAsr.FinalResultTimeout, cancellationToken);
}
catch (TimeoutException)
{
logger.LogWarning(
"Timed out waiting {Timeout} for the final FunASR result from {Endpoint}",
options.FunAsr.FinalResultTimeout,
endpoint);
}
await connection.CloseOutputAsync(cancellationToken);
await receiveTask;
segments.TryComplete();
}
catch (Exception exception)
{
segments.TryComplete(exception);
}
}
public async ValueTask DisposeAsync()
{
if (backendLifecycle is IAsyncDisposable disposable)
{
await disposable.DisposeAsync();
}
}
private async Task ReceiveAsync(
IFunAsrWebSocketConnection connection,
FunAsrMessageParser parser,
ChannelWriter<TranscriptionSegment> segments,
TaskCompletionSource finalReceived,
CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var message = await connection.ReceiveTextAsync(cancellationToken);
if (message is null)
{
finalReceived.TrySetResult();
return;
}
var parsed = parser.Parse(message);
foreach (var segment in parsed.Segments)
{
await segments.WriteAsync(segment, cancellationToken);
logger.LogInformation(
"FunASR emitted segment at {Start} for {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
}
if (parsed.IsTerminal)
{
finalReceived.TrySetResult();
return;
}
}
}
private async Task SendStartAsync(
IFunAsrWebSocketConnection connection,
AudioChunk firstChunk,
CancellationToken cancellationToken)
{
var message = new Dictionary<string, object?>
{
["mode"] = options.FunAsr.Mode,
["chunk_size"] = NormalizeChunkSize(options.FunAsr.ChunkSize),
["chunk_interval"] = options.FunAsr.ChunkInterval,
["encoder_chunk_look_back"] = options.FunAsr.EncoderChunkLookBack,
["decoder_chunk_look_back"] = options.FunAsr.DecoderChunkLookBack,
["audio_fs"] = firstChunk.SampleRate,
["wav_name"] = options.FunAsr.WavName,
["wav_format"] = "pcm",
["is_speaking"] = true,
["hotwords"] = await LoadHotwordsAsync(cancellationToken),
["itn"] = options.FunAsr.Itn
};
await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken);
}
private async Task<string> LoadHotwordsAsync(CancellationToken cancellationToken)
{
var path = VaultPath.Resolve(options.Vault.DictationWordsPath);
if (!File.Exists(path))
{
return "";
}
var words = await File.ReadAllLinesAsync(path, cancellationToken);
var hotwords = words
.Select(line => line.Trim().TrimStart('-', '*').Trim())
.Where(line => !string.IsNullOrWhiteSpace(line))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToDictionary(word => word, _ => 20);
return hotwords.Count == 0 ? "" : JsonSerializer.Serialize(hotwords);
}
private static int[] NormalizeChunkSize(int[] configuredChunkSize)
{
return configuredChunkSize.Length == 3 ? configuredChunkSize : [5, 10, 5];
}
}
@@ -0,0 +1,207 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrTranscriptFinalizer
{
private const string JsonStart = "__MEETING_ASSISTANT_DIARIZATION_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_DIARIZATION_JSON_END__";
private readonly ICommandRunner commandRunner;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrTranscriptFinalizer> logger;
public FunAsrTranscriptFinalizer(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrTranscriptFinalizer> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value;
this.logger = logger;
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
CancellationToken cancellationToken)
{
var diarization = options.FunAsr.Diarization;
if (!diarization.Enabled)
{
return [];
}
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
if (!File.Exists(fullAudioPath))
{
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
}
var result = await RunDiarizationAsync(fullAudioPath, cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"FunASR diarization failed with exit code {result.ExitCode}: {result.StandardError}");
}
var json = ExtractJson(result.StandardOutput);
var segments = ParseSegments(json);
logger.LogInformation("FunASR final diarization produced {SegmentCount} sentence segments", segments.Count);
return segments;
}
private async Task<CommandResult> RunDiarizationAsync(
string fullAudioPath,
CancellationToken cancellationToken)
{
var backend = options.FunAsr.Backend;
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
using var timeoutSource = options.FunAsr.Diarization.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(options.FunAsr.Diarization.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
return await commandRunner.RunAsync(
backend.DockerCommand,
BuildDockerArguments(fullAudioPath, modelsFolder),
linkedSource?.Token ?? cancellationToken);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"FunASR diarization timed out after {options.FunAsr.Diarization.CommandTimeout}.");
}
}
private string[] BuildDockerArguments(string fullAudioPath, string modelsFolder)
{
var backend = options.FunAsr.Backend;
return
[
"run",
"--rm",
"-v",
$"{fullAudioPath}:/workspace/input.wav:ro",
"-v",
$"{modelsFolder}:/workspace/models",
backend.Image,
"bash",
"-lc",
BuildPythonCommand()
];
}
private string BuildPythonCommand()
{
var diarization = options.FunAsr.Diarization;
var presetSpeakerCount = diarization.PresetSpeakerCount is int value
? $", preset_spk_num={value}"
: "";
return
"cat > /tmp/meeting_assistant_diarize.py <<'PY'\n"
+ "import json\n"
+ "from funasr import AutoModel\n"
+ "model = AutoModel(\n"
+ $" model={JsonSerializer.Serialize(diarization.AsrModel)},\n"
+ $" vad_model={JsonSerializer.Serialize(diarization.VadModel)},\n"
+ $" punc_model={JsonSerializer.Serialize(diarization.PunctuationModel)},\n"
+ $" spk_model={JsonSerializer.Serialize(diarization.SpeakerModel)},\n"
+ " device='cpu',\n"
+ " disable_update=True,\n"
+ ")\n"
+ "res = model.generate(\n"
+ " input='/workspace/input.wav',\n"
+ $" batch_size_s={diarization.BatchSizeSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)},\n"
+ $" batch_size_threshold_s={diarization.BatchSizeThresholdSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)}"
+ presetSpeakerCount
+ "\n)\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(res, ensure_ascii=False, default=str))\n"
+ $"print('{JsonEnd}')\n"
+ "PY\n"
+ "MODELSCOPE_CACHE=/workspace/models/modelscope python /tmp/meeting_assistant_diarize.py";
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidOperationException("FunASR diarization output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidOperationException("FunASR diarization output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private static IReadOnlyList<TranscriptionSegment> ParseSegments(string json)
{
using var document = JsonDocument.Parse(json);
var segments = new List<TranscriptionSegment>();
foreach (var result in document.RootElement.EnumerateArray())
{
if (!result.TryGetProperty("sentence_info", out var sentences)
|| sentences.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var sentence in sentences.EnumerateArray())
{
var text = ReadString(sentence, "text").Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromMilliseconds(ReadDouble(sentence, "start")),
TimeSpan.FromMilliseconds(ReadDouble(sentence, "end")),
$"Speaker {ReadInt(sentence, "spk")}",
text));
}
}
return segments;
}
private static string ReadString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static double ReadDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetDouble(out var value)
? value
: 0;
}
private static int ReadInt(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetInt32(out var value)
? value
: 0;
}
}
@@ -0,0 +1,84 @@
using System.Net.WebSockets;
namespace MeetingAssistant.Transcription;
public interface IFunAsrBackendReadinessProbe
{
Task WaitUntilReadyAsync(Uri endpoint, TimeSpan timeout, CancellationToken cancellationToken);
}
public sealed class FunAsrWebSocketBackendReadinessProbe : IFunAsrBackendReadinessProbe
{
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(1);
public async Task WaitUntilReadyAsync(
Uri endpoint,
TimeSpan timeout,
CancellationToken cancellationToken)
{
if (timeout <= TimeSpan.Zero)
{
return;
}
using var timeoutSource = new CancellationTokenSource(timeout);
using var linkedSource = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken,
timeoutSource.Token);
while (!linkedSource.IsCancellationRequested)
{
try
{
using var client = new ClientWebSocket();
await client.ConnectAsync(endpoint, linkedSource.Token).ConfigureAwait(false);
await CloseQuietlyAsync(client, linkedSource.Token).ConfigureAwait(false);
return;
}
catch (OperationCanceledException)
{
break;
}
catch (Exception exception) when (IsTransientReadinessFailure(exception))
{
try
{
await Task.Delay(RetryDelay, linkedSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
}
}
throw new TimeoutException(
$"Timed out waiting {timeout} for managed FunASR backend WebSocket readiness at {endpoint}.");
}
private static async Task CloseQuietlyAsync(ClientWebSocket client, CancellationToken cancellationToken)
{
if (client.State != WebSocketState.Open)
{
return;
}
try
{
await client.CloseOutputAsync(
WebSocketCloseStatus.NormalClosure,
"readiness probe complete",
cancellationToken).ConfigureAwait(false);
}
catch (WebSocketException)
{
}
}
private static bool IsTransientReadinessFailure(Exception exception)
{
return exception is WebSocketException
or HttpRequestException
or IOException;
}
}
@@ -0,0 +1,17 @@
namespace MeetingAssistant.Transcription;
public interface IFunAsrWebSocketConnectionFactory
{
Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken);
}
public interface IFunAsrWebSocketConnection : IAsyncDisposable
{
Task SendTextAsync(string message, CancellationToken cancellationToken);
Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken);
Task<string?> ReceiveTextAsync(CancellationToken cancellationToken);
Task CloseOutputAsync(CancellationToken cancellationToken);
}
@@ -0,0 +1,31 @@
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public interface ISpeechRecognitionPipeline : IAsyncDisposable
{
Task InitializeAsync(CancellationToken cancellationToken);
Task WaitUntilReadyAsync(CancellationToken cancellationToken);
ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken);
Task CompleteAsync(CancellationToken cancellationToken);
IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(CancellationToken cancellationToken);
Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
string audioPath,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken);
}
public interface ISpeechRecognitionPipelineFactory
{
ISpeechRecognitionPipeline Create();
}
public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null)
{
public static SpeechRecognitionPipelineOptions Default { get; } = new();
}
@@ -0,0 +1,10 @@
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public interface IStreamingTranscriptionProvider
{
IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
CancellationToken cancellationToken);
}
@@ -0,0 +1,420 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class PyannoteTranscriptFinalizer
{
private const string JsonStart = "__MEETING_ASSISTANT_PYANNOTE_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_PYANNOTE_JSON_END__";
private readonly ICommandRunner commandRunner;
private readonly MeetingAssistantOptions options;
private readonly ILogger<PyannoteTranscriptFinalizer> logger;
public PyannoteTranscriptFinalizer(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<PyannoteTranscriptFinalizer> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value;
this.logger = logger;
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
return await FinalizeAsync(
audioPath,
liveSegments,
options.WhisperLocal.Diarization,
pipelineOptions,
cancellationToken);
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
if (!diarization.Enabled || liveSegments.Count == 0)
{
return [];
}
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
if (!File.Exists(fullAudioPath))
{
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
}
var token = ResolveToken(diarization);
if (string.IsNullOrWhiteSpace(token))
{
logger.LogWarning(
"Pyannote diarization is enabled but no Hugging Face token is configured directly or via {TokenEnv}",
diarization.TokenEnv);
return [];
}
var result = await RunDiarizationAsync(fullAudioPath, token, diarization, pipelineOptions, cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"pyannote diarization failed with exit code {result.ExitCode}: {result.StandardError}");
}
var turns = ParseTurns(ExtractJson(result.StandardOutput));
var segments = diarization.AlignmentMode == PyannoteAlignmentMode.PyannoteTurns
? BuildTurnSegments(liveSegments, turns)
: AssignSpeakers(liveSegments, turns);
logger.LogInformation(
"pyannote diarization produced {TurnCount} speaker turns and built {SegmentCount} transcript segments using {AlignmentMode}",
turns.Count,
segments.Count,
diarization.AlignmentMode);
return segments;
}
private async Task<CommandResult> RunDiarizationAsync(
string fullAudioPath,
string token,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
var modelsFolder = VaultPath.Resolve(diarization.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
using var timeoutSource = diarization.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(diarization.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
if (diarization.BuildImage)
{
await EnsureDockerImageAsync(diarization, modelsFolder, linkedSource?.Token ?? cancellationToken);
}
return await commandRunner.RunAsync(
diarization.DockerCommand,
BuildDockerArguments(diarization, fullAudioPath, modelsFolder, pipelineOptions),
linkedSource?.Token ?? cancellationToken,
new Dictionary<string, string> { ["HF_TOKEN"] = token });
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"pyannote diarization timed out after {diarization.CommandTimeout}.");
}
}
private async Task EnsureDockerImageAsync(
PyannoteDiarizationOptions diarization,
string modelsFolder,
CancellationToken cancellationToken)
{
var inspectResult = await commandRunner.RunAsync(
diarization.DockerCommand,
["image", "inspect", "--format", "{{.Id}}", diarization.Image],
cancellationToken);
if (inspectResult.ExitCode == 0)
{
return;
}
var dockerfilePath = Path.Combine(modelsFolder, "docker", $"{SanitizeFileName(diarization.Image)}.Dockerfile");
Directory.CreateDirectory(Path.GetDirectoryName(dockerfilePath)!);
await File.WriteAllTextAsync(dockerfilePath, BuildDockerfile(diarization), cancellationToken);
var result = await commandRunner.RunAsync(
diarization.DockerCommand,
["build", "-t", diarization.Image, "-f", dockerfilePath, Path.GetDirectoryName(dockerfilePath)!],
cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"pyannote Docker image build failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
{
return
$"FROM {diarization.BaseImage}\n"
+ "ENV DEBIAN_FRONTEND=noninteractive\n"
+ "RUN apt-get update \\\n"
+ " && apt-get install -y --no-install-recommends ffmpeg libsndfile1 \\\n"
+ " && rm -rf /var/lib/apt/lists/*\n"
+ "RUN python -m pip install --upgrade pip \\\n"
+ " && python -m pip install pyannote.audio soundfile\n";
}
private string[] BuildDockerArguments(
PyannoteDiarizationOptions diarization,
string fullAudioPath,
string modelsFolder,
SpeechRecognitionPipelineOptions pipelineOptions)
{
return
[
"run",
"--rm",
"-e",
"HF_TOKEN",
"-v",
$"{fullAudioPath}:/workspace/input.wav:ro",
"-v",
$"{modelsFolder}:/workspace/cache",
"-e",
"HF_HOME=/workspace/cache/huggingface",
"-e",
"XDG_CACHE_HOME=/workspace/cache",
"-e",
"PIP_CACHE_DIR=/workspace/cache/pip",
"-e",
"TORCH_HOME=/workspace/cache/torch",
diarization.Image,
"sh",
"-lc",
BuildPythonCommand(diarization, pipelineOptions)
];
}
private static string BuildPythonCommand(
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions)
{
var model = diarization.Model;
var annotationProperty = diarization.AnnotationSource == PyannoteAnnotationSource.ExclusiveSpeakerDiarization
? "exclusive_speaker_diarization"
: "speaker_diarization";
return
"cat > /tmp/meeting_assistant_pyannote.py <<'PY'\n"
+ "import json\n"
+ "import os\n"
+ "import torch\n"
+ "from pyannote.audio import Pipeline\n"
+ $"pipeline = Pipeline.from_pretrained({JsonSerializer.Serialize(model)}, token=os.environ.get('HF_TOKEN'))\n"
+ "pipeline.to(torch.device('cpu'))\n"
+ BuildPipelineInvocation(pipelineOptions)
+ $"diarization = getattr(result, {JsonSerializer.Serialize(annotationProperty)}, result)\n"
+ "turns = []\n"
+ "for turn, _, speaker in diarization.itertracks(yield_label=True):\n"
+ " turns.append({'start': float(turn.start), 'end': float(turn.end), 'speaker': str(speaker)})\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(turns, ensure_ascii=False))\n"
+ $"print('{JsonEnd}')\n"
+ "PY\n"
+ "python /tmp/meeting_assistant_pyannote.py";
}
private static string BuildPipelineInvocation(SpeechRecognitionPipelineOptions pipelineOptions)
{
return pipelineOptions.NumSpeakers is > 0
? $"result = pipeline('/workspace/input.wav', num_speakers={pipelineOptions.NumSpeakers.Value})\n"
: "result = pipeline('/workspace/input.wav')\n";
}
private static IReadOnlyList<TranscriptionSegment> BuildTurnSegments(
IReadOnlyList<TranscriptionSegment> liveSegments,
IReadOnlyList<SpeakerTurn> turns)
{
if (turns.Count == 0)
{
return [];
}
var segments = new List<TranscriptionSegment>();
foreach (var turn in turns)
{
var text = BuildTurnText(liveSegments, turn);
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromSeconds(turn.Start),
TimeSpan.FromSeconds(turn.End),
turn.Speaker,
text));
}
return segments.Count == 0
? AssignSpeakers(liveSegments, turns)
: segments;
}
private static string BuildTurnText(IReadOnlyList<TranscriptionSegment> liveSegments, SpeakerTurn turn)
{
var parts = new List<string>();
foreach (var segment in liveSegments)
{
var segmentStart = segment.Start.TotalSeconds;
var segmentEnd = segment.End > segment.Start
? segment.End.TotalSeconds
: segmentStart;
var overlapStart = Math.Max(segmentStart, turn.Start);
var overlapEnd = Math.Min(segmentEnd, turn.End);
if (overlapEnd <= overlapStart)
{
continue;
}
var part = SliceTextByTime(segment.Text, segmentStart, segmentEnd, overlapStart, overlapEnd);
if (!string.IsNullOrWhiteSpace(part))
{
parts.Add(part);
}
}
return string.Join(' ', parts);
}
private static string SliceTextByTime(
string text,
double segmentStart,
double segmentEnd,
double overlapStart,
double overlapEnd)
{
var words = text
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (words.Length == 0)
{
return "";
}
var duration = Math.Max(0.001, segmentEnd - segmentStart);
var startFraction = Math.Clamp((overlapStart - segmentStart) / duration, 0, 1);
var endFraction = Math.Clamp((overlapEnd - segmentStart) / duration, 0, 1);
var startIndex = Math.Clamp((int)Math.Floor(startFraction * words.Length), 0, words.Length - 1);
var endIndex = Math.Clamp((int)Math.Ceiling(endFraction * words.Length), startIndex + 1, words.Length);
return string.Join(' ', words[startIndex..endIndex]);
}
private static IReadOnlyList<TranscriptionSegment> AssignSpeakers(
IReadOnlyList<TranscriptionSegment> liveSegments,
IReadOnlyList<SpeakerTurn> turns)
{
if (turns.Count == 0)
{
return [];
}
return liveSegments
.Select(segment =>
{
var speaker = FindBestSpeaker(segment, turns) ?? segment.Speaker;
return segment with { Speaker = speaker };
})
.ToList();
}
private static string? FindBestSpeaker(TranscriptionSegment segment, IReadOnlyList<SpeakerTurn> turns)
{
var segmentStart = segment.Start.TotalSeconds;
var segmentEnd = segment.End > segment.Start
? segment.End.TotalSeconds
: segmentStart;
var bestOverlap = 0d;
string? bestSpeaker = null;
foreach (var turn in turns)
{
var overlap = Math.Min(segmentEnd, turn.End) - Math.Max(segmentStart, turn.Start);
if (overlap > bestOverlap)
{
bestOverlap = overlap;
bestSpeaker = turn.Speaker;
}
}
return bestSpeaker;
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidOperationException("pyannote output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidOperationException("pyannote output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private static IReadOnlyList<SpeakerTurn> ParseTurns(string json)
{
using var document = JsonDocument.Parse(json);
var turns = new List<SpeakerTurn>();
foreach (var item in document.RootElement.EnumerateArray())
{
var speaker = ReadString(item, "speaker");
if (string.IsNullOrWhiteSpace(speaker))
{
continue;
}
turns.Add(new SpeakerTurn(
ReadDouble(item, "start"),
ReadDouble(item, "end"),
speaker));
}
return turns;
}
private static string ResolveToken(PyannoteDiarizationOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Token))
{
return options.Token;
}
return string.IsNullOrWhiteSpace(options.TokenEnv)
? ""
: Environment.GetEnvironmentVariable(options.TokenEnv) ?? "";
}
private static string ReadString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static double ReadDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetDouble(out var value)
? value
: 0;
}
private static string SanitizeFileName(string value)
{
var invalid = Path.GetInvalidFileNameChars();
return string.Concat(value.Select(character => invalid.Contains(character) ? '_' : character));
}
private sealed record SpeakerTurn(double Start, double End, string Speaker);
}
@@ -0,0 +1,77 @@
namespace MeetingAssistant.Transcription;
public sealed class SpeechRecognitionPipelineHostedService : IHostedService
{
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
private readonly ILogger<SpeechRecognitionPipelineHostedService> logger;
private CancellationTokenSource? startupCancellation;
private Task? startupTask;
private ISpeechRecognitionPipeline? startupPipeline;
public SpeechRecognitionPipelineHostedService(
ISpeechRecognitionPipelineFactory pipelineFactory,
ILogger<SpeechRecognitionPipelineHostedService> logger)
{
this.pipelineFactory = pipelineFactory;
this.logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
startupCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
startupPipeline = pipelineFactory.Create();
startupTask = Task.Run(
() => WarmUpPipelineAsync(startupPipeline, startupCancellation.Token),
CancellationToken.None);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken cancellationToken)
{
var cancellation = startupCancellation;
var task = startupTask;
var pipeline = startupPipeline;
if (cancellation is null || task is null || pipeline is null)
{
return;
}
startupCancellation = null;
startupTask = null;
startupPipeline = null;
await cancellation.CancelAsync();
try
{
await task.WaitAsync(cancellationToken);
}
catch (OperationCanceledException)
{
}
finally
{
await pipeline.DisposeAsync();
cancellation.Dispose();
}
}
private async Task WarmUpPipelineAsync(
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
try
{
await pipeline.InitializeAsync(cancellationToken);
await pipeline.WaitUntilReadyAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Speech recognition pipeline warm-up was cancelled during application shutdown");
}
catch (Exception exception)
{
logger.LogError(
exception,
"Speech recognition pipeline warm-up failed; recording can still start and recognition will retry when audio is processed");
}
}
}
@@ -0,0 +1,114 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
{
private readonly IStreamingTranscriptionProvider transcriptionProvider;
private readonly Channel<AudioChunk> audio = Channel.CreateUnbounded<AudioChunk>();
private readonly Channel<TranscriptionSegment> liveTranscript = Channel.CreateUnbounded<TranscriptionSegment>();
private readonly List<TranscriptionSegment> liveSegments = [];
private Task? transcriptionTask;
private bool initialized;
protected StreamingSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
{
this.transcriptionProvider = transcriptionProvider;
}
public virtual Task InitializeAsync(CancellationToken cancellationToken)
{
if (initialized)
{
return Task.CompletedTask;
}
initialized = true;
transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None);
return Task.CompletedTask;
}
public virtual Task WaitUntilReadyAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
EnsureInitialized();
return audio.Writer.WriteAsync(chunk, cancellationToken);
}
public async Task CompleteAsync(CancellationToken cancellationToken)
{
EnsureInitialized();
audio.Writer.TryComplete();
if (transcriptionTask is not null)
{
await transcriptionTask.WaitAsync(cancellationToken);
}
}
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
EnsureInitialized();
await foreach (var segment in liveTranscript.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
}
public async Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
string audioPath,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
EnsureInitialized();
await CompleteAsync(cancellationToken);
var finalSegments = await BuildFinishedTranscriptAsync(audioPath, liveSegments, options, cancellationToken);
return finalSegments.Count == 0 ? liveSegments : finalSegments;
}
public virtual ValueTask DisposeAsync()
{
audio.Writer.TryComplete();
liveTranscript.Writer.TryComplete();
return ValueTask.CompletedTask;
}
protected abstract Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken);
private async Task TranscribeAsync(CancellationToken cancellationToken)
{
try
{
await foreach (var segment in transcriptionProvider.TranscribeAsync(
audio.Reader.ReadAllAsync(cancellationToken),
cancellationToken))
{
liveSegments.Add(segment);
await liveTranscript.Writer.WriteAsync(segment, cancellationToken);
}
liveTranscript.Writer.TryComplete();
}
catch (Exception exception)
{
liveTranscript.Writer.TryComplete(exception);
}
}
private void EnsureInitialized()
{
if (!initialized)
{
throw new InvalidOperationException("Speech recognition pipeline must be initialized before use.");
}
}
}
@@ -0,0 +1,3 @@
namespace MeetingAssistant.Transcription;
public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text);
@@ -0,0 +1,23 @@
namespace MeetingAssistant.Transcription;
public sealed class WhisperLocalSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
private readonly PyannoteTranscriptFinalizer transcriptFinalizer;
public WhisperLocalSpeechRecognitionPipeline(
WhisperLocalStreamingTranscriptionProvider transcriptionProvider,
PyannoteTranscriptFinalizer transcriptFinalizer)
: base(transcriptionProvider)
{
this.transcriptFinalizer = transcriptFinalizer;
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, options, cancellationToken);
}
}
@@ -0,0 +1,168 @@
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
using NAudio.Wave;
using Whisper.net;
namespace MeetingAssistant.Transcription;
public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<WhisperLocalStreamingTranscriptionProvider> logger;
public WhisperLocalStreamingTranscriptionProvider(
IOptions<MeetingAssistantOptions> options,
ILogger<WhisperLocalStreamingTranscriptionProvider> logger)
{
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath));
if (!File.Exists(modelPath))
{
throw new FileNotFoundException($"Whisper model was not found at '{modelPath}'.", modelPath);
}
using var factory = WhisperFactory.FromPath(modelPath);
using var processor = factory.CreateBuilder()
.WithLanguage(options.WhisperLocal.Language)
.Build();
var window = new List<AudioChunk>();
var windowDuration = TimeSpan.Zero;
var windowTarget = TimeSpan.FromSeconds(Math.Max(1, options.WhisperLocal.WindowSeconds));
var offset = TimeSpan.Zero;
logger.LogInformation(
"Starting local Whisper streaming transcription with model {ModelPath} and {WindowSeconds}s windows",
modelPath,
windowTarget.TotalSeconds);
await foreach (var chunk in audio.WithCancellation(cancellationToken))
{
window.Add(chunk);
windowDuration += chunk.Duration;
if (windowDuration >= windowTarget)
{
var segments = 0;
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
{
segments++;
logger.LogInformation(
"Whisper emitted segment at {Start}: {Text}",
segment.Start,
segment.Text);
yield return segment;
}
logger.LogInformation(
"Processed Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
offset,
windowDuration,
segments);
offset += windowDuration;
window.Clear();
windowDuration = TimeSpan.Zero;
}
}
if (window.Count > 0)
{
var segments = 0;
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
{
segments++;
logger.LogInformation(
"Whisper emitted segment at {Start}: {Text}",
segment.Start,
segment.Text);
yield return segment;
}
logger.LogInformation(
"Processed final Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
offset,
windowDuration,
segments);
}
}
private async IAsyncEnumerable<TranscriptionSegment> ProcessWindow(
WhisperProcessor processor,
IReadOnlyList<AudioChunk> chunks,
TimeSpan offset,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var stream = CreateWaveStream(chunks);
await foreach (var result in processor.ProcessAsync(stream, cancellationToken))
{
var text = result.Text?.Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
if (IsControlMarker(text))
{
logger.LogInformation("Whisper emitted control marker {Marker}; skipping transcript output", text);
continue;
}
yield return new TranscriptionSegment(offset + result.Start, offset + result.End, "Unknown", text);
}
}
private static bool IsControlMarker(string text)
{
return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]";
}
private MemoryStream CreateWaveStream(IReadOnlyList<AudioChunk> chunks)
{
var first = chunks[0];
var stream = new MemoryStream();
var pcm = new MemoryStream();
foreach (var chunk in chunks)
{
if (chunk.SampleRate != first.SampleRate || chunk.Channels != first.Channels)
{
logger.LogWarning("Dropping audio chunk with mismatched format while creating Whisper window");
continue;
}
pcm.Write(chunk.Pcm, 0, chunk.Pcm.Length);
}
WriteWaveHeader(stream, first.SampleRate, first.Channels, (int)pcm.Length);
pcm.Position = 0;
pcm.CopyTo(stream);
stream.Position = 0;
return stream;
}
private static void WriteWaveHeader(Stream stream, int sampleRate, int channels, int dataLength)
{
using var writer = new BinaryWriter(stream, System.Text.Encoding.ASCII, leaveOpen: true);
writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"));
writer.Write(36 + dataLength);
writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE"));
writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt "));
writer.Write(16);
writer.Write((short)1);
writer.Write((short)channels);
writer.Write(sampleRate);
writer.Write(sampleRate * channels * sizeof(short));
writer.Write((short)(channels * sizeof(short)));
writer.Write((short)16);
writer.Write(System.Text.Encoding.ASCII.GetBytes("data"));
writer.Write(dataLength);
}
}