Manuel/meeting-assistant
Public Access
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity Open in Coder

Add resilient Azure offline transcription backlog
PR and Push Build/Test / build-and-test (push) Successful in 18m41s
Details

Browse Source
This commit is contained in:
codex
2026-06-15 17:26:34 +02:00
parent b22754ce5d
commit b774ccc375
35 changed files with 2456 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs
+275 -84
View File
@@ -33,69 +33,78 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
yield break;
}
var firstChunk = enumerator.Current;
using var pushStream = AudioInputStream.CreatePushStream(
AudioStreamFormat.GetWaveFormatPCM(
(uint)firstChunk.SampleRate,
16,
(byte)firstChunk.Channels));
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
var speechConfig = CreateSpeechConfig();
using var recognizer = CreateTranscriber(speechConfig, audioConfig);
AddPhraseList(recognizer, pipelineOptions.DictationWords);
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
var nextChunk = enumerator.Current;
var reconnectAttempt = 0;
var markerId = "";
var disconnectedMarkerWritten = false;
recognizer.Transcribed += (_, args) =>
while (true)
{
if (args.Result.Reason != ResultReason.RecognizedSpeech
|| string.IsNullOrWhiteSpace(args.Result.Text))
{
return;
}
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
var segment = new TranscriptionSegment(
start,
start + args.Result.Duration,
NormalizeSpeakerId(args.Result.SpeakerId),
args.Result.Text.Trim());
logger.LogInformation(
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
segments.Writer.TryWrite(segment);
};
recognizer.Canceled += (_, args) =>
{
if (args.Reason == CancellationReason.Error)
{
segments.Writer.TryComplete(new InvalidOperationException(
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
return;
}
segments.Writer.TryComplete();
};
await recognizer.StartTranscribingAsync();
var pumpTask = Task.Run(
() => PumpAudioAsync(
firstChunk,
var session = StartSession(
nextChunk,
enumerator,
pushStream,
recognizer,
segments.Writer,
options.AzureSpeech.RecognitionStopTimeout,
cancellationToken),
CancellationToken.None);
pipelineOptions,
markerId,
cancellationToken);
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
await foreach (var segment in session.Segments.ReadAllAsync(cancellationToken))
{
if (segment.ReplacesMarkerId is not null)
{
reconnectAttempt = 0;
disconnectedMarkerWritten = false;
}
yield return segment;
}
var result = await session.Completion.WaitAsync(cancellationToken);
if (result.IsCompleted)
{
yield break;
}
reconnectAttempt++;
markerId = string.IsNullOrWhiteSpace(markerId)
? $"azure-reconnect-{Guid.NewGuid():N}"
: markerId;
nextChunk = result.NextChunk ?? nextChunk;
if (reconnectAttempt <= Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker))
{
logger.LogWarning(
"Azure Speech reconnecting after connection interruption, attempt {Attempt}/{MaxAttempts}: {ErrorDetails}",
reconnectAttempt,
Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker),
result.ErrorDetails);
yield return new TranscriptionSegment(
TimeSpan.Zero,
TimeSpan.Zero,
"System",
$"<Reconnecting... {reconnectAttempt}/{Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker)}>",
TranscriptionSegmentKind.Marker,
MarkerId: markerId);
}
else if (!disconnectedMarkerWritten)
{
disconnectedMarkerWritten = true;
logger.LogWarning(
"Azure Speech remains disconnected after {AttemptCount} reconnect attempt(s); recording continues and transcription will retry: {ErrorDetails}",
reconnectAttempt - 1,
result.ErrorDetails);
yield return new TranscriptionSegment(
TimeSpan.Zero,
TimeSpan.Zero,
"System",
"<Azure Speech disconnected. The meeting can continue; transcription and summarization will continue once Azure Speech reconnects.>",
TranscriptionSegmentKind.Marker,
MarkerId: markerId);
}
if (options.AzureSpeech.ReconnectDelay > TimeSpan.Zero)
{
await Task.Delay(options.AzureSpeech.ReconnectDelay, cancellationToken);
}
}
await pumpTask.WaitAsync(cancellationToken);
}
finally
{
@@ -110,6 +119,117 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
}
}
private AzureSpeechSession StartSession(
AudioChunk firstChunk,
IAsyncEnumerator<AudioChunk> audio,
SpeechRecognitionPipelineOptions pipelineOptions,
string reconnectMarkerId,
CancellationToken cancellationToken)
{
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
var completion = Task.Run(
() => RunSessionAsync(
firstChunk,
audio,
pipelineOptions,
reconnectMarkerId,
segments.Writer,
cancellationToken),
CancellationToken.None);
return new AzureSpeechSession(segments.Reader, completion);
}
private async Task<AzureSpeechSessionResult> RunSessionAsync(
AudioChunk firstChunk,
IAsyncEnumerator<AudioChunk> audio,
SpeechRecognitionPipelineOptions pipelineOptions,
string reconnectMarkerId,
ChannelWriter<TranscriptionSegment> segments,
CancellationToken cancellationToken)
{
try
{
using var pushStream = AudioInputStream.CreatePushStream(
AudioStreamFormat.GetWaveFormatPCM(
(uint)firstChunk.SampleRate,
16,
(byte)firstChunk.Channels));
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
var speechConfig = CreateSpeechConfig();
using var recognizer = CreateTranscriber(speechConfig, audioConfig);
AddPhraseList(recognizer, pipelineOptions.DictationWords);
var sessionStop = new TaskCompletionSource<AzureSpeechSessionResult>(
TaskCreationOptions.RunContinuationsAsynchronously);
var pendingReconnectMarkerId = reconnectMarkerId;
recognizer.Transcribed += (_, args) =>
{
if (args.Result.Reason != ResultReason.RecognizedSpeech
|| string.IsNullOrWhiteSpace(args.Result.Text))
{
return;
}
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
var replacesMarkerId = string.IsNullOrWhiteSpace(pendingReconnectMarkerId)
? null
: Interlocked.Exchange(ref pendingReconnectMarkerId, null);
var segment = new TranscriptionSegment(
start,
start + args.Result.Duration,
NormalizeSpeakerId(args.Result.SpeakerId),
args.Result.Text.Trim(),
ReplacesMarkerId: replacesMarkerId);
logger.LogInformation(
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
segments.TryWrite(segment);
};
recognizer.Canceled += (_, args) =>
{
if (args.Reason == CancellationReason.Error)
{
logger.LogWarning(
"Azure Speech recognition canceled with {ErrorCode}: {ErrorDetails}",
args.ErrorCode,
args.ErrorDetails);
sessionStop.TrySetResult(AzureSpeechSessionResult.Reconnect(
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
return;
}
sessionStop.TrySetResult(AzureSpeechSessionResult.Completed());
};
recognizer.SessionStopped += (_, _) => sessionStop.TrySetResult(AzureSpeechSessionResult.Completed());
await recognizer.StartTranscribingAsync();
return await PumpAudioAsync(
firstChunk,
audio,
pushStream,
recognizer,
sessionStop.Task,
options.AzureSpeech.RecognitionStopTimeout,
cancellationToken);
}
catch (Exception exception) when (IsTransientConnectionException(exception))
{
logger.LogWarning(exception, "Azure Speech session failed transiently; reconnecting.");
return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message);
}
catch (Exception exception)
{
segments.TryComplete(exception);
throw;
}
finally
{
segments.TryComplete();
}
}
internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig)
{
var languages = GetAutoDetectLanguages(options.AzureSpeech);
@@ -234,47 +354,118 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
: "Continuous";
}
private static async Task PumpAudioAsync(
private static async Task<AzureSpeechSessionResult> PumpAudioAsync(
AudioChunk firstChunk,
IAsyncEnumerator<AudioChunk> audio,
PushAudioInputStream pushStream,
ConversationTranscriber recognizer,
ChannelWriter<TranscriptionSegment> segments,
Task<AzureSpeechSessionResult> sessionStop,
TimeSpan recognitionStopTimeout,
CancellationToken cancellationToken)
{
WriteChunk(pushStream, firstChunk, firstChunk);
while (true)
{
var moveNextTask = audio.MoveNextAsync().AsTask();
var completedTask = await Task.WhenAny(moveNextTask, sessionStop);
if (completedTask == sessionStop)
{
var stop = await sessionStop;
if (stop.IsCompleted)
{
return AzureSpeechSessionResult.Completed();
}
if (!await moveNextTask)
{
return AzureSpeechSessionResult.Completed();
}
return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails);
}
if (!await moveNextTask)
{
break;
}
if (sessionStop.IsCompleted)
{
var stop = await sessionStop;
if (!stop.IsCompleted)
{
return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails);
}
}
cancellationToken.ThrowIfCancellationRequested();
WriteChunk(pushStream, firstChunk, audio.Current);
}
pushStream.Close();
try
{
WriteChunk(pushStream, firstChunk, firstChunk);
while (await audio.MoveNextAsync())
var stopTask = recognizer.StopTranscribingAsync();
if (recognitionStopTimeout > TimeSpan.Zero)
{
cancellationToken.ThrowIfCancellationRequested();
WriteChunk(pushStream, firstChunk, audio.Current);
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
}
pushStream.Close();
try
else
{
var stopTask = recognizer.StopTranscribingAsync();
if (recognitionStopTimeout > TimeSpan.Zero)
{
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
}
else
{
await stopTask.WaitAsync(cancellationToken);
}
await stopTask.WaitAsync(cancellationToken);
}
catch (TimeoutException)
{
// The SDK can outlive short diagnostic streams while waiting for final service events.
}
segments.TryComplete();
}
catch (Exception exception)
catch (TimeoutException)
{
segments.TryComplete(exception);
// The SDK can outlive short diagnostic streams while waiting for final service events.
}
catch (Exception exception) when (IsTransientConnectionException(exception))
{
return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message);
}
return AzureSpeechSessionResult.Completed();
}
private static bool IsTransientConnectionException(Exception exception)
{
if (exception is OperationCanceledException)
{
return false;
}
return exception is TimeoutException
or IOException
or HttpRequestException
or System.Net.Sockets.SocketException
|| exception.Message.Contains("connection", StringComparison.OrdinalIgnoreCase)
|| exception.Message.Contains("websocket", StringComparison.OrdinalIgnoreCase)
|| exception.Message.Contains("network", StringComparison.OrdinalIgnoreCase)
|| exception.Message.Contains("transport", StringComparison.OrdinalIgnoreCase);
}
private sealed record AzureSpeechSession(
ChannelReader<TranscriptionSegment> Segments,
Task<AzureSpeechSessionResult> Completion);
private sealed record AzureSpeechSessionResult(
bool IsCompleted,
AudioChunk? NextChunk,
string? ErrorDetails)
{
public static AzureSpeechSessionResult Completed()
{
return new AzureSpeechSessionResult(true, null, null);
}
public static AzureSpeechSessionResult Reconnect(AudioChunk nextChunk, string? errorDetails)
{
return new AzureSpeechSessionResult(false, nextChunk, errorDetails);
}
public static AzureSpeechSessionResult Reconnect(string? errorDetails)
{
return new AzureSpeechSessionResult(false, null, errorDetails);
}
}
MeetingAssistant/Transcription/TranscriptionSegment.cs
+14 -1
View File
@@ -1,3 +1,16 @@
namespace MeetingAssistant.Transcription;
public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text);
public sealed record TranscriptionSegment(
TimeSpan Start,
TimeSpan End,
string Speaker,
string Text,
TranscriptionSegmentKind Kind = TranscriptionSegmentKind.Speech,
string? MarkerId = null,
string? ReplacesMarkerId = null);
public enum TranscriptionSegmentKind
{
Speech,
Marker
}