Public Access
Add resilient Azure offline transcription backlog
PR and Push Build/Test / build-and-test (push) Successful in 18m41s
PR and Push Build/Test / build-and-test (push) Successful in 18m41s
This commit is contained in:
@@ -223,6 +223,10 @@ public sealed class AzureSpeechOptions
|
||||
|
||||
public TimeSpan RecognitionStopTimeout { get; set; } = TimeSpan.FromMinutes(3);
|
||||
|
||||
public int ReconnectAttemptsBeforeDisconnectedMarker { get; set; } = 5;
|
||||
|
||||
public TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
public bool DiarizeIntermediateResults { get; set; } = true;
|
||||
|
||||
public string? PostProcessingOption { get; set; }
|
||||
|
||||
@@ -37,6 +37,7 @@ builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
|
||||
builder.Services.AddSingleton<IMeetingNoteOpener, ObsidianMeetingNoteOpener>();
|
||||
builder.Services.AddSingleton<IMeetingArtifactStore, MarkdownMeetingArtifactStore>();
|
||||
builder.Services.AddSingleton<IMeetingRunArtifactCleaner, MeetingRunArtifactCleaner>();
|
||||
builder.Services.AddSingleton<IOfflineTranscriptionBacklog, FileOfflineTranscriptionBacklog>();
|
||||
builder.Services.AddSingleton<IRecordedAudioStore, TemporaryRecordedAudioStore>();
|
||||
builder.Services.AddSingleton<IDictationWordStore, MarkdownDictationWordStore>();
|
||||
builder.Services.AddSingleton<IMeetingInactivityClock, SystemMeetingInactivityClock>();
|
||||
@@ -116,9 +117,11 @@ builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, AzureSpeechReco
|
||||
builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, FunAsrSpeechRecognitionPipelineBuilder>();
|
||||
builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, WhisperLocalSpeechRecognitionPipelineBuilder>();
|
||||
builder.Services.AddSingleton<ISpeechRecognitionPipelineFactory, ConfiguredSpeechRecognitionPipelineFactory>();
|
||||
builder.Services.AddSingleton<OfflineTranscriptionBacklogProcessor>();
|
||||
builder.Services.AddSingleton<MeetingRecordingCoordinator>();
|
||||
builder.Services.AddHostedService<CalendarRecordingPromptScheduler>();
|
||||
builder.Services.AddHostedService<TemporaryRecordingCleanupHostedService>();
|
||||
builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
|
||||
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
||||
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
||||
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBacklog
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<FileOfflineTranscriptionBacklog> logger;
|
||||
|
||||
public FileOfflineTranscriptionBacklog(
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<FileOfflineTranscriptionBacklog> logger)
|
||||
{
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = GetBacklogFolder(options);
|
||||
Directory.CreateDirectory(folder);
|
||||
var path = GetItemPath(folder, item.Id);
|
||||
await File.WriteAllTextAsync(path, JsonSerializer.Serialize(item, JsonOptions), cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Queued offline transcription backlog item {BacklogItemId} for recording {RecordingPath}",
|
||||
item.Id,
|
||||
item.AudioPath);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = GetBacklogFolder(options);
|
||||
if (!Directory.Exists(folder))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var items = new List<OfflineTranscriptionBacklogItem>();
|
||||
foreach (var path in Directory.EnumerateFiles(folder, "*.json", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var content = await File.ReadAllTextAsync(path, cancellationToken);
|
||||
if (JsonSerializer.Deserialize<OfflineTranscriptionBacklogItem>(content, JsonOptions) is { } item)
|
||||
{
|
||||
items.Add(item);
|
||||
}
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read offline transcription backlog item {BacklogItemPath}", path);
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read offline transcription backlog item {BacklogItemPath}", path);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
public async Task CompleteAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = GetBacklogFolder(options);
|
||||
var path = GetItemPath(folder, id);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
OfflineTranscriptionBacklogItem? item = null;
|
||||
try
|
||||
{
|
||||
item = JsonSerializer.Deserialize<OfflineTranscriptionBacklogItem>(
|
||||
await File.ReadAllTextAsync(path, cancellationToken),
|
||||
JsonOptions);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path);
|
||||
}
|
||||
|
||||
File.Delete(path);
|
||||
if (item is not null && File.Exists(item.AudioPath))
|
||||
{
|
||||
DeleteCompletedAudio(item.AudioPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetBacklogFolder(MeetingAssistantOptions options)
|
||||
{
|
||||
return Path.Combine(VaultPath.Resolve(options.Recording.TemporaryRecordingsFolder), "offline-transcription-backlog");
|
||||
}
|
||||
|
||||
private static string GetItemPath(string folder, string id)
|
||||
{
|
||||
return Path.Combine(folder, $"{id}.json");
|
||||
}
|
||||
|
||||
private void DeleteCompletedAudio(string audioPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(audioPath);
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not delete completed offline transcription recording file {RecordingPath}",
|
||||
audioPath);
|
||||
}
|
||||
catch (UnauthorizedAccessException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not delete completed offline transcription recording file {RecordingPath}",
|
||||
audioPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,13 +14,30 @@ public interface ITranscriptStore
|
||||
|
||||
Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken)
|
||||
{
|
||||
return AppendLineAsync(
|
||||
return AppendLineAndDiscardReferenceAsync(
|
||||
session,
|
||||
TranscriptLineFormatter.Format(segment).Line,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken);
|
||||
private async Task AppendLineAndDiscardReferenceAsync(
|
||||
TranscriptSession session,
|
||||
string line,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_ = await AppendLineAsync(session, line, cancellationToken);
|
||||
}
|
||||
|
||||
Task<TranscriptLineReference> AppendLineAsync(
|
||||
TranscriptSession session,
|
||||
string line,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
string replacementLine,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task ReplaceAsync(
|
||||
TranscriptSession session,
|
||||
@@ -47,11 +64,18 @@ public interface ITranscriptStore
|
||||
|
||||
public sealed record TranscriptSession(string TranscriptPath);
|
||||
|
||||
public sealed record TranscriptLineReference(string TranscriptPath, int BodyLineIndex, string OriginalLine);
|
||||
|
||||
public static class TranscriptLineFormatter
|
||||
{
|
||||
public static FormattedTranscriptLine Format(TranscriptionSegment segment)
|
||||
{
|
||||
var speaker = NormalizeSpeaker(segment.Speaker);
|
||||
if (segment.Kind == TranscriptionSegmentKind.Marker)
|
||||
{
|
||||
return new FormattedTranscriptLine(segment.Text, speaker);
|
||||
}
|
||||
|
||||
return new FormattedTranscriptLine(
|
||||
$"[{segment.Start:hh\\:mm\\:ss}] {speaker}: {segment.Text}",
|
||||
speaker);
|
||||
|
||||
@@ -31,6 +31,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly IMeetingRunArtifactCleaner artifactCleaner;
|
||||
private readonly IMeetingInactivityPromptService inactivityPromptService;
|
||||
private readonly IMeetingInactivityClock inactivityClock;
|
||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
@@ -59,7 +60,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
IMeetingScreenshotService? screenshotService = null,
|
||||
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
||||
IMeetingInactivityPromptService? inactivityPromptService = null,
|
||||
IMeetingInactivityClock? inactivityClock = null)
|
||||
IMeetingInactivityClock? inactivityClock = null,
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
||||
{
|
||||
this.audioSource = audioSource;
|
||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||
@@ -79,6 +81,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
|
||||
this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService();
|
||||
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -301,7 +304,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation");
|
||||
if (IsAzureSpeechRun(run))
|
||||
{
|
||||
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
|
||||
run.MarkQueuedForOfflineTranscription();
|
||||
await offlineTranscriptionBacklog.EnqueueAsync(CreateOfflineBacklogItem(run), cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation");
|
||||
}
|
||||
|
||||
run.CancelTranscription();
|
||||
await run.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
@@ -424,6 +437,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
await run.Pipeline.CompleteAsync(cancellationToken);
|
||||
await run.WaitForLiveTranscriptTasksAsync(cancellationToken);
|
||||
await run.WaitForTranscriptWorkflowTasksAsync(cancellationToken);
|
||||
|
||||
run.ResetSpeakerIdentification();
|
||||
run.MarkProfileSwitched();
|
||||
@@ -471,6 +485,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
await captureTask;
|
||||
await run.Pipeline.CompleteAsync(run.TranscriptionCancellation);
|
||||
await run.WaitForLiveTranscriptTasksAsync(run.TranscriptionCancellation);
|
||||
await run.WaitForTranscriptWorkflowTasksAsync(run.TranscriptionCancellation);
|
||||
run.CancelLiveIdentification();
|
||||
await liveIdentificationTask;
|
||||
await run.RecordedAudio.DisposeAsync();
|
||||
@@ -528,7 +543,15 @@ public sealed class MeetingRecordingCoordinator
|
||||
finally
|
||||
{
|
||||
run.CancelLiveIdentification();
|
||||
await run.RecordedAudio.DeleteAsync(CancellationToken.None);
|
||||
if (run.IsQueuedForOfflineTranscription)
|
||||
{
|
||||
await run.RecordedAudio.DisposeAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
await run.RecordedAudio.DeleteAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
await run.DisposePipelinesAsync();
|
||||
await CompleteRunAsync(run);
|
||||
}
|
||||
@@ -582,6 +605,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
{
|
||||
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
|
||||
{
|
||||
if (segment.Kind == TranscriptionSegmentKind.Marker)
|
||||
{
|
||||
await AppendTranscriptSegmentAsync(run, segment, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (segment.ReplacesMarkerId is not null)
|
||||
{
|
||||
run.ResetSpeakerIdentification();
|
||||
}
|
||||
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
var sample = run.TryAddSpeakerSample(segment);
|
||||
if (sample is not null)
|
||||
@@ -607,8 +641,69 @@ public sealed class MeetingRecordingCoordinator
|
||||
TranscriptionSegment segment,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var line = await TransformTranscriptLineAsync(run, segment, cancellationToken);
|
||||
await transcriptStore.AppendLineAsync(run.Session, line, cancellationToken);
|
||||
var formatted = TranscriptLineFormatter.Format(segment);
|
||||
var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId &&
|
||||
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference)
|
||||
? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken)
|
||||
: segment.Kind == TranscriptionSegmentKind.Marker &&
|
||||
segment.MarkerId is { } existingMarkerId &&
|
||||
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference)
|
||||
? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken)
|
||||
: await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
|
||||
if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId)
|
||||
{
|
||||
run.SetTranscriptMarker(markerId, lineReference);
|
||||
return;
|
||||
}
|
||||
|
||||
run.AddTranscriptWorkflowTask(
|
||||
() => ApplyTranscriptLineWorkflowAsync(run, lineReference, formatted, run.TranscriptionCancellation));
|
||||
}
|
||||
|
||||
private async Task<TranscriptLineReference> ReplaceTranscriptMarkerAsync(
|
||||
RecordingRun run,
|
||||
TranscriptLineReference markerReference,
|
||||
FormattedTranscriptLine formatted,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await transcriptStore.ReplaceLineAsync(run.Session, markerReference, formatted.Line, cancellationToken);
|
||||
return markerReference with { OriginalLine = formatted.Line };
|
||||
}
|
||||
|
||||
private async Task ApplyTranscriptLineWorkflowAsync(
|
||||
RecordingRun run,
|
||||
TranscriptLineReference lineReference,
|
||||
FormattedTranscriptLine formatted,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var transformed = await TryTransformTranscriptLineAsync(
|
||||
run,
|
||||
formatted.Line,
|
||||
formatted.Speaker,
|
||||
cancellationToken);
|
||||
if (!string.Equals(formatted.Line, transformed, StringComparison.Ordinal))
|
||||
{
|
||||
try
|
||||
{
|
||||
await transcriptStore.ReplaceLineAsync(
|
||||
run.Session,
|
||||
lineReference,
|
||||
transformed,
|
||||
cancellationToken);
|
||||
run.RecordTranscriptLineRewrite(formatted.Line, transformed);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Transcript line rewrite failed for speaker {Speaker}; keeping original transcript line",
|
||||
formatted.Speaker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplaceTranscriptSegmentsAsync(
|
||||
@@ -616,28 +711,41 @@ public sealed class MeetingRecordingCoordinator
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var lines = new List<string>(segments.Count);
|
||||
foreach (var segment in segments)
|
||||
{
|
||||
lines.Add(await TransformTranscriptLineAsync(run, segment, cancellationToken));
|
||||
}
|
||||
var lines = segments
|
||||
.Select(segment => run.ApplyTranscriptLineRewrite(TranscriptLineFormatter.Format(segment).Line))
|
||||
.ToList();
|
||||
|
||||
await transcriptStore.ReplaceLinesAsync(run.Session, lines, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<string> TransformTranscriptLineAsync(
|
||||
private async Task<string> TryTransformTranscriptLineAsync(
|
||||
RecordingRun run,
|
||||
TranscriptionSegment segment,
|
||||
string formattedLine,
|
||||
string speaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var formatted = TranscriptLineFormatter.Format(segment);
|
||||
return await meetingWorkflowEngine.TransformTranscriptLineAsync(
|
||||
MeetingWorkflowEvent.TranscriptLine(
|
||||
run.Artifacts,
|
||||
formatted.Line,
|
||||
formatted.Speaker),
|
||||
run.Options,
|
||||
cancellationToken);
|
||||
try
|
||||
{
|
||||
return await meetingWorkflowEngine.TransformTranscriptLineAsync(
|
||||
MeetingWorkflowEvent.TranscriptLine(
|
||||
run.Artifacts,
|
||||
formattedLine,
|
||||
speaker),
|
||||
run.Options,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Transcript line workflow failed for speaker {Speaker}; keeping original transcript line",
|
||||
speaker);
|
||||
return formattedLine;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
||||
@@ -1459,17 +1567,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
string? meetingNotePath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var dictationWords = await dictationWordProvider.ReadOptionalWordsAsync(runOptions, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(meetingNotePath))
|
||||
{
|
||||
return new SpeechRecognitionPipelineOptions(DictationWords: dictationWords);
|
||||
}
|
||||
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
|
||||
var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee));
|
||||
return attendeeCount > 1
|
||||
? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords)
|
||||
: new SpeechRecognitionPipelineOptions(DictationWords: dictationWords);
|
||||
var meetingNote = string.IsNullOrWhiteSpace(meetingNotePath)
|
||||
? null
|
||||
: await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
|
||||
return await RecordingSpeechRecognitionPipelineOptions.BuildAsync(
|
||||
dictationWordProvider,
|
||||
runOptions,
|
||||
meetingNote,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private LaunchProfile ResolveProfile(string? launchProfileName)
|
||||
@@ -1515,6 +1620,25 @@ public sealed class MeetingRecordingCoordinator
|
||||
return minimumDuration > TimeSpan.Zero && stoppedAt - run.StartedAt < minimumDuration;
|
||||
}
|
||||
|
||||
private static bool IsAzureSpeechRun(RecordingRun run)
|
||||
{
|
||||
return run.Options.Recording.TranscriptionProvider.Equals("azure-speech", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static OfflineTranscriptionBacklogItem CreateOfflineBacklogItem(RecordingRun run)
|
||||
{
|
||||
return new OfflineTranscriptionBacklogItem(
|
||||
Id: Guid.NewGuid().ToString("N"),
|
||||
AudioPath: run.RecordedAudio.AudioPath,
|
||||
TranscriptPath: run.Session.TranscriptPath,
|
||||
MeetingNotePath: run.MeetingNotePath,
|
||||
AssistantContextPath: run.Artifacts.AssistantContextPath,
|
||||
SummaryPath: run.Artifacts.SummaryPath,
|
||||
StartedAt: run.StartedAt,
|
||||
InferredEndTime: run.InferredEndTime,
|
||||
LaunchProfileName: run.LaunchProfileName);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<TimeSpan> GetInactivityPromptThresholds(
|
||||
RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
@@ -1566,10 +1690,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly object stateGate = new();
|
||||
private readonly object liveSegmentsGate = new();
|
||||
private readonly object liveTranscriptTasksGate = new();
|
||||
private readonly object transcriptWorkflowTasksGate = new();
|
||||
private readonly object liveIdentificationGate = new();
|
||||
private readonly object transcriptActivityGate = new();
|
||||
private readonly List<TranscriptionSegment> liveSegments = [];
|
||||
private readonly List<Task> liveTranscriptTasks = [];
|
||||
private Task transcriptWorkflowTail = Task.CompletedTask;
|
||||
private readonly ConcurrentDictionary<string, TranscriptLineReference> transcriptMarkers = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, string> transcriptLineRewrites = new(StringComparer.Ordinal);
|
||||
private readonly List<ISpeechRecognitionPipeline> pipelines = [];
|
||||
private readonly ConcurrentDictionary<string, string> speakerMappings = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly SpeakerAudioSampleCollector speakerSampleCollector;
|
||||
@@ -1656,6 +1784,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public bool IsAborted { get; private set; }
|
||||
|
||||
public bool IsQueuedForOfflineTranscription { get; private set; }
|
||||
|
||||
public bool HasSwitchedProfile { get; private set; }
|
||||
|
||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||
@@ -1689,6 +1819,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
TranscriptionCancellationSource.Cancel();
|
||||
}
|
||||
|
||||
public void MarkQueuedForOfflineTranscription()
|
||||
{
|
||||
IsQueuedForOfflineTranscription = true;
|
||||
}
|
||||
|
||||
public void CancelLiveIdentification()
|
||||
{
|
||||
LiveIdentificationCancellationSource.Cancel();
|
||||
@@ -1745,6 +1880,41 @@ public sealed class MeetingRecordingCoordinator
|
||||
return Task.WhenAll(tasks).WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void AddTranscriptWorkflowTask(Func<Task> taskFactory)
|
||||
{
|
||||
lock (transcriptWorkflowTasksGate)
|
||||
{
|
||||
transcriptWorkflowTail = RunAfterAsync(transcriptWorkflowTail, taskFactory);
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitForTranscriptWorkflowTasksAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Task task;
|
||||
lock (transcriptWorkflowTasksGate)
|
||||
{
|
||||
task = transcriptWorkflowTail;
|
||||
}
|
||||
|
||||
return task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task RunAfterAsync(Task previous, Func<Task> next)
|
||||
{
|
||||
await previous.ConfigureAwait(false);
|
||||
await next().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public void SetTranscriptMarker(string markerId, TranscriptLineReference lineReference)
|
||||
{
|
||||
transcriptMarkers[markerId] = lineReference;
|
||||
}
|
||||
|
||||
public bool TryTakeTranscriptMarker(string markerId, out TranscriptLineReference lineReference)
|
||||
{
|
||||
return transcriptMarkers.TryRemove(markerId, out lineReference!);
|
||||
}
|
||||
|
||||
public async Task DisposePipelinesAsync()
|
||||
{
|
||||
foreach (var pipeline in pipelines.Distinct())
|
||||
@@ -1817,6 +1987,18 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
}
|
||||
|
||||
public void RecordTranscriptLineRewrite(string originalLine, string replacementLine)
|
||||
{
|
||||
transcriptLineRewrites[originalLine] = replacementLine;
|
||||
}
|
||||
|
||||
public string ApplyTranscriptLineRewrite(string line)
|
||||
{
|
||||
return transcriptLineRewrites.TryGetValue(line, out var replacement)
|
||||
? replacement
|
||||
: line;
|
||||
}
|
||||
|
||||
public void RecordTranscriptActivity(TranscriptionSegment segment, DateTimeOffset arrivedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment.Text))
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IOfflineTranscriptionBacklog
|
||||
{
|
||||
Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task CompleteAsync(string id, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record OfflineTranscriptionBacklogItem(
|
||||
string Id,
|
||||
string AudioPath,
|
||||
string TranscriptPath,
|
||||
string MeetingNotePath,
|
||||
string AssistantContextPath,
|
||||
string SummaryPath,
|
||||
DateTimeOffset StartedAt,
|
||||
DateTimeOffset? InferredEndTime,
|
||||
string LaunchProfileName);
|
||||
|
||||
public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBacklog
|
||||
{
|
||||
public static readonly NoopOfflineTranscriptionBacklog Instance = new();
|
||||
|
||||
private NoopOfflineTranscriptionBacklog()
|
||||
{
|
||||
}
|
||||
|
||||
public Task EnqueueAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<OfflineTranscriptionBacklogItem>>([]);
|
||||
}
|
||||
|
||||
public Task CompleteAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class OfflineTranscriptionBacklogHostedService : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1);
|
||||
|
||||
private readonly OfflineTranscriptionBacklogProcessor processor;
|
||||
private readonly ILogger<OfflineTranscriptionBacklogHostedService> logger;
|
||||
|
||||
public OfflineTranscriptionBacklogHostedService(
|
||||
OfflineTranscriptionBacklogProcessor processor,
|
||||
ILogger<OfflineTranscriptionBacklogHostedService> logger)
|
||||
{
|
||||
this.processor = processor;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var processed = await processor.ProcessPendingAsync(stoppingToken);
|
||||
if (processed > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Processed {ProcessedCount} offline transcription backlog item(s)",
|
||||
processed);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Offline transcription backlog processing failed; it will be retried later");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(RetryDelay, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class OfflineTranscriptionBacklogProcessor
|
||||
{
|
||||
private const int MaxChunkDurationMilliseconds = 1000;
|
||||
|
||||
private readonly IOfflineTranscriptionBacklog backlog;
|
||||
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
|
||||
private readonly ITranscriptStore transcriptStore;
|
||||
private readonly IMeetingNoteStore meetingNoteStore;
|
||||
private readonly IMeetingArtifactStore meetingArtifactStore;
|
||||
private readonly IMeetingSummaryPipeline summaryPipeline;
|
||||
private readonly IMeetingWorkflowEngine workflowEngine;
|
||||
private readonly IRecordingDictationWordProvider dictationWordProvider;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<OfflineTranscriptionBacklogProcessor> logger;
|
||||
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
||||
|
||||
public OfflineTranscriptionBacklogProcessor(
|
||||
IOfflineTranscriptionBacklog backlog,
|
||||
ISpeechRecognitionPipelineFactory pipelineFactory,
|
||||
ITranscriptStore transcriptStore,
|
||||
IMeetingNoteStore meetingNoteStore,
|
||||
IMeetingArtifactStore meetingArtifactStore,
|
||||
IMeetingSummaryPipeline summaryPipeline,
|
||||
IMeetingWorkflowEngine workflowEngine,
|
||||
IRecordingDictationWordProvider dictationWordProvider,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<OfflineTranscriptionBacklogProcessor> logger,
|
||||
ILaunchProfileOptionsProvider? launchProfiles = null)
|
||||
{
|
||||
this.backlog = backlog;
|
||||
this.pipelineFactory = pipelineFactory;
|
||||
this.transcriptStore = transcriptStore;
|
||||
this.meetingNoteStore = meetingNoteStore;
|
||||
this.meetingArtifactStore = meetingArtifactStore;
|
||||
this.summaryPipeline = summaryPipeline;
|
||||
this.workflowEngine = workflowEngine;
|
||||
this.dictationWordProvider = dictationWordProvider;
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
this.launchProfiles = launchProfiles;
|
||||
}
|
||||
|
||||
public async Task<int> ProcessPendingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var processed = 0;
|
||||
foreach (var item in await backlog.ListAsync(cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (await TryProcessAsync(item, cancellationToken))
|
||||
{
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async Task<bool> TryProcessAsync(
|
||||
OfflineTranscriptionBacklogItem item,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessAsync(item, cancellationToken);
|
||||
await backlog.CompleteAsync(item.Id, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}",
|
||||
item.Id,
|
||||
item.AudioPath);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not process offline transcription backlog item {BacklogItemId}; it will be retried later",
|
||||
item.Id);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessAsync(
|
||||
OfflineTranscriptionBacklogItem item,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var runOptions = ResolveOptions(item.LaunchProfileName);
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
item.MeetingNotePath,
|
||||
item.TranscriptPath,
|
||||
item.AssistantContextPath,
|
||||
item.SummaryPath);
|
||||
var session = new TranscriptSession(item.TranscriptPath);
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(item.MeetingNotePath, cancellationToken);
|
||||
var pipelineOptions = await RecordingSpeechRecognitionPipelineOptions.BuildAsync(
|
||||
dictationWordProvider,
|
||||
runOptions,
|
||||
meetingNote,
|
||||
cancellationToken);
|
||||
|
||||
await using var pipeline = pipelineFactory.Create(item.LaunchProfileName);
|
||||
await pipeline.InitializeAsync(pipelineOptions, cancellationToken);
|
||||
await pipeline.WaitUntilReadyAsync(cancellationToken);
|
||||
await WriteRecordingToPipelineAsync(item.AudioPath, pipeline, cancellationToken);
|
||||
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(
|
||||
item.AudioPath,
|
||||
pipelineOptions,
|
||||
cancellationToken);
|
||||
|
||||
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
|
||||
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
|
||||
|
||||
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
|
||||
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
|
||||
await transcriptStore.UpdateMetadataAsync(session, artifacts, completedMeetingNote, cancellationToken);
|
||||
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(artifacts, completedMeetingNote, cancellationToken);
|
||||
|
||||
await TransitionMeetingAsync(
|
||||
artifacts,
|
||||
runOptions,
|
||||
AssistantContextState.Transcribing,
|
||||
AssistantContextState.Summarizing,
|
||||
cancellationToken);
|
||||
var summaryResult = await summaryPipeline.RunAsync(artifacts, runOptions, cancellationToken);
|
||||
await TransitionMeetingAsync(
|
||||
artifacts,
|
||||
runOptions,
|
||||
AssistantContextState.Summarizing,
|
||||
summaryResult.Succeeded ? AssistantContextState.Finished : AssistantContextState.Error,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private MeetingAssistantOptions ResolveOptions(string? launchProfileName)
|
||||
{
|
||||
if (launchProfiles is not null)
|
||||
{
|
||||
return launchProfiles.GetRequiredProfile(launchProfileName).Options;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
private async Task<List<string>> TransformTranscriptLinesAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions runOptions,
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var lines = new List<string>(segments.Count);
|
||||
foreach (var segment in segments)
|
||||
{
|
||||
var formatted = TranscriptLineFormatter.Format(segment);
|
||||
lines.Add(await TransformTranscriptLineAsync(artifacts, runOptions, formatted, cancellationToken));
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
private async Task<string> TransformTranscriptLineAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions runOptions,
|
||||
FormattedTranscriptLine formatted,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await workflowEngine.TransformTranscriptLineAsync(
|
||||
MeetingWorkflowEvent.TranscriptLine(artifacts, formatted.Line, formatted.Speaker),
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Offline transcript line workflow failed for speaker {Speaker}; keeping original transcript line",
|
||||
formatted.Speaker);
|
||||
return formatted.Line;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TransitionMeetingAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions runOptions,
|
||||
AssistantContextState from,
|
||||
AssistantContextState to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await meetingArtifactStore.UpdateAssistantContextStateAsync(artifacts, to, cancellationToken);
|
||||
await workflowEngine.RunAsync(
|
||||
MeetingWorkflowEvent.StateTransition(artifacts, from, to),
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteRecordingToPipelineAsync(
|
||||
string audioPath,
|
||||
ISpeechRecognitionPipeline pipeline,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var reader = new WaveFileReader(audioPath);
|
||||
var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond;
|
||||
var chunkSize = Math.Max(
|
||||
reader.WaveFormat.BlockAlign,
|
||||
bytesPerSecond * MaxChunkDurationMilliseconds / 1000);
|
||||
chunkSize -= chunkSize % reader.WaveFormat.BlockAlign;
|
||||
|
||||
var buffer = new byte[chunkSize];
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken);
|
||||
if (read == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await pipeline.WriteAsync(
|
||||
new AudioChunk(
|
||||
buffer[..read],
|
||||
reader.WaveFormat.SampleRate,
|
||||
reader.WaveFormat.Channels),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public static class RecordingSpeechRecognitionPipelineOptions
|
||||
{
|
||||
public static async Task<SpeechRecognitionPipelineOptions> BuildAsync(
|
||||
IRecordingDictationWordProvider dictationWordProvider,
|
||||
MeetingAssistantOptions runOptions,
|
||||
MeetingNote? meetingNote,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var dictationWords = await dictationWordProvider.ReadOptionalWordsAsync(runOptions, cancellationToken);
|
||||
var attendeeCount = meetingNote?.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee)) ?? 0;
|
||||
return attendeeCount > 1
|
||||
? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords)
|
||||
: new SpeechRecognitionPipelineOptions(DictationWords: dictationWords);
|
||||
}
|
||||
}
|
||||
@@ -7,13 +7,16 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<TemporaryRecordedAudioStore> logger;
|
||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||
|
||||
public TemporaryRecordedAudioStore(
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<TemporaryRecordedAudioStore> logger)
|
||||
ILogger<TemporaryRecordedAudioStore> logger,
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
||||
{
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||
}
|
||||
|
||||
public Task<IRecordedAudioSink> CreateSessionAsync(CancellationToken cancellationToken)
|
||||
@@ -35,21 +38,28 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
|
||||
return Task.FromResult<IRecordedAudioSink>(new TemporaryRecordedAudioSink(path, format, logger));
|
||||
}
|
||||
|
||||
public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken)
|
||||
public async Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = GetTemporaryRecordingsFolder(options);
|
||||
if (!Directory.Exists(folder))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
return;
|
||||
}
|
||||
|
||||
var queuedAudioPaths = (await offlineTranscriptionBacklog.ListAsync(cancellationToken))
|
||||
.Select(item => item.AudioPath)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var path in Directory.EnumerateFiles(folder, "*.wav", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (queuedAudioPaths.Contains(path))
|
||||
{
|
||||
logger.LogInformation("Keeping queued offline transcription recording file {RecordingPath}", path);
|
||||
continue;
|
||||
}
|
||||
|
||||
DeleteFile(path);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string GetTemporaryRecordingsFolder(MeetingAssistantOptions options)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
@@ -8,6 +9,7 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<VaultTranscriptStore> logger;
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> fileGates = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public VaultTranscriptStore(IOptions<MeetingAssistantOptions> options, ILogger<VaultTranscriptStore> logger)
|
||||
{
|
||||
@@ -36,9 +38,50 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
return new TranscriptSession(path);
|
||||
}
|
||||
|
||||
public Task AppendLineAsync(TranscriptSession session, string line, CancellationToken cancellationToken)
|
||||
public async Task<TranscriptLineReference> AppendLineAsync(
|
||||
TranscriptSession session,
|
||||
string line,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return AppendToBodyAsync(session.TranscriptPath, line + Environment.NewLine, cancellationToken);
|
||||
return await WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
() => AppendToBodyAsync(session.TranscriptPath, line, cancellationToken),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
string replacementLine,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
async () =>
|
||||
{
|
||||
if (!File.Exists(session.TranscriptPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken);
|
||||
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content);
|
||||
var lines = SplitLines(body);
|
||||
if (lineReference.BodyLineIndex < 0 ||
|
||||
lineReference.BodyLineIndex >= lines.Count ||
|
||||
!string.Equals(lines[lineReference.BodyLineIndex], lineReference.OriginalLine, StringComparison.Ordinal))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not replace transcript line in {TranscriptPath} at body line {BodyLineIndex} because the original line no longer matched",
|
||||
session.TranscriptPath,
|
||||
lineReference.BodyLineIndex);
|
||||
return;
|
||||
}
|
||||
|
||||
lines[lineReference.BodyLineIndex] = replacementLine;
|
||||
await WriteBodyAsync(session.TranscriptPath, frontmatter, string.Join(Environment.NewLine, lines), cancellationToken);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task ReplaceLinesAsync(
|
||||
@@ -46,18 +89,21 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
IReadOnlyList<string> replacementLines,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = File.Exists(session.TranscriptPath)
|
||||
? File.ReadAllText(session.TranscriptPath)
|
||||
: "";
|
||||
var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing);
|
||||
var body = "# Meeting Transcript"
|
||||
+ Environment.NewLine
|
||||
+ Environment.NewLine
|
||||
+ string.Concat(replacementLines.Select(line => line + Environment.NewLine));
|
||||
var content = string.IsNullOrWhiteSpace(frontmatter)
|
||||
? body
|
||||
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body;
|
||||
return File.WriteAllTextAsync(session.TranscriptPath, content, cancellationToken);
|
||||
return WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
async () =>
|
||||
{
|
||||
var existing = File.Exists(session.TranscriptPath)
|
||||
? File.ReadAllText(session.TranscriptPath)
|
||||
: "";
|
||||
var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing);
|
||||
var body = "# Meeting Transcript"
|
||||
+ Environment.NewLine
|
||||
+ Environment.NewLine
|
||||
+ string.Concat(replacementLines.Select(line => line + Environment.NewLine));
|
||||
await WriteBodyAsync(session.TranscriptPath, frontmatter, body, cancellationToken);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task UpdateMetadataAsync(
|
||||
@@ -66,50 +112,108 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
MeetingNote meetingNote,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var body = File.Exists(session.TranscriptPath)
|
||||
? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body
|
||||
: "# Meeting Transcript" + Environment.NewLine + Environment.NewLine;
|
||||
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
|
||||
artifacts,
|
||||
meetingNote,
|
||||
MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"),
|
||||
session.TranscriptPath);
|
||||
|
||||
await File.WriteAllTextAsync(
|
||||
await WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
MeetingArtifactFrontmatterRenderer.Render(frontmatter, body),
|
||||
async () =>
|
||||
{
|
||||
var body = File.Exists(session.TranscriptPath)
|
||||
? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body
|
||||
: "# Meeting Transcript" + Environment.NewLine + Environment.NewLine;
|
||||
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
|
||||
artifacts,
|
||||
meetingNote,
|
||||
MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"),
|
||||
session.TranscriptPath);
|
||||
|
||||
await File.WriteAllTextAsync(
|
||||
session.TranscriptPath,
|
||||
MeetingArtifactFrontmatterRenderer.Render(frontmatter, body),
|
||||
cancellationToken);
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task AppendToBodyAsync(
|
||||
private static async Task<TranscriptLineReference> AppendToBodyAsync(
|
||||
string path,
|
||||
string text,
|
||||
string line,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
await File.AppendAllTextAsync(path, text, cancellationToken);
|
||||
return;
|
||||
await File.AppendAllTextAsync(path, line + Environment.NewLine, cancellationToken);
|
||||
return new TranscriptLineReference(path, 0, line);
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(path, cancellationToken);
|
||||
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content);
|
||||
var bodyLineIndex = GetAppendBodyLineIndex(body);
|
||||
if (string.IsNullOrWhiteSpace(frontmatter))
|
||||
{
|
||||
await File.AppendAllTextAsync(path, text, cancellationToken);
|
||||
return;
|
||||
await File.AppendAllTextAsync(path, line + Environment.NewLine, cancellationToken);
|
||||
return new TranscriptLineReference(path, bodyLineIndex, line);
|
||||
}
|
||||
|
||||
var updated = "---"
|
||||
+ Environment.NewLine
|
||||
+ frontmatter
|
||||
+ Environment.NewLine
|
||||
+ "---"
|
||||
+ Environment.NewLine
|
||||
+ Environment.NewLine
|
||||
+ body
|
||||
+ text;
|
||||
await File.WriteAllTextAsync(path, updated, cancellationToken);
|
||||
await WriteBodyAsync(path, frontmatter, body + line + Environment.NewLine, cancellationToken);
|
||||
return new TranscriptLineReference(path, bodyLineIndex, line);
|
||||
}
|
||||
|
||||
private async Task WithFileGateAsync(
|
||||
string path,
|
||||
Func<Task> action,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await WithFileGateAsync(
|
||||
path,
|
||||
async () =>
|
||||
{
|
||||
await action();
|
||||
return true;
|
||||
},
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<T> WithFileGateAsync<T>(
|
||||
string path,
|
||||
Func<Task<T>> action,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = fileGates.GetOrAdd(path, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
return await action();
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteBodyAsync(
|
||||
string path,
|
||||
string frontmatter,
|
||||
string body,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var content = string.IsNullOrWhiteSpace(frontmatter)
|
||||
? body
|
||||
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body;
|
||||
await File.WriteAllTextAsync(path, content, cancellationToken);
|
||||
}
|
||||
|
||||
private static int GetAppendBodyLineIndex(string body)
|
||||
{
|
||||
var lines = SplitLines(body);
|
||||
return lines.Count > 0 && lines[^1].Length == 0
|
||||
? lines.Count - 1
|
||||
: lines.Count;
|
||||
}
|
||||
|
||||
private static List<string> SplitLines(string text)
|
||||
{
|
||||
return text
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Split('\n')
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,8 @@ public enum MeetingTaskbarAction
|
||||
StartRecording,
|
||||
StopRecording,
|
||||
AbortRecording,
|
||||
SwitchProfile
|
||||
SwitchProfile,
|
||||
Exit
|
||||
}
|
||||
|
||||
public sealed record MeetingTaskbarMenu(
|
||||
@@ -61,6 +62,10 @@ public static class MeetingTaskbarMenuBuilder
|
||||
}
|
||||
}
|
||||
|
||||
items.Add(new MeetingTaskbarMenuItem(
|
||||
"Exit",
|
||||
MeetingTaskbarAction.Exit));
|
||||
|
||||
return new MeetingTaskbarMenu(
|
||||
status.State,
|
||||
BuildTooltip(status),
|
||||
@@ -96,3 +101,11 @@ public static class MeetingTaskbarMenuBuilder
|
||||
: $"{text}\t{hotkey.Trim()}";
|
||||
}
|
||||
}
|
||||
|
||||
public static class MeetingTaskbarExitPolicy
|
||||
{
|
||||
public static bool RequiresConfirmation(RecordingStatus status)
|
||||
{
|
||||
return status.State != RecordingProcessState.Idle;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#if WINDOWS
|
||||
using System.Drawing;
|
||||
using System.Windows;
|
||||
using H.NotifyIcon.Core;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Recording;
|
||||
@@ -14,6 +15,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
private readonly MeetingRecordingCoordinator coordinator;
|
||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||
private readonly IWorkflowRulesEditorWindowService rulesEditorWindow;
|
||||
private readonly IHostApplicationLifetime applicationLifetime;
|
||||
private readonly ILogger<UnoTaskbarIconService> logger;
|
||||
private readonly object sync = new();
|
||||
private CancellationTokenSource? refreshCancellation;
|
||||
@@ -28,11 +30,13 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
IWorkflowRulesEditorWindowService rulesEditorWindow,
|
||||
IHostApplicationLifetime applicationLifetime,
|
||||
ILogger<UnoTaskbarIconService> logger)
|
||||
{
|
||||
this.coordinator = coordinator;
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.rulesEditorWindow = rulesEditorWindow;
|
||||
this.applicationLifetime = applicationLifetime;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -182,7 +186,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
var popupMenu = new PopupMenu();
|
||||
for (var index = 0; index < menu.Items.Count; index++)
|
||||
{
|
||||
if (index == 1)
|
||||
if (index == 1 ||
|
||||
(menu.Items[index].Action == MeetingTaskbarAction.Exit &&
|
||||
menu.Items[index - 1].Action != MeetingTaskbarAction.EditRules))
|
||||
{
|
||||
popupMenu.Items.Add(new PopupMenuSeparator());
|
||||
}
|
||||
@@ -222,6 +228,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
case MeetingTaskbarAction.SwitchProfile:
|
||||
await coordinator.ToggleAsync(item.ProfileName, CancellationToken.None);
|
||||
break;
|
||||
case MeetingTaskbarAction.Exit:
|
||||
ExitApplication();
|
||||
break;
|
||||
}
|
||||
|
||||
RefreshVisualState();
|
||||
@@ -247,5 +256,32 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
menu.Items.Select(item => $"{item.Action}:{item.ProfileName}:{item.Text}"));
|
||||
}
|
||||
|
||||
private void ExitApplication()
|
||||
{
|
||||
var status = coordinator.CurrentStatus;
|
||||
if (MeetingTaskbarExitPolicy.RequiresConfirmation(status) && !ConfirmExitDuringProcessing(status.State))
|
||||
{
|
||||
logger.LogInformation("Taskbar exit canceled while meeting state was {State}", status.State);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation("Taskbar exit requested while meeting state was {State}", status.State);
|
||||
applicationLifetime.StopApplication();
|
||||
}
|
||||
|
||||
private static bool ConfirmExitDuringProcessing(RecordingProcessState state)
|
||||
{
|
||||
var activity = state == RecordingProcessState.Recording
|
||||
? "A meeting is still recording and transcribing."
|
||||
: "A stopped meeting is still transcribing, recognizing speakers, or summarizing.";
|
||||
var result = MessageBox.Show(
|
||||
$"{activity}\n\nExit Meeting Assistant anyway?",
|
||||
"Exit Meeting Assistant",
|
||||
MessageBoxButton.YesNo,
|
||||
MessageBoxImage.Warning,
|
||||
MessageBoxResult.No);
|
||||
|
||||
return result == MessageBoxResult.Yes;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -33,69 +33,78 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
|
||||
yield break;
|
||||
}
|
||||
|
||||
var firstChunk = enumerator.Current;
|
||||
using var pushStream = AudioInputStream.CreatePushStream(
|
||||
AudioStreamFormat.GetWaveFormatPCM(
|
||||
(uint)firstChunk.SampleRate,
|
||||
16,
|
||||
(byte)firstChunk.Channels));
|
||||
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
|
||||
var speechConfig = CreateSpeechConfig();
|
||||
using var recognizer = CreateTranscriber(speechConfig, audioConfig);
|
||||
AddPhraseList(recognizer, pipelineOptions.DictationWords);
|
||||
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
var nextChunk = enumerator.Current;
|
||||
var reconnectAttempt = 0;
|
||||
var markerId = "";
|
||||
var disconnectedMarkerWritten = false;
|
||||
|
||||
recognizer.Transcribed += (_, args) =>
|
||||
while (true)
|
||||
{
|
||||
if (args.Result.Reason != ResultReason.RecognizedSpeech
|
||||
|| string.IsNullOrWhiteSpace(args.Result.Text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
|
||||
var segment = new TranscriptionSegment(
|
||||
start,
|
||||
start + args.Result.Duration,
|
||||
NormalizeSpeakerId(args.Result.SpeakerId),
|
||||
args.Result.Text.Trim());
|
||||
logger.LogInformation(
|
||||
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
|
||||
segment.Start,
|
||||
segment.Speaker,
|
||||
segment.Text);
|
||||
segments.Writer.TryWrite(segment);
|
||||
};
|
||||
recognizer.Canceled += (_, args) =>
|
||||
{
|
||||
if (args.Reason == CancellationReason.Error)
|
||||
{
|
||||
segments.Writer.TryComplete(new InvalidOperationException(
|
||||
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
|
||||
return;
|
||||
}
|
||||
|
||||
segments.Writer.TryComplete();
|
||||
};
|
||||
|
||||
await recognizer.StartTranscribingAsync();
|
||||
var pumpTask = Task.Run(
|
||||
() => PumpAudioAsync(
|
||||
firstChunk,
|
||||
var session = StartSession(
|
||||
nextChunk,
|
||||
enumerator,
|
||||
pushStream,
|
||||
recognizer,
|
||||
segments.Writer,
|
||||
options.AzureSpeech.RecognitionStopTimeout,
|
||||
cancellationToken),
|
||||
CancellationToken.None);
|
||||
pipelineOptions,
|
||||
markerId,
|
||||
cancellationToken);
|
||||
|
||||
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
yield return segment;
|
||||
await foreach (var segment in session.Segments.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
if (segment.ReplacesMarkerId is not null)
|
||||
{
|
||||
reconnectAttempt = 0;
|
||||
disconnectedMarkerWritten = false;
|
||||
}
|
||||
|
||||
yield return segment;
|
||||
}
|
||||
|
||||
var result = await session.Completion.WaitAsync(cancellationToken);
|
||||
if (result.IsCompleted)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
reconnectAttempt++;
|
||||
markerId = string.IsNullOrWhiteSpace(markerId)
|
||||
? $"azure-reconnect-{Guid.NewGuid():N}"
|
||||
: markerId;
|
||||
nextChunk = result.NextChunk ?? nextChunk;
|
||||
if (reconnectAttempt <= Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Azure Speech reconnecting after connection interruption, attempt {Attempt}/{MaxAttempts}: {ErrorDetails}",
|
||||
reconnectAttempt,
|
||||
Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker),
|
||||
result.ErrorDetails);
|
||||
yield return new TranscriptionSegment(
|
||||
TimeSpan.Zero,
|
||||
TimeSpan.Zero,
|
||||
"System",
|
||||
$"<Reconnecting... {reconnectAttempt}/{Math.Max(1, options.AzureSpeech.ReconnectAttemptsBeforeDisconnectedMarker)}>",
|
||||
TranscriptionSegmentKind.Marker,
|
||||
MarkerId: markerId);
|
||||
}
|
||||
else if (!disconnectedMarkerWritten)
|
||||
{
|
||||
disconnectedMarkerWritten = true;
|
||||
logger.LogWarning(
|
||||
"Azure Speech remains disconnected after {AttemptCount} reconnect attempt(s); recording continues and transcription will retry: {ErrorDetails}",
|
||||
reconnectAttempt - 1,
|
||||
result.ErrorDetails);
|
||||
yield return new TranscriptionSegment(
|
||||
TimeSpan.Zero,
|
||||
TimeSpan.Zero,
|
||||
"System",
|
||||
"<Azure Speech disconnected. The meeting can continue; transcription and summarization will continue once Azure Speech reconnects.>",
|
||||
TranscriptionSegmentKind.Marker,
|
||||
MarkerId: markerId);
|
||||
}
|
||||
|
||||
if (options.AzureSpeech.ReconnectDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(options.AzureSpeech.ReconnectDelay, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
await pumpTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -110,6 +119,117 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
|
||||
}
|
||||
}
|
||||
|
||||
private AzureSpeechSession StartSession(
|
||||
AudioChunk firstChunk,
|
||||
IAsyncEnumerator<AudioChunk> audio,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
string reconnectMarkerId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
|
||||
var completion = Task.Run(
|
||||
() => RunSessionAsync(
|
||||
firstChunk,
|
||||
audio,
|
||||
pipelineOptions,
|
||||
reconnectMarkerId,
|
||||
segments.Writer,
|
||||
cancellationToken),
|
||||
CancellationToken.None);
|
||||
return new AzureSpeechSession(segments.Reader, completion);
|
||||
}
|
||||
|
||||
private async Task<AzureSpeechSessionResult> RunSessionAsync(
|
||||
AudioChunk firstChunk,
|
||||
IAsyncEnumerator<AudioChunk> audio,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
string reconnectMarkerId,
|
||||
ChannelWriter<TranscriptionSegment> segments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var pushStream = AudioInputStream.CreatePushStream(
|
||||
AudioStreamFormat.GetWaveFormatPCM(
|
||||
(uint)firstChunk.SampleRate,
|
||||
16,
|
||||
(byte)firstChunk.Channels));
|
||||
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
|
||||
var speechConfig = CreateSpeechConfig();
|
||||
using var recognizer = CreateTranscriber(speechConfig, audioConfig);
|
||||
AddPhraseList(recognizer, pipelineOptions.DictationWords);
|
||||
var sessionStop = new TaskCompletionSource<AzureSpeechSessionResult>(
|
||||
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var pendingReconnectMarkerId = reconnectMarkerId;
|
||||
|
||||
recognizer.Transcribed += (_, args) =>
|
||||
{
|
||||
if (args.Result.Reason != ResultReason.RecognizedSpeech
|
||||
|| string.IsNullOrWhiteSpace(args.Result.Text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
|
||||
var replacesMarkerId = string.IsNullOrWhiteSpace(pendingReconnectMarkerId)
|
||||
? null
|
||||
: Interlocked.Exchange(ref pendingReconnectMarkerId, null);
|
||||
var segment = new TranscriptionSegment(
|
||||
start,
|
||||
start + args.Result.Duration,
|
||||
NormalizeSpeakerId(args.Result.SpeakerId),
|
||||
args.Result.Text.Trim(),
|
||||
ReplacesMarkerId: replacesMarkerId);
|
||||
logger.LogInformation(
|
||||
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
|
||||
segment.Start,
|
||||
segment.Speaker,
|
||||
segment.Text);
|
||||
segments.TryWrite(segment);
|
||||
};
|
||||
recognizer.Canceled += (_, args) =>
|
||||
{
|
||||
if (args.Reason == CancellationReason.Error)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Azure Speech recognition canceled with {ErrorCode}: {ErrorDetails}",
|
||||
args.ErrorCode,
|
||||
args.ErrorDetails);
|
||||
sessionStop.TrySetResult(AzureSpeechSessionResult.Reconnect(
|
||||
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
|
||||
return;
|
||||
}
|
||||
|
||||
sessionStop.TrySetResult(AzureSpeechSessionResult.Completed());
|
||||
};
|
||||
recognizer.SessionStopped += (_, _) => sessionStop.TrySetResult(AzureSpeechSessionResult.Completed());
|
||||
|
||||
await recognizer.StartTranscribingAsync();
|
||||
return await PumpAudioAsync(
|
||||
firstChunk,
|
||||
audio,
|
||||
pushStream,
|
||||
recognizer,
|
||||
sessionStop.Task,
|
||||
options.AzureSpeech.RecognitionStopTimeout,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (IsTransientConnectionException(exception))
|
||||
{
|
||||
logger.LogWarning(exception, "Azure Speech session failed transiently; reconnecting.");
|
||||
return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
segments.TryComplete(exception);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
segments.TryComplete();
|
||||
}
|
||||
}
|
||||
|
||||
internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig)
|
||||
{
|
||||
var languages = GetAutoDetectLanguages(options.AzureSpeech);
|
||||
@@ -234,47 +354,118 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
|
||||
: "Continuous";
|
||||
}
|
||||
|
||||
private static async Task PumpAudioAsync(
|
||||
private static async Task<AzureSpeechSessionResult> PumpAudioAsync(
|
||||
AudioChunk firstChunk,
|
||||
IAsyncEnumerator<AudioChunk> audio,
|
||||
PushAudioInputStream pushStream,
|
||||
ConversationTranscriber recognizer,
|
||||
ChannelWriter<TranscriptionSegment> segments,
|
||||
Task<AzureSpeechSessionResult> sessionStop,
|
||||
TimeSpan recognitionStopTimeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
WriteChunk(pushStream, firstChunk, firstChunk);
|
||||
while (true)
|
||||
{
|
||||
var moveNextTask = audio.MoveNextAsync().AsTask();
|
||||
var completedTask = await Task.WhenAny(moveNextTask, sessionStop);
|
||||
if (completedTask == sessionStop)
|
||||
{
|
||||
var stop = await sessionStop;
|
||||
if (stop.IsCompleted)
|
||||
{
|
||||
return AzureSpeechSessionResult.Completed();
|
||||
}
|
||||
|
||||
if (!await moveNextTask)
|
||||
{
|
||||
return AzureSpeechSessionResult.Completed();
|
||||
}
|
||||
|
||||
return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails);
|
||||
}
|
||||
|
||||
if (!await moveNextTask)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (sessionStop.IsCompleted)
|
||||
{
|
||||
var stop = await sessionStop;
|
||||
if (!stop.IsCompleted)
|
||||
{
|
||||
return AzureSpeechSessionResult.Reconnect(audio.Current, stop.ErrorDetails);
|
||||
}
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
WriteChunk(pushStream, firstChunk, audio.Current);
|
||||
}
|
||||
|
||||
pushStream.Close();
|
||||
try
|
||||
{
|
||||
WriteChunk(pushStream, firstChunk, firstChunk);
|
||||
while (await audio.MoveNextAsync())
|
||||
var stopTask = recognizer.StopTranscribingAsync();
|
||||
if (recognitionStopTimeout > TimeSpan.Zero)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
WriteChunk(pushStream, firstChunk, audio.Current);
|
||||
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
|
||||
}
|
||||
|
||||
pushStream.Close();
|
||||
try
|
||||
else
|
||||
{
|
||||
var stopTask = recognizer.StopTranscribingAsync();
|
||||
if (recognitionStopTimeout > TimeSpan.Zero)
|
||||
{
|
||||
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await stopTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
await stopTask.WaitAsync(cancellationToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// The SDK can outlive short diagnostic streams while waiting for final service events.
|
||||
}
|
||||
|
||||
segments.TryComplete();
|
||||
}
|
||||
catch (Exception exception)
|
||||
catch (TimeoutException)
|
||||
{
|
||||
segments.TryComplete(exception);
|
||||
// The SDK can outlive short diagnostic streams while waiting for final service events.
|
||||
}
|
||||
catch (Exception exception) when (IsTransientConnectionException(exception))
|
||||
{
|
||||
return AzureSpeechSessionResult.Reconnect(firstChunk, exception.Message);
|
||||
}
|
||||
|
||||
return AzureSpeechSessionResult.Completed();
|
||||
}
|
||||
|
||||
private static bool IsTransientConnectionException(Exception exception)
|
||||
{
|
||||
if (exception is OperationCanceledException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return exception is TimeoutException
|
||||
or IOException
|
||||
or HttpRequestException
|
||||
or System.Net.Sockets.SocketException
|
||||
|| exception.Message.Contains("connection", StringComparison.OrdinalIgnoreCase)
|
||||
|| exception.Message.Contains("websocket", StringComparison.OrdinalIgnoreCase)
|
||||
|| exception.Message.Contains("network", StringComparison.OrdinalIgnoreCase)
|
||||
|| exception.Message.Contains("transport", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private sealed record AzureSpeechSession(
|
||||
ChannelReader<TranscriptionSegment> Segments,
|
||||
Task<AzureSpeechSessionResult> Completion);
|
||||
|
||||
private sealed record AzureSpeechSessionResult(
|
||||
bool IsCompleted,
|
||||
AudioChunk? NextChunk,
|
||||
string? ErrorDetails)
|
||||
{
|
||||
public static AzureSpeechSessionResult Completed()
|
||||
{
|
||||
return new AzureSpeechSessionResult(true, null, null);
|
||||
}
|
||||
|
||||
public static AzureSpeechSessionResult Reconnect(AudioChunk nextChunk, string? errorDetails)
|
||||
{
|
||||
return new AzureSpeechSessionResult(false, nextChunk, errorDetails);
|
||||
}
|
||||
|
||||
public static AzureSpeechSessionResult Reconnect(string? errorDetails)
|
||||
{
|
||||
return new AzureSpeechSessionResult(false, null, errorDetails);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,16 @@
|
||||
namespace MeetingAssistant.Transcription;
|
||||
|
||||
public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text);
|
||||
public sealed record TranscriptionSegment(
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
string Speaker,
|
||||
string Text,
|
||||
TranscriptionSegmentKind Kind = TranscriptionSegmentKind.Speech,
|
||||
string? MarkerId = null,
|
||||
string? ReplacesMarkerId = null);
|
||||
|
||||
public enum TranscriptionSegmentKind
|
||||
{
|
||||
Speech,
|
||||
Marker
|
||||
}
|
||||
|
||||
@@ -117,26 +117,53 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
|
||||
|
||||
foreach (var rule in rules)
|
||||
{
|
||||
if (!MatchesTrigger(rule, workflowEvent) ||
|
||||
!EvaluateConditions(rule.If, meeting, context))
|
||||
try
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if (!MatchesTrigger(rule, workflowEvent) ||
|
||||
!EvaluateConditions(rule.If, meeting, context))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Applying meeting workflow rule {RuleName} for event {EventType}",
|
||||
rule.Name,
|
||||
workflowEvent.Type);
|
||||
var model = CreateTemplateModel(meeting, context);
|
||||
foreach (var step in rule.Steps)
|
||||
logger.LogInformation(
|
||||
"Triggered meeting workflow rule {RuleName} for event {EventType}",
|
||||
rule.Name,
|
||||
workflowEvent.Type);
|
||||
var ruleNoteChanged = false;
|
||||
var originalTranscriptLine = context.TranscriptLine;
|
||||
var model = CreateTemplateModel(meeting, context);
|
||||
foreach (var step in rule.Steps)
|
||||
{
|
||||
var stepChanged = await ApplyStepAsync(
|
||||
step,
|
||||
meeting,
|
||||
context,
|
||||
model,
|
||||
cancellationToken);
|
||||
ruleNoteChanged |= stepChanged;
|
||||
noteChanged |= stepChanged;
|
||||
model = CreateTemplateModel(meeting, context);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Completed meeting workflow rule {RuleName} for event {EventType}; noteChanged={NoteChanged}, transcriptLineChanged={TranscriptLineChanged}",
|
||||
rule.Name,
|
||||
workflowEvent.Type,
|
||||
ruleNoteChanged,
|
||||
!string.Equals(originalTranscriptLine, context.TranscriptLine, StringComparison.Ordinal));
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
noteChanged |= await ApplyStepAsync(
|
||||
step,
|
||||
meeting,
|
||||
context,
|
||||
model,
|
||||
cancellationToken);
|
||||
model = CreateTemplateModel(meeting, context);
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Meeting workflow rule {RuleName} failed for event {EventType}",
|
||||
rule.Name,
|
||||
workflowEvent.Type);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Net;
|
||||
using RazorLight;
|
||||
|
||||
namespace MeetingAssistant.Workflow;
|
||||
@@ -20,10 +21,11 @@ internal sealed class MeetingWorkflowTemplateRenderer
|
||||
return template;
|
||||
}
|
||||
|
||||
return await razorEngine.CompileRenderStringAsync(
|
||||
var rendered = await razorEngine.CompileRenderStringAsync(
|
||||
Guid.NewGuid().ToString("N"),
|
||||
EscapeEmailAddressAtSigns(template),
|
||||
model);
|
||||
return WebUtility.HtmlDecode(rendered);
|
||||
}
|
||||
|
||||
public static bool ContainsRazorTemplateSyntax(string value)
|
||||
|
||||
Reference in New Issue
Block a user