Add meeting assistant speech and summary automation

MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs

@@ -22,6 +22,7 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
 
     public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions pipelineOptions,
         [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
     {
         await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
@@ -38,7 +39,8 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
                 (byte)firstChunk.Channels));
         using var audioConfig = AudioConfig.FromStreamInput(pushStream);
         var speechConfig = CreateSpeechConfig();
-        using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
+        using var recognizer = CreateTranscriber(speechConfig, audioConfig);
+        AddPhraseList(recognizer, pipelineOptions.DictationWords);
         var segments = Channel.CreateUnbounded<TranscriptionSegment>();
 
         recognizer.Transcribed += (_, args) =>
@@ -94,7 +96,51 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
         await pumpTask.WaitAsync(cancellationToken);
     }
 
-    private SpeechConfig CreateSpeechConfig()
+    internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig)
+    {
+        var languages = GetAutoDetectLanguages(options.AzureSpeech);
+        if (languages.Count == 0)
+        {
+            return new ConversationTranscriber(speechConfig, audioConfig);
+        }
+
+        logger.LogInformation(
+            "Azure Speech auto language detection enabled for {Languages}",
+            string.Join(", ", languages));
+        return new ConversationTranscriber(
+            speechConfig,
+            AutoDetectSourceLanguageConfig.FromLanguages(languages.ToArray()),
+            audioConfig);
+    }
+
+    internal void AddPhraseList(ConversationTranscriber recognizer, IReadOnlyList<string>? words)
+    {
+        var phrases = NormalizeDictationWords(words);
+        if (phrases.Count == 0)
+        {
+            return;
+        }
+
+        var phraseList = PhraseListGrammar.FromRecognizer(recognizer);
+        foreach (var phrase in phrases)
+        {
+            phraseList.AddPhrase(phrase);
+        }
+
+        phraseList.SetWeight(options.AzureSpeech.PhraseListWeight);
+        logger.LogInformation("Configured Azure Speech phrase list with {PhraseCount} phrase(s)", phrases.Count);
+    }
+
+    internal static IReadOnlyList<string> NormalizeDictationWords(IReadOnlyList<string>? words)
+    {
+        return (words ?? [])
+            .Select(word => word.Trim())
+            .Where(word => !string.IsNullOrWhiteSpace(word))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    internal SpeechConfig CreateSpeechConfig()
     {
         var azure = options.AzureSpeech;
         var key = ResolveKey(azure);
@@ -104,10 +150,19 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
                 $"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
         }
 
-        var speechConfig = string.IsNullOrWhiteSpace(azure.Region)
-            ? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key)
-            : SpeechConfig.FromSubscription(key, azure.Region);
-        speechConfig.SpeechRecognitionLanguage = azure.Language;
+        var speechConfig = SpeechConfig.FromEndpoint(GetEndpoint(azure), key);
+        var autoDetectLanguages = GetAutoDetectLanguages(azure);
+        if (autoDetectLanguages.Count == 0)
+        {
+            speechConfig.SpeechRecognitionLanguage = azure.Language;
+        }
+        else
+        {
+            speechConfig.SetProperty(
+                PropertyId.SpeechServiceConnection_LanguageIdMode,
+                NormalizeLanguageIdMode(azure.LanguageIdMode));
+        }
+
         speechConfig.OutputFormat = OutputFormat.Detailed;
         speechConfig.SetProperty(
             PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
@@ -115,6 +170,42 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
         return speechConfig;
     }
 
+    internal static IReadOnlyList<string> GetAutoDetectLanguages(AzureSpeechOptions options)
+    {
+        if (!options.Language.Equals("auto", StringComparison.OrdinalIgnoreCase))
+        {
+            return [];
+        }
+
+        return options.AutoDetectLanguages
+            .Select(language => language.Trim())
+            .Where(language => !string.IsNullOrWhiteSpace(language))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    internal static Uri GetEndpoint(AzureSpeechOptions options)
+    {
+        if (!string.IsNullOrWhiteSpace(options.Endpoint))
+        {
+            return new Uri(options.Endpoint);
+        }
+
+        if (string.IsNullOrWhiteSpace(options.Region))
+        {
+            throw new InvalidOperationException("Azure Speech region or endpoint must be configured.");
+        }
+
+        return new Uri($"wss://{options.Region}.stt.speech.microsoft.com/speech/universal/v2");
+    }
+
+    internal static string NormalizeLanguageIdMode(string? value)
+    {
+        return value?.Trim().Equals("AtStart", StringComparison.OrdinalIgnoreCase) == true
+            ? "AtStart"
+            : "Continuous";
+    }
+
     private static async Task PumpAudioAsync(
         AudioChunk firstChunk,
         IAsyncEnumerator<AudioChunk> audio,
@@ -176,15 +267,21 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
             : speakerId;
     }
 
-    private static string ResolveKey(AzureSpeechOptions options)
+    internal static string ResolveKey(AzureSpeechOptions options)
     {
         if (!string.IsNullOrWhiteSpace(options.Key))
         {
             return options.Key;
         }
 
-        return string.IsNullOrWhiteSpace(options.KeyEnv)
-            ? ""
-            : Environment.GetEnvironmentVariable(options.KeyEnv) ?? "";
+        if (string.IsNullOrWhiteSpace(options.KeyEnv))
+        {
+            return "";
+        }
+
+        return Environment.GetEnvironmentVariable(options.KeyEnv) ??
+            Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.User) ??
+            Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.Machine) ??
+            "";
     }
 }

MeetingAssistant/Transcription/FunAsrStreamingTranscriptionProvider.cs

@@ -26,10 +26,11 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti
 
     public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions options,
         [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var segments = Channel.CreateUnbounded<TranscriptionSegment>();
-        var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None);
+        var session = Task.Run(() => RunSessionAsync(audio, options, segments.Writer, cancellationToken), CancellationToken.None);
 
         await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
         {
@@ -41,6 +42,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti
 
     private async Task RunSessionAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions pipelineOptions,
         ChannelWriter<TranscriptionSegment> segments,
         CancellationToken cancellationToken)
     {
@@ -61,7 +63,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti
             var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
             var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken);
 
-            await SendStartAsync(connection, firstChunk, cancellationToken);
+            await SendStartAsync(connection, firstChunk, pipelineOptions.DictationWords, cancellationToken);
             await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken);
 
             while (await enumerator.MoveNextAsync())
@@ -139,6 +141,7 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti
     private async Task SendStartAsync(
         IFunAsrWebSocketConnection connection,
         AudioChunk firstChunk,
+        IReadOnlyList<string>? dictationWords,
         CancellationToken cancellationToken)
     {
         var message = new Dictionary<string, object?>
@@ -152,24 +155,17 @@ public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscripti
             ["wav_name"] = options.FunAsr.WavName,
             ["wav_format"] = "pcm",
             ["is_speaking"] = true,
-            ["hotwords"] = await LoadHotwordsAsync(cancellationToken),
+            ["hotwords"] = BuildHotwords(dictationWords),
             ["itn"] = options.FunAsr.Itn
         };
 
         await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken);
     }
 
-    private async Task<string> LoadHotwordsAsync(CancellationToken cancellationToken)
+    internal static string BuildHotwords(IReadOnlyList<string>? dictationWords)
     {
-        var path = VaultPath.Resolve(options.Vault.DictationWordsPath);
-        if (!File.Exists(path))
-        {
-            return "";
-        }
-
-        var words = await File.ReadAllLinesAsync(path, cancellationToken);
-        var hotwords = words
-            .Select(line => line.Trim().TrimStart('-', '*').Trim())
+        var hotwords = (dictationWords ?? [])
+            .Select(word => word.Trim())
             .Where(line => !string.IsNullOrWhiteSpace(line))
             .Distinct(StringComparer.OrdinalIgnoreCase)
             .ToDictionary(word => word, _ => 20);

MeetingAssistant/Transcription/IDictationWordStore.cs

@@ -0,0 +1,8 @@
+namespace MeetingAssistant.Transcription;
+
+public interface IDictationWordStore
+{
+    Task<IReadOnlyList<string>> ReadWordsAsync(CancellationToken cancellationToken);
+
+    Task<IReadOnlyList<string>> AddWordAsync(string word, CancellationToken cancellationToken);
+}

MeetingAssistant/Transcription/ISpeechRecognitionPipeline.cs

@@ -6,6 +6,10 @@ public interface ISpeechRecognitionPipeline : IAsyncDisposable
 {
     Task InitializeAsync(CancellationToken cancellationToken);
 
+    Task InitializeAsync(
+        SpeechRecognitionPipelineOptions options,
+        CancellationToken cancellationToken);
+
     Task WaitUntilReadyAsync(CancellationToken cancellationToken);
 
     ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken);
@@ -25,7 +29,9 @@ public interface ISpeechRecognitionPipelineFactory
     ISpeechRecognitionPipeline Create();
 }
 
-public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null)
+public sealed record SpeechRecognitionPipelineOptions(
+    int? NumSpeakers = null,
+    IReadOnlyList<string>? DictationWords = null)
 {
     public static SpeechRecognitionPipelineOptions Default { get; } = new();
 }

MeetingAssistant/Transcription/IStreamingTranscriptionProvider.cs

@@ -6,5 +6,6 @@ public interface IStreamingTranscriptionProvider
 {
     IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions options,
         CancellationToken cancellationToken);
 }

MeetingAssistant/Transcription/MarkdownDictationWordStore.cs

@@ -0,0 +1,68 @@
+using Microsoft.Extensions.Options;
+
+namespace MeetingAssistant.Transcription;
+
+public sealed class MarkdownDictationWordStore : IDictationWordStore
+{
+    private readonly MeetingAssistantOptions options;
+
+    public MarkdownDictationWordStore(IOptions<MeetingAssistantOptions> options)
+    {
+        this.options = options.Value;
+    }
+
+    public async Task<IReadOnlyList<string>> ReadWordsAsync(CancellationToken cancellationToken)
+    {
+        var path = GetPath();
+        if (!File.Exists(path))
+        {
+            return [];
+        }
+
+        var lines = await File.ReadAllLinesAsync(path, cancellationToken);
+        return Normalize(lines);
+    }
+
+    public async Task<IReadOnlyList<string>> AddWordAsync(string word, CancellationToken cancellationToken)
+    {
+        var path = GetPath();
+        Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+        var existing = File.Exists(path)
+            ? await File.ReadAllLinesAsync(path, cancellationToken)
+            : [];
+        var words = Normalize(existing.Append(word));
+
+        await File.WriteAllTextAsync(
+            path,
+            string.Join(Environment.NewLine, words.Select(value => $"- {value}")) + Environment.NewLine,
+            cancellationToken);
+        return words;
+    }
+
+    private string GetPath()
+    {
+        return VaultPath.Resolve(options.Vault, options.Vault.DictationWordsPath);
+    }
+
+    internal static IReadOnlyList<string> Normalize(IEnumerable<string> lines)
+    {
+        return lines
+            .Select(ParseLine)
+            .Where(word => !string.IsNullOrWhiteSpace(word))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .Order(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    private static string ParseLine(string line)
+    {
+        var trimmed = line.Trim();
+        if (trimmed.StartsWith("- ", StringComparison.Ordinal) ||
+            trimmed.StartsWith("* ", StringComparison.Ordinal))
+        {
+            trimmed = trimmed[2..].Trim();
+        }
+
+        return trimmed.Trim('`', '"', '\'').Trim();
+    }
+}

MeetingAssistant/Transcription/StreamingSpeechRecognitionPipeline.cs

@@ -9,6 +9,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip
     private readonly Channel<AudioChunk> audio = Channel.CreateUnbounded<AudioChunk>();
     private readonly Channel<TranscriptionSegment> liveTranscript = Channel.CreateUnbounded<TranscriptionSegment>();
     private readonly List<TranscriptionSegment> liveSegments = [];
+    private SpeechRecognitionPipelineOptions options = SpeechRecognitionPipelineOptions.Default;
     private Task? transcriptionTask;
     private bool initialized;
 
@@ -18,6 +19,13 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip
     }
 
     public virtual Task InitializeAsync(CancellationToken cancellationToken)
+    {
+        return InitializeAsync(SpeechRecognitionPipelineOptions.Default, cancellationToken);
+    }
+
+    public virtual Task InitializeAsync(
+        SpeechRecognitionPipelineOptions options,
+        CancellationToken cancellationToken)
     {
         if (initialized)
         {
@@ -25,6 +33,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip
         }
 
         initialized = true;
+        this.options = options;
         transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None);
         return Task.CompletedTask;
     }
@@ -90,6 +99,7 @@ public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPip
         {
             await foreach (var segment in transcriptionProvider.TranscribeAsync(
                 audio.Reader.ReadAllAsync(cancellationToken),
+                options,
                 cancellationToken))
             {
                 liveSegments.Add(segment);

MeetingAssistant/Transcription/WhisperLocalStreamingTranscriptionProvider.cs

@@ -20,6 +20,7 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans
 
     public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions pipelineOptions,
         [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
     {
         var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath));
@@ -29,9 +30,7 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans
         }
 
         using var factory = WhisperFactory.FromPath(modelPath);
-        using var processor = factory.CreateBuilder()
-            .WithLanguage(options.WhisperLocal.Language)
-            .Build();
+        using var processor = CreateProcessor(factory, pipelineOptions.DictationWords);
 
         var window = new List<AudioChunk>();
         var windowDuration = TimeSpan.Zero;
@@ -119,6 +118,37 @@ public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTrans
         }
     }
 
+    private WhisperProcessor CreateProcessor(
+        WhisperFactory factory,
+        IReadOnlyList<string>? dictationWords)
+    {
+        var builder = factory.CreateBuilder()
+            .WithLanguage(options.WhisperLocal.Language);
+        var prompt = BuildPrompt(dictationWords);
+        if (!string.IsNullOrWhiteSpace(prompt))
+        {
+            builder.WithPrompt(prompt)
+                .WithCarryInitialPrompt(true);
+            logger.LogInformation(
+                "Configured Whisper prompt with {WordCount} dictation word(s)",
+                dictationWords?.Count ?? 0);
+        }
+
+        return builder.Build();
+    }
+
+    internal static string BuildPrompt(IReadOnlyList<string>? dictationWords)
+    {
+        var words = (dictationWords ?? [])
+            .Select(word => word.Trim())
+            .Where(word => !string.IsNullOrWhiteSpace(word))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+        return words.Count == 0
+            ? ""
+            : "Prefer these domain terms and acronyms: " + string.Join(", ", words) + ".";
+    }
+
     private static bool IsControlMarker(string text)
     {
         return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]";