Add meeting assistant speech and summary automation

MeetingAssistant/Transcription/AzureSpeechStreamingTranscriptionProvider.cs

@@ -22,6 +22,7 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
 
     public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
         IAsyncEnumerable<AudioChunk> audio,
+        SpeechRecognitionPipelineOptions pipelineOptions,
         [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
     {
         await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
@@ -38,7 +39,8 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
                 (byte)firstChunk.Channels));
         using var audioConfig = AudioConfig.FromStreamInput(pushStream);
         var speechConfig = CreateSpeechConfig();
-        using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
+        using var recognizer = CreateTranscriber(speechConfig, audioConfig);
+        AddPhraseList(recognizer, pipelineOptions.DictationWords);
         var segments = Channel.CreateUnbounded<TranscriptionSegment>();
 
         recognizer.Transcribed += (_, args) =>
@@ -94,7 +96,51 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
         await pumpTask.WaitAsync(cancellationToken);
     }
 
-    private SpeechConfig CreateSpeechConfig()
+    internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig)
+    {
+        var languages = GetAutoDetectLanguages(options.AzureSpeech);
+        if (languages.Count == 0)
+        {
+            return new ConversationTranscriber(speechConfig, audioConfig);
+        }
+
+        logger.LogInformation(
+            "Azure Speech auto language detection enabled for {Languages}",
+            string.Join(", ", languages));
+        return new ConversationTranscriber(
+            speechConfig,
+            AutoDetectSourceLanguageConfig.FromLanguages(languages.ToArray()),
+            audioConfig);
+    }
+
+    internal void AddPhraseList(ConversationTranscriber recognizer, IReadOnlyList<string>? words)
+    {
+        var phrases = NormalizeDictationWords(words);
+        if (phrases.Count == 0)
+        {
+            return;
+        }
+
+        var phraseList = PhraseListGrammar.FromRecognizer(recognizer);
+        foreach (var phrase in phrases)
+        {
+            phraseList.AddPhrase(phrase);
+        }
+
+        phraseList.SetWeight(options.AzureSpeech.PhraseListWeight);
+        logger.LogInformation("Configured Azure Speech phrase list with {PhraseCount} phrase(s)", phrases.Count);
+    }
+
+    internal static IReadOnlyList<string> NormalizeDictationWords(IReadOnlyList<string>? words)
+    {
+        return (words ?? [])
+            .Select(word => word.Trim())
+            .Where(word => !string.IsNullOrWhiteSpace(word))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    internal SpeechConfig CreateSpeechConfig()
     {
         var azure = options.AzureSpeech;
         var key = ResolveKey(azure);
@@ -104,10 +150,19 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
                 $"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
         }
 
-        var speechConfig = string.IsNullOrWhiteSpace(azure.Region)
-            ? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key)
-            : SpeechConfig.FromSubscription(key, azure.Region);
-        speechConfig.SpeechRecognitionLanguage = azure.Language;
+        var speechConfig = SpeechConfig.FromEndpoint(GetEndpoint(azure), key);
+        var autoDetectLanguages = GetAutoDetectLanguages(azure);
+        if (autoDetectLanguages.Count == 0)
+        {
+            speechConfig.SpeechRecognitionLanguage = azure.Language;
+        }
+        else
+        {
+            speechConfig.SetProperty(
+                PropertyId.SpeechServiceConnection_LanguageIdMode,
+                NormalizeLanguageIdMode(azure.LanguageIdMode));
+        }
+
         speechConfig.OutputFormat = OutputFormat.Detailed;
         speechConfig.SetProperty(
             PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
@@ -115,6 +170,42 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
         return speechConfig;
     }
 
+    internal static IReadOnlyList<string> GetAutoDetectLanguages(AzureSpeechOptions options)
+    {
+        if (!options.Language.Equals("auto", StringComparison.OrdinalIgnoreCase))
+        {
+            return [];
+        }
+
+        return options.AutoDetectLanguages
+            .Select(language => language.Trim())
+            .Where(language => !string.IsNullOrWhiteSpace(language))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    internal static Uri GetEndpoint(AzureSpeechOptions options)
+    {
+        if (!string.IsNullOrWhiteSpace(options.Endpoint))
+        {
+            return new Uri(options.Endpoint);
+        }
+
+        if (string.IsNullOrWhiteSpace(options.Region))
+        {
+            throw new InvalidOperationException("Azure Speech region or endpoint must be configured.");
+        }
+
+        return new Uri($"wss://{options.Region}.stt.speech.microsoft.com/speech/universal/v2");
+    }
+
+    internal static string NormalizeLanguageIdMode(string? value)
+    {
+        return value?.Trim().Equals("AtStart", StringComparison.OrdinalIgnoreCase) == true
+            ? "AtStart"
+            : "Continuous";
+    }
+
     private static async Task PumpAudioAsync(
         AudioChunk firstChunk,
         IAsyncEnumerator<AudioChunk> audio,
@@ -176,15 +267,21 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
             : speakerId;
     }
 
-    private static string ResolveKey(AzureSpeechOptions options)
+    internal static string ResolveKey(AzureSpeechOptions options)
     {
         if (!string.IsNullOrWhiteSpace(options.Key))
         {
             return options.Key;
         }
 
-        return string.IsNullOrWhiteSpace(options.KeyEnv)
-            ? ""
-            : Environment.GetEnvironmentVariable(options.KeyEnv) ?? "";
+        if (string.IsNullOrWhiteSpace(options.KeyEnv))
+        {
+            return "";
+        }
+
+        return Environment.GetEnvironmentVariable(options.KeyEnv) ??
+            Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.User) ??
+            Environment.GetEnvironmentVariable(options.KeyEnv, EnvironmentVariableTarget.Machine) ??
+            "";
     }
 }