Implement meeting assistant v1

This commit is contained in:
2026-05-20 02:06:16 +02:00
parent 90df1edc03
commit 0297bcc0f6
120 changed files with 11883 additions and 180 deletions
@@ -0,0 +1,150 @@
using System.Runtime.InteropServices;
using System.Threading;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Hotkeys;
public sealed class GlobalHotkeyService : BackgroundService
{
private const int HotkeyId = 0x4D41;
private const int WmHotkey = 0x0312;
private const int WmQuit = 0x0012;
private readonly MeetingAssistantOptions options;
private readonly MeetingRecordingCoordinator coordinator;
private readonly ILogger<GlobalHotkeyService> logger;
private TaskCompletionSource? messageLoopCompletion;
private uint messageThreadId;
public GlobalHotkeyService(
IOptions<MeetingAssistantOptions> options,
MeetingRecordingCoordinator coordinator,
ILogger<GlobalHotkeyService> logger)
{
this.options = options.Value;
this.coordinator = coordinator;
this.logger = logger;
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!OperatingSystem.IsWindows())
{
logger.LogInformation("Global hotkey registration is only supported on Windows");
return Task.CompletedTask;
}
messageLoopCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var messageThread = new Thread(() =>
{
try
{
RunMessageLoop(stoppingToken);
messageLoopCompletion.TrySetResult();
}
catch (Exception exception)
{
messageLoopCompletion.TrySetException(exception);
}
})
{
IsBackground = true,
Name = "Meeting Assistant Hotkey Loop"
};
messageThread.Start();
return messageLoopCompletion.Task;
}
private void RunMessageLoop(CancellationToken stoppingToken)
{
var hotkey = HotkeyDefinition.Parse(options.Hotkey.Toggle);
messageThreadId = GetCurrentThreadId();
using var stoppingRegistration = stoppingToken.Register(() => PostQuitToMessageThread());
if (!RegisterHotKey(IntPtr.Zero, HotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey))
{
logger.LogWarning("Could not register global hotkey {Hotkey}", options.Hotkey.Toggle);
return;
}
logger.LogInformation("Registered global hotkey {Hotkey}", options.Hotkey.Toggle);
try
{
while (!stoppingToken.IsCancellationRequested && GetMessage(out var message, IntPtr.Zero, 0, 0) > 0)
{
if (message.Message == WmHotkey)
{
_ = Task.Run(ToggleRecordingAsync, CancellationToken.None);
}
}
}
finally
{
UnregisterHotKey(IntPtr.Zero, HotkeyId);
messageThreadId = 0;
}
}
public override Task StopAsync(CancellationToken cancellationToken)
{
PostQuitToMessageThread();
return base.StopAsync(cancellationToken);
}
private async Task ToggleRecordingAsync()
{
try
{
var status = await coordinator.ToggleAsync(CancellationToken.None);
logger.LogInformation("Hotkey toggled recording. IsRecording={IsRecording}", status.IsRecording);
}
catch (Exception exception)
{
logger.LogError(exception, "Hotkey toggle failed");
}
}
private void PostQuitToMessageThread()
{
var threadId = messageThreadId;
if (threadId != 0)
{
PostThreadMessage(threadId, WmQuit, UIntPtr.Zero, IntPtr.Zero);
}
}
[DllImport("user32.dll", SetLastError = true)]
private static extern bool RegisterHotKey(IntPtr hWnd, int id, uint fsModifiers, uint vk);
[DllImport("user32.dll", SetLastError = true)]
private static extern bool UnregisterHotKey(IntPtr hWnd, int id);
[DllImport("user32.dll")]
private static extern int GetMessage(out NativeMessage lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax);
[DllImport("kernel32.dll")]
private static extern uint GetCurrentThreadId();
[DllImport("user32.dll", SetLastError = true)]
private static extern bool PostThreadMessage(uint idThread, int msg, UIntPtr wParam, IntPtr lParam);
[StructLayout(LayoutKind.Sequential)]
private struct NativeMessage
{
public IntPtr Hwnd;
public int Message;
public UIntPtr WParam;
public IntPtr LParam;
public uint Time;
public NativePoint Point;
}
[StructLayout(LayoutKind.Sequential)]
private struct NativePoint
{
public int X;
public int Y;
}
}
@@ -0,0 +1,72 @@
namespace MeetingAssistant.Hotkeys;
[Flags]
public enum HotkeyModifiers : uint
{
None = 0,
Alt = 0x0001,
Control = 0x0002,
Shift = 0x0004,
Windows = 0x0008
}
public sealed record HotkeyDefinition(HotkeyModifiers Modifiers, uint VirtualKey)
{
public static HotkeyDefinition Parse(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Hotkey cannot be empty.", nameof(value));
}
var modifiers = HotkeyModifiers.None;
uint? virtualKey = null;
foreach (var rawPart in value.Split('+', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
var part = rawPart.ToUpperInvariant();
modifiers |= part switch
{
"CTRL" or "CONTROL" => HotkeyModifiers.Control,
"ALT" => HotkeyModifiers.Alt,
"SHIFT" => HotkeyModifiers.Shift,
"WIN" or "WINDOWS" => HotkeyModifiers.Windows,
_ => HotkeyModifiers.None
};
if (modifiers.HasFlagForToken(part))
{
continue;
}
virtualKey = ParseVirtualKey(part);
}
return virtualKey is null
? throw new FormatException($"Hotkey '{value}' does not contain a key.")
: new HotkeyDefinition(modifiers, virtualKey.Value);
}
private static uint ParseVirtualKey(string part)
{
if (part.Length == 1 && char.IsLetterOrDigit(part[0]))
{
return part[0];
}
if (part.Length is 2 or 3 && part[0] == 'F' && int.TryParse(part[1..], out var functionKey) && functionKey is >= 1 and <= 24)
{
return (uint)(0x70 + functionKey - 1);
}
throw new FormatException($"Unsupported hotkey key '{part}'.");
}
}
file static class HotkeyModifierExtensions
{
public static bool HasFlagForToken(this HotkeyModifiers _, string token)
{
return token is "CTRL" or "CONTROL" or "ALT" or "SHIFT" or "WIN" or "WINDOWS";
}
}
+20 -1
View File
@@ -1,9 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<TargetFrameworks>net10.0;net10.0-windows</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.6.1" />
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.50.0" />
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="Whisper.net" Version="1.9.0" />
<PackageReference Include="Whisper.net.Runtime" Version="1.9.0" />
<PackageReference Include="YamlDotNet" Version="17.1.0" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' != 'net10.0-windows'">
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
<Compile Remove="Recording\NaudioCaptureSource.cs" />
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
</ItemGroup>
</Project>
+236
View File
@@ -0,0 +1,236 @@
namespace MeetingAssistant;
public sealed class MeetingAssistantOptions
{
public HotkeyOptions Hotkey { get; set; } = new();
public VaultOptions Vault { get; set; } = new();
public RecordingOptions Recording { get; set; } = new();
public WhisperLocalOptions WhisperLocal { get; set; } = new();
public AzureSpeechOptions AzureSpeech { get; set; } = new();
public FunAsrOptions FunAsr { get; set; } = new();
public AgentOptions Agent { get; set; } = new();
public ApiOptions Api { get; set; } = new();
}
public sealed class HotkeyOptions
{
public string Toggle { get; set; } = "Ctrl+Alt+M";
}
public sealed class VaultOptions
{
public string TranscriptsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Transcripts";
public string MeetingNotesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Notes";
public string AssistantContextFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Assistant Context";
public string SummariesFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\Summaries";
public string ProjectsFolder { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Projects";
public string DictationWordsPath { get; set; } = @"%USERPROFILE%\OpenCloud\Persönlich\Vault\Meetings\dictation-words.md";
}
public sealed class RecordingOptions
{
public string TranscriptionProvider { get; set; } = "funasr";
public int SampleRate { get; set; } = 16000;
public int Channels { get; set; } = 1;
public TimeSpan StopProcessingTimeout { get; set; } = TimeSpan.FromMinutes(10);
public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings";
}
public sealed class WhisperLocalOptions
{
public string ModelPath { get; set; } = @"models\ggml-base.bin";
public string Language { get; set; } = "auto";
public double WindowSeconds { get; set; } = 15;
public PyannoteDiarizationOptions Diarization { get; set; } = new();
}
public sealed class PyannoteDiarizationOptions
{
public bool Enabled { get; set; } = true;
public string DockerCommand { get; set; } = "docker";
public string BaseImage { get; set; } = "python:3.11-slim";
public string Image { get; set; } = "meeting-assistant-pyannote:local";
public bool BuildImage { get; set; } = true;
public string ModelsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Pyannote\models";
public string Model { get; set; } = "pyannote/speaker-diarization-3.1";
public PyannoteAnnotationSource AnnotationSource { get; set; } = PyannoteAnnotationSource.SpeakerDiarization;
public PyannoteAlignmentMode AlignmentMode { get; set; } = PyannoteAlignmentMode.PyannoteTurns;
public string TokenEnv { get; set; } = "HF_TOKEN";
public string? Token { get; set; }
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
}
public sealed class AzureSpeechOptions
{
public string Endpoint { get; set; } = "https://germanywestcentral.api.cognitive.microsoft.com/";
public string Region { get; set; } = "germanywestcentral";
public string Language { get; set; } = "en-US";
public string? Key { get; set; }
public string KeyEnv { get; set; } = "AZURE_SPEECH_KEY";
public TimeSpan RecognitionStopTimeout { get; set; } = TimeSpan.FromSeconds(10);
public bool DiarizeIntermediateResults { get; set; } = true;
}
public enum PyannoteAnnotationSource
{
SpeakerDiarization,
ExclusiveSpeakerDiarization
}
public enum PyannoteAlignmentMode
{
BestOverlap,
PyannoteTurns
}
public sealed class FunAsrOptions
{
public string Endpoint { get; set; } = "ws://127.0.0.1:10095";
public string Mode { get; set; } = "2pass";
public int[] ChunkSize { get; set; } = [5, 10, 5];
public int ChunkInterval { get; set; } = 10;
public int EncoderChunkLookBack { get; set; } = 4;
public int DecoderChunkLookBack { get; set; } = 1;
public bool Itn { get; set; } = true;
public bool EmitOnlinePartials { get; set; }
public string WavName { get; set; } = "meeting";
public TimeSpan FinalResultTimeout { get; set; } = TimeSpan.FromSeconds(10);
public FunAsrBackendOptions Backend { get; set; } = new();
public FunAsrDiarizationOptions Diarization { get; set; } = new();
}
public sealed class FunAsrBackendOptions
{
public bool Enabled { get; set; } = true;
public string DockerCommand { get; set; } = "docker";
public string Image { get; set; } = "registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.13";
public string ContainerName { get; set; } = "meeting-assistant-funasr";
public string ModelsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\FunASR\models";
public int ContainerPort { get; set; } = 10095;
public bool Privileged { get; set; } = true;
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
public TimeSpan StartupTimeout { get; set; } = TimeSpan.FromMinutes(10);
public bool StopOnDispose { get; set; } = true;
public string ServerCommand { get; set; } = "cd /workspace/FunASR/runtime && bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --itn-dir thuduj12/fst_itn_zh --certfile 0 --hotword /workspace/models/hotwords.txt && tail -f /dev/null";
}
public sealed class FunAsrDiarizationOptions
{
public bool Enabled { get; set; } = true;
public string AsrModel { get; set; } = "paraformer-zh";
public string VadModel { get; set; } = "fsmn-vad";
public string PunctuationModel { get; set; } = "ct-punc";
public string SpeakerModel { get; set; } = "cam++";
public double BatchSizeSeconds { get; set; } = 300;
public double BatchSizeThresholdSeconds { get; set; } = 60;
public int? PresetSpeakerCount { get; set; }
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
}
public sealed class AgentOptions
{
public string Endpoint { get; set; } = "https://litellm.schweigert.cloud";
public string? Key { get; set; }
public string KeyEnv { get; set; } = "LITELLM_API_KEY";
public string Model { get; set; } = "copilot-gpt-5.5";
public bool EnableThinking { get; set; } = true;
public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium;
public int ReconnectionAttempts { get; set; } = 2;
public TimeSpan ReconnectionDelay { get; set; } = TimeSpan.FromSeconds(2);
public int ContextWindowTokens { get; set; } = 128000;
public int MaxOutputTokens { get; set; } = 8192;
public bool EnableCompaction { get; set; } = true;
public double CompactionRemainingRatio { get; set; } = 0.10;
public string ResponsesCompactPath { get; set; } = "responses/compact";
}
public sealed class ApiOptions
{
public string PublicBaseUrl { get; set; } = "http://localhost:5090";
}
public enum ReasoningEffortOption
{
None,
Low,
Medium,
High,
ExtraHigh
}
@@ -0,0 +1,24 @@
namespace MeetingAssistant.MeetingNotes;
public interface IMeetingArtifactStore
{
Task CreateAssistantContextAsync(
MeetingSessionArtifacts artifacts,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken);
Task UpdateAssistantContextStateAsync(
MeetingSessionArtifacts artifacts,
AssistantContextState state,
CancellationToken cancellationToken);
}
public enum AssistantContextState
{
Transcribing,
SpeakerRecognition,
Summarizing,
Finished,
Error
}
@@ -0,0 +1,24 @@
namespace MeetingAssistant.MeetingNotes;
public interface IMeetingMetadataProvider
{
Task<MeetingMetadata?> GetCurrentMeetingAsync(
DateTimeOffset startedAt,
CancellationToken cancellationToken);
}
public sealed record MeetingMetadata(
string Title,
IReadOnlyList<string> Attendees,
string Agenda,
DateTimeOffset? ScheduledEnd = null);
public sealed class NoopMeetingMetadataProvider : IMeetingMetadataProvider
{
public Task<MeetingMetadata?> GetCurrentMeetingAsync(
DateTimeOffset startedAt,
CancellationToken cancellationToken)
{
return Task.FromResult<MeetingMetadata?>(null);
}
}
@@ -0,0 +1,6 @@
namespace MeetingAssistant.MeetingNotes;
public interface IMeetingNoteOpener
{
Task OpenAsync(string notePath, CancellationToken cancellationToken);
}
@@ -0,0 +1,8 @@
namespace MeetingAssistant.MeetingNotes;
public interface IMeetingNoteStore
{
Task<MeetingNote> SaveAsync(MeetingNote note, CancellationToken cancellationToken);
Task<MeetingNote> ReadAsync(string path, CancellationToken cancellationToken);
}
@@ -0,0 +1,76 @@
namespace MeetingAssistant.MeetingNotes;
internal readonly record struct MarkdownDocument(
bool HasFrontmatter,
string Frontmatter,
string Body);
internal static class MarkdownDocumentParser
{
public static MarkdownDocument SplitOptional(string content)
{
using var reader = new StringReader(content);
if (reader.ReadLine() != "---")
{
return new MarkdownDocument(false, "", content);
}
var yaml = new List<string>();
string? line;
while ((line = reader.ReadLine()) is not null)
{
if (line == "---")
{
return new MarkdownDocument(
true,
string.Join(Environment.NewLine, yaml),
RemoveBodySeparator(reader.ReadToEnd()));
}
yaml.Add(line);
}
return new MarkdownDocument(false, "", content);
}
public static MarkdownDocument SplitRequired(
string content,
string missingFrontmatterMessage,
string unclosedFrontmatterMessage)
{
using var reader = new StringReader(content);
if (reader.ReadLine() != "---")
{
throw new InvalidDataException(missingFrontmatterMessage);
}
var yaml = new List<string>();
string? line;
while ((line = reader.ReadLine()) is not null)
{
if (line == "---")
{
return new MarkdownDocument(
true,
string.Join(Environment.NewLine, yaml),
RemoveBodySeparator(reader.ReadToEnd()));
}
yaml.Add(line);
}
throw new InvalidDataException(unclosedFrontmatterMessage);
}
private static string RemoveBodySeparator(string body)
{
if (body.StartsWith("\r\n", StringComparison.Ordinal))
{
return body[2..];
}
return body.StartsWith('\n')
? body[1..]
: body;
}
}
@@ -0,0 +1,123 @@
namespace MeetingAssistant.MeetingNotes;
using YamlDotNet.Serialization;
public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
{
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
private readonly ILogger<MarkdownMeetingArtifactStore> logger;
public MarkdownMeetingArtifactStore(ILogger<MarkdownMeetingArtifactStore> logger)
{
this.logger = logger;
}
public async Task CreateAssistantContextAsync(
MeetingSessionArtifacts artifacts,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
var content = Render(
artifacts,
AssistantContextState.Transcribing,
agenda,
scheduledEnd,
"# Assistant Context" + Environment.NewLine + Environment.NewLine + "## Live Context" + Environment.NewLine);
await File.WriteAllTextAsync(artifacts.AssistantContextPath, content, cancellationToken);
logger.LogInformation("Created assistant context note {AssistantContextPath}", artifacts.AssistantContextPath);
}
public async Task UpdateAssistantContextStateAsync(
MeetingSessionArtifacts artifacts,
AssistantContextState state,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
var existing = File.Exists(artifacts.AssistantContextPath)
? MarkdownDocumentParser.SplitOptional(await File.ReadAllTextAsync(artifacts.AssistantContextPath, cancellationToken))
: new MarkdownDocument(false, "", "## Live Context" + Environment.NewLine);
var body = existing.Body;
var existingFrontmatter = ReadFrontmatter(existing.Frontmatter);
var agenda = existingFrontmatter.Agenda ?? "";
var scheduledEnd = ParseDateTime(existingFrontmatter.ScheduledEnd);
await File.WriteAllTextAsync(
artifacts.AssistantContextPath,
Render(artifacts, state, agenda, scheduledEnd, body),
cancellationToken);
logger.LogInformation(
"Updated assistant context note {AssistantContextPath} state to {State}",
artifacts.AssistantContextPath,
ToYamlState(state));
}
private static string Render(
MeetingSessionArtifacts artifacts,
AssistantContextState state,
string? agenda,
DateTimeOffset? scheduledEnd,
string body)
{
var frontmatter = new MeetingArtifactFrontmatter
{
Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, artifacts.AssistantContextPath, "Meeting Note"),
Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, artifacts.AssistantContextPath, "Transcript"),
Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, artifacts.AssistantContextPath, "Summary"),
State = ToYamlState(state),
Agenda = agenda ?? "",
ScheduledEnd = scheduledEnd
};
return MeetingArtifactFrontmatterRenderer.Render(
frontmatter,
body);
}
private static AssistantContextFrontmatterYaml ReadFrontmatter(string frontmatter)
{
if (string.IsNullOrWhiteSpace(frontmatter))
{
return new AssistantContextFrontmatterYaml();
}
return YamlDeserializer.Deserialize<AssistantContextFrontmatterYaml>(frontmatter)
?? new AssistantContextFrontmatterYaml();
}
private static DateTimeOffset? ParseDateTime(string? value)
{
return DateTimeOffset.TryParse(value, out var parsed)
? parsed
: null;
}
private sealed class AssistantContextFrontmatterYaml
{
[YamlMember(Alias = "agenda")]
public string? Agenda { get; set; }
[YamlMember(Alias = "scheduled_end")]
public string? ScheduledEnd { get; set; }
}
private static string ToYamlState(AssistantContextState state)
{
return state switch
{
AssistantContextState.Transcribing => "transcribing",
AssistantContextState.SpeakerRecognition => "speaker recognition",
AssistantContextState.Summarizing => "summarizing",
AssistantContextState.Finished => "finished",
AssistantContextState.Error => "error",
_ => "error"
};
}
}
@@ -0,0 +1,176 @@
using Microsoft.Extensions.Options;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
namespace MeetingAssistant.MeetingNotes;
public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<MarkdownMeetingNoteStore> logger;
private readonly IDeserializer yamlDeserializer;
public MarkdownMeetingNoteStore(
IOptions<MeetingAssistantOptions> options,
ILogger<MarkdownMeetingNoteStore> logger)
{
this.options = options.Value;
this.logger = logger;
yamlDeserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
}
public async Task<MeetingNote> SaveAsync(MeetingNote note, CancellationToken cancellationToken)
{
var folder = VaultPath.Resolve(options.Vault.MeetingNotesFolder);
Directory.CreateDirectory(folder);
var path = string.IsNullOrWhiteSpace(note.Path)
? Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Slugify(note.Frontmatter.Title)}.md")
: note.Path;
var frontmatter = PrepareFrontmatter(note.Frontmatter, path);
var content = Render(frontmatter, note.UserNotes);
await File.WriteAllTextAsync(path, content, cancellationToken);
logger.LogInformation("Saved meeting note {MeetingNotePath}", path);
return new MeetingNote(path, frontmatter, note.UserNotes);
}
public async Task<MeetingNote> ReadAsync(string path, CancellationToken cancellationToken)
{
var content = await File.ReadAllTextAsync(path, cancellationToken);
var document = MarkdownDocumentParser.SplitRequired(
content,
"Meeting note does not start with YAML frontmatter.",
"Meeting note frontmatter is not closed.");
var yaml = yamlDeserializer.Deserialize<MeetingNoteYaml>(document.Frontmatter) ?? new MeetingNoteYaml();
var frontmatter = new MeetingNoteFrontmatter
{
Title = yaml.Title ?? "",
StartTime = ParseDateTime(yaml.StartTime),
EndTime = ParseDateTime(yaml.EndTime),
Attendees = yaml.Attendees ?? [],
Projects = yaml.Projects ?? [],
Transcript = yaml.Transcript ?? "",
AssistantContext = yaml.AssistantContext ?? "",
Summary = yaml.Summary ?? ""
};
return new MeetingNote(path, frontmatter, document.Body);
}
private MeetingNoteFrontmatter PrepareFrontmatter(MeetingNoteFrontmatter frontmatter, string notePath)
{
return new MeetingNoteFrontmatter
{
Title = frontmatter.Title,
StartTime = frontmatter.StartTime,
EndTime = frontmatter.EndTime,
Attendees = frontmatter.Attendees,
Projects = frontmatter.Projects,
Transcript = ObsidianLink.ToWikiLink(frontmatter.Transcript, notePath, "Transcript"),
AssistantContext = ObsidianLink.ToWikiLink(frontmatter.AssistantContext, notePath, "Assistant Context"),
Summary = ObsidianLink.ToWikiLink(frontmatter.Summary, notePath, "Summary")
};
}
private static string Render(MeetingNoteFrontmatter frontmatter, string userNotes)
{
var lines = new List<string>
{
"---",
$"title: {EscapeScalar(frontmatter.Title)}",
$"start_time: {EscapeNullableDateTime(frontmatter.StartTime)}",
$"end_time: {EscapeNullableDateTime(frontmatter.EndTime)}",
"attendees:"
};
lines.AddRange(frontmatter.Attendees.Select(attendee => $"- {EscapeListItem(attendee)}"));
lines.Add("projects:");
lines.AddRange(frontmatter.Projects.Select(project => $"- {EscapeListItem(project)}"));
lines.Add($"transcript: {EscapeQuoted(frontmatter.Transcript)}");
lines.Add($"assistant_context: {EscapeQuoted(frontmatter.AssistantContext)}");
lines.Add($"summary: {EscapeQuoted(frontmatter.Summary)}");
lines.Add("---");
lines.Add("");
lines.Add(userNotes);
return string.Join(Environment.NewLine, lines);
}
private static string Slugify(string value)
{
var slug = new string(value
.ToLowerInvariant()
.Select(character => char.IsLetterOrDigit(character) ? character : '-')
.ToArray());
slug = string.Join("-", slug.Split('-', StringSplitOptions.RemoveEmptyEntries));
return string.IsNullOrWhiteSpace(slug) ? "meeting" : slug;
}
private static string EscapeScalar(string value)
{
return string.IsNullOrWhiteSpace(value) ? "\"\"" : EscapeListItem(value);
}
private static string EscapeQuoted(string value)
{
return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
}
private static string EscapeNullableDateTime(DateTimeOffset? value)
{
return value is null ? "\"\"" : EscapeQuoted(value.Value.ToString("O"));
}
private static DateTimeOffset? ParseDateTime(string? value)
{
return DateTimeOffset.TryParse(value, out var parsed)
? parsed
: null;
}
private static string EscapeListItem(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "\"\"";
}
if (value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal))
{
return EscapeQuoted(value);
}
return value;
}
private sealed class MeetingNoteYaml
{
[YamlMember(Alias = "title")]
public string? Title { get; set; }
[YamlMember(Alias = "start_time")]
public string? StartTime { get; set; }
[YamlMember(Alias = "end_time")]
public string? EndTime { get; set; }
[YamlMember(Alias = "attendees")]
public List<string>? Attendees { get; set; }
[YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
[YamlMember(Alias = "transcript")]
public string? Transcript { get; set; }
[YamlMember(Alias = "assistant_context")]
public string? AssistantContext { get; set; }
[YamlMember(Alias = "summary")]
public string? Summary { get; set; }
}
}
@@ -0,0 +1,148 @@
using System.Text;
namespace MeetingAssistant.MeetingNotes;
public sealed class MeetingArtifactFrontmatter
{
public string Title { get; set; } = "";
public DateTimeOffset? StartTime { get; set; }
public DateTimeOffset? EndTime { get; set; }
public string Meeting { get; set; } = "";
public string Transcript { get; set; } = "";
public string AssistantContext { get; set; } = "";
public string Summary { get; set; } = "";
public string? State { get; set; }
public string? Agenda { get; set; }
public DateTimeOffset? ScheduledEnd { get; set; }
}
public static class MeetingArtifactFrontmatterRenderer
{
public static string Render(
MeetingArtifactFrontmatter frontmatter,
string body)
{
var builder = new StringBuilder();
builder.AppendLine("---");
AppendScalar(builder, "title", frontmatter.Title);
AppendDateTime(builder, "start_time", frontmatter.StartTime);
AppendDateTime(builder, "end_time", frontmatter.EndTime);
AppendQuoted(builder, "meeting", frontmatter.Meeting);
AppendQuoted(builder, "transcript", frontmatter.Transcript);
AppendQuoted(builder, "assistant_context", frontmatter.AssistantContext);
AppendQuoted(builder, "summary", frontmatter.Summary);
if (!string.IsNullOrWhiteSpace(frontmatter.State))
{
AppendScalar(builder, "state", frontmatter.State);
}
if (frontmatter.Agenda is not null)
{
AppendBlockScalar(builder, "agenda", frontmatter.Agenda);
}
if (frontmatter.ScheduledEnd is not null)
{
AppendDateTime(builder, "scheduled_end", frontmatter.ScheduledEnd);
}
builder.AppendLine("---");
builder.AppendLine();
builder.Append(body.TrimStart('\r', '\n'));
return builder.ToString();
}
public static (string Frontmatter, string Body) Split(string content)
{
var document = MarkdownDocumentParser.SplitOptional(content);
return (document.Frontmatter, document.Body);
}
public static MeetingArtifactFrontmatter Create(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
string title,
string sourceNotePath,
string? state = null)
{
return new MeetingArtifactFrontmatter
{
Title = title,
StartTime = meetingNote.Frontmatter.StartTime,
EndTime = meetingNote.Frontmatter.EndTime,
Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, sourceNotePath, "Meeting Note"),
Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, sourceNotePath, "Transcript"),
AssistantContext = ObsidianLink.ToWikiLink(artifacts.AssistantContextPath, sourceNotePath, "Assistant Context"),
Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, sourceNotePath, "Summary"),
State = state
};
}
public static string DefaultTitle(MeetingNote meetingNote, string fallback)
{
return string.IsNullOrWhiteSpace(meetingNote.Frontmatter.Title)
? fallback
: meetingNote.Frontmatter.Title;
}
private static void AppendScalar(StringBuilder builder, string key, string? value)
{
builder.Append(key);
builder.Append(": ");
builder.AppendLine(string.IsNullOrWhiteSpace(value) ? "\"\"" : EscapeListItem(value));
}
private static void AppendQuoted(StringBuilder builder, string key, string value)
{
builder.Append(key);
builder.Append(": ");
builder.AppendLine(EscapeQuoted(value));
}
private static void AppendDateTime(StringBuilder builder, string key, DateTimeOffset? value)
{
builder.Append(key);
builder.Append(": ");
builder.AppendLine(value is null ? "\"\"" : EscapeQuoted(value.Value.ToString("O")));
}
private static void AppendBlockScalar(StringBuilder builder, string key, string value)
{
if (string.IsNullOrEmpty(value))
{
builder.Append(key);
builder.AppendLine(": \"\"");
return;
}
builder.Append(key);
builder.AppendLine(": |-");
foreach (var line in value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n'))
{
builder.Append(" ");
builder.AppendLine(line);
}
}
private static string EscapeQuoted(string value)
{
return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
}
private static string EscapeListItem(string value)
{
return value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal)
? EscapeQuoted(value)
: value;
}
}
@@ -0,0 +1,25 @@
namespace MeetingAssistant.MeetingNotes;
public sealed record MeetingNote(
string Path,
MeetingNoteFrontmatter Frontmatter,
string UserNotes);
public sealed class MeetingNoteFrontmatter
{
public string Title { get; set; } = "";
public DateTimeOffset? StartTime { get; set; }
public DateTimeOffset? EndTime { get; set; }
public List<string> Attendees { get; set; } = [];
public List<string> Projects { get; set; } = [];
public string Transcript { get; set; } = "";
public string AssistantContext { get; set; } = "";
public string Summary { get; set; } = "";
}
@@ -0,0 +1,17 @@
namespace MeetingAssistant.MeetingNotes;
public static class MeetingNoteActionLinks
{
public static string CreateSummaryRetryLink(
string publicBaseUrl,
string summaryPath,
string label = "Retry summary generation")
{
var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl)
? "http://localhost:5090"
: publicBaseUrl.TrimEnd('/');
var summaryFileName = Path.GetFileName(summaryPath);
var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}";
return $"[{label}]({url})";
}
}
@@ -0,0 +1,31 @@
namespace MeetingAssistant.MeetingNotes;
public static class MeetingNoteTemplate
{
public static MeetingNote Create(
string title,
DateTimeOffset? startTime = null,
DateTimeOffset? endTime = null,
IEnumerable<string>? attendees = null,
IEnumerable<string>? projects = null,
string transcriptPath = "",
string assistantContextPath = "",
string summaryPath = "",
string userNotes = "")
{
var notePath = "";
var frontmatter = new MeetingNoteFrontmatter
{
Title = title,
StartTime = startTime,
EndTime = endTime,
Attendees = attendees?.Where(value => !string.IsNullOrWhiteSpace(value)).ToList() ?? [],
Projects = projects?.Where(value => !string.IsNullOrWhiteSpace(value)).ToList() ?? [],
Transcript = transcriptPath,
AssistantContext = assistantContextPath,
Summary = summaryPath
};
return new MeetingNote(notePath, frontmatter, userNotes);
}
}
@@ -0,0 +1,7 @@
namespace MeetingAssistant.MeetingNotes;
public sealed record MeetingSessionArtifacts(
string MeetingNotePath,
string TranscriptPath,
string AssistantContextPath,
string SummaryPath);
@@ -0,0 +1,22 @@
namespace MeetingAssistant.MeetingNotes;
public static class ObsidianLink
{
public static string ToWikiLink(string pathOrLink, string sourceNotePath, string label)
{
if (string.IsNullOrWhiteSpace(pathOrLink) || pathOrLink.StartsWith("[[", StringComparison.Ordinal))
{
return pathOrLink;
}
var sourceFolder = Path.GetDirectoryName(sourceNotePath) ?? "";
var relativePath = Path.GetRelativePath(sourceFolder, pathOrLink);
var withoutExtension = Path.Combine(
Path.GetDirectoryName(relativePath) ?? "",
Path.GetFileNameWithoutExtension(relativePath))
.Replace(Path.DirectorySeparatorChar, '/')
.Replace(Path.AltDirectorySeparatorChar, '/');
return $"[[{withoutExtension}|{label}]]";
}
}
@@ -0,0 +1,26 @@
using System.Diagnostics;
namespace MeetingAssistant.MeetingNotes;
public sealed class ObsidianMeetingNoteOpener : IMeetingNoteOpener
{
private readonly ILogger<ObsidianMeetingNoteOpener> logger;
public ObsidianMeetingNoteOpener(ILogger<ObsidianMeetingNoteOpener> logger)
{
this.logger = logger;
}
public Task OpenAsync(string notePath, CancellationToken cancellationToken)
{
var uri = $"obsidian://open?path={Uri.EscapeDataString(Path.GetFullPath(notePath))}";
Process.Start(new ProcessStartInfo
{
FileName = uri,
UseShellExecute = true
});
logger.LogInformation("Opened meeting note in Obsidian via {ObsidianUri}", uri);
return Task.CompletedTask;
}
}
@@ -0,0 +1,289 @@
using System.Runtime.InteropServices;
namespace MeetingAssistant.MeetingNotes;
public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProvider
{
private const int OutlookCalendarFolder = 9;
private readonly ILogger<OutlookClassicMeetingMetadataProvider> logger;
public OutlookClassicMeetingMetadataProvider(ILogger<OutlookClassicMeetingMetadataProvider> logger)
{
this.logger = logger;
}
public Task<MeetingMetadata?> GetCurrentMeetingAsync(
DateTimeOffset startedAt,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return Task.FromResult(GetCurrentMeeting(startedAt));
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogDebug(exception, "Outlook Classic meeting metadata was not available");
return Task.FromResult<MeetingMetadata?>(null);
}
}
private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt)
{
object? application = null;
object? session = null;
object? calendar = null;
object? items = null;
object? restrictedItems = null;
try
{
var applicationType = Type.GetTypeFromProgID("Outlook.Application");
if (applicationType is null)
{
return null;
}
application = Activator.CreateInstance(applicationType);
if (application is null)
{
return null;
}
session = GetProperty(application, "Session");
calendar = Invoke(session!, "GetDefaultFolder", OutlookCalendarFolder);
items = GetProperty(calendar!, "Items");
SetProperty(items!, "IncludeRecurrences", true);
Invoke(items!, "Sort", "[Start]");
var startedLocal = startedAt.LocalDateTime;
var windowStart = startedLocal.AddMinutes(-1);
var windowEnd = startedLocal.AddMinutes(5);
var filter =
$"[Start] <= '{windowEnd:g}' AND [End] >= '{windowStart:g}'";
restrictedItems = Invoke(items!, "Restrict", filter);
var candidates = new List<OutlookAppointmentCandidate>();
var count = Convert.ToInt32(GetProperty(restrictedItems!, "Count"));
for (var index = 1; index <= count; index++)
{
var appointment = Invoke(restrictedItems!, "Item", index);
if (appointment is not null && IsTeamsAppointment(appointment))
{
candidates.Add(new OutlookAppointmentCandidate(
appointment,
Convert.ToDateTime(GetProperty(appointment, "Start")),
Convert.ToDateTime(GetProperty(appointment, "End"))));
}
else
{
ReleaseComObject(appointment);
}
}
var selected = OutlookMeetingCandidateSelector.Select(
candidates,
startedLocal,
candidate => candidate.Start,
candidate => candidate.End);
if (selected is null)
{
foreach (var candidate in candidates)
{
ReleaseComObject(candidate.Appointment);
}
return null;
}
try
{
var appointment = selected.Appointment;
var title = Convert.ToString(GetProperty(appointment, "Subject")) ?? "";
var attendees = ReadAttendees(appointment);
var body = Convert.ToString(GetProperty(appointment, "Body")) ?? "";
return new MeetingMetadata(
title.Trim(),
attendees,
ExtractAgenda(body),
ToLocalOffset(selected.End));
}
finally
{
foreach (var candidate in candidates)
{
ReleaseComObject(candidate.Appointment);
}
}
}
finally
{
ReleaseComObject(restrictedItems);
ReleaseComObject(items);
ReleaseComObject(calendar);
ReleaseComObject(session);
ReleaseComObject(application);
}
}
internal static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
private static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(GetProperty(appointment, "Body")) ?? "";
return ContainsTeamsMarker(subject) ||
ContainsTeamsMarker(location) ||
ContainsTeamsMarker(body);
}
private static bool ContainsTeamsMarker(string value)
{
return value.Contains("teams.microsoft.com", StringComparison.OrdinalIgnoreCase) ||
value.Contains("Microsoft Teams", StringComparison.OrdinalIgnoreCase) ||
value.Contains("Join the meeting now", StringComparison.OrdinalIgnoreCase) ||
value.Contains("Join Microsoft Teams Meeting", StringComparison.OrdinalIgnoreCase);
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return ContainsTeamsMarker(line);
}
private static IReadOnlyList<string> ReadAttendees(object appointment)
{
var attendees = new List<string>();
var organizer = Convert.ToString(GetProperty(appointment, "Organizer"));
if (!string.IsNullOrWhiteSpace(organizer))
{
attendees.Add(organizer.Trim());
}
object? recipients = null;
try
{
recipients = GetProperty(appointment, "Recipients");
var count = Convert.ToInt32(GetProperty(recipients!, "Count"));
for (var index = 1; index <= count; index++)
{
object? recipient = null;
try
{
recipient = Invoke(recipients!, "Item", index);
var formatted = FormatRecipient(recipient!);
if (!string.IsNullOrWhiteSpace(formatted))
{
attendees.Add(formatted);
}
}
finally
{
ReleaseComObject(recipient);
}
}
}
finally
{
ReleaseComObject(recipients);
}
return attendees
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static DateTimeOffset ToLocalOffset(DateTime value)
{
return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local));
}
private static string FormatRecipient(object recipient)
{
var name = Convert.ToString(GetProperty(recipient, "Name"))?.Trim() ?? "";
var email = Convert.ToString(GetProperty(recipient, "Address"))?.Trim() ?? "";
if (string.IsNullOrWhiteSpace(name))
{
return email;
}
if (string.IsNullOrWhiteSpace(email) ||
email.Contains('/'))
{
return name;
}
return $"{name} <{email}>";
}
private static object? GetProperty(object target, string name)
{
return target.GetType().InvokeMember(
name,
System.Reflection.BindingFlags.GetProperty,
null,
target,
null);
}
private static void SetProperty(object target, string name, object value)
{
target.GetType().InvokeMember(
name,
System.Reflection.BindingFlags.SetProperty,
null,
target,
[value]);
}
private static object? Invoke(object target, string name, params object[] arguments)
{
return target.GetType().InvokeMember(
name,
System.Reflection.BindingFlags.InvokeMethod,
null,
target,
arguments);
}
private static void ReleaseComObject(object? value)
{
if (value is not null && Marshal.IsComObject(value))
{
Marshal.FinalReleaseComObject(value);
}
}
private sealed record OutlookAppointmentCandidate(
object Appointment,
DateTime Start,
DateTime End);
}
@@ -0,0 +1,38 @@
namespace MeetingAssistant.MeetingNotes;
internal static class OutlookMeetingCandidateSelector
{
private static readonly TimeSpan MinimumRemainingOverlap = TimeSpan.FromMinutes(5);
private static readonly TimeSpan UpcomingStartWindow = TimeSpan.FromMinutes(5);
public static T? Select<T>(
IReadOnlyList<T> candidates,
DateTime startedAt,
Func<T, DateTime> getStart,
Func<T, DateTime> getEnd)
where T : class
{
var goodOverlaps = candidates
.Where(candidate =>
getStart(candidate) <= startedAt &&
getEnd(candidate) >= startedAt.Add(MinimumRemainingOverlap))
.ToList();
if (goodOverlaps.Count == 1)
{
return goodOverlaps[0];
}
if (goodOverlaps.Count > 1)
{
return null;
}
var upcoming = candidates
.Where(candidate =>
getStart(candidate) > startedAt &&
getStart(candidate) <= startedAt.Add(UpcomingStartWindow))
.OrderBy(getStart)
.ToList();
return upcoming.Count == 1 ? upcoming[0] : null;
}
}
+180 -1
View File
@@ -1,4 +1,51 @@
using MeetingAssistant;
using MeetingAssistant.Hotkeys;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Recording;
using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
#if WINDOWS
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeetingAudioSource(
services.GetRequiredService<MicrophoneAudioSource>(),
services.GetRequiredService<SystemAudioSource>(),
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
#else
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
#endif
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
builder.Services.AddSingleton<IMeetingNoteOpener, ObsidianMeetingNoteOpener>();
builder.Services.AddSingleton<IMeetingArtifactStore, MarkdownMeetingArtifactStore>();
builder.Services.AddSingleton<IRecordedAudioStore, TemporaryRecordedAudioStore>();
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
builder.Services.AddSingleton<IMeetingSummaryPipeline, OpenAiMeetingSummaryAgentPipeline>();
builder.Services.AddSingleton<AsrDiagnosticService>();
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
builder.Services.AddSingleton<IFunAsrBackendLifecycle, FunAsrDockerBackendLifecycle>();
builder.Services.AddSingleton<IFunAsrWebSocketConnectionFactory, ClientFunAsrWebSocketConnectionFactory>();
builder.Services.AddTransient<FunAsrStreamingTranscriptionProvider>();
builder.Services.AddTransient<WhisperLocalStreamingTranscriptionProvider>();
builder.Services.AddTransient<AzureSpeechStreamingTranscriptionProvider>();
builder.Services.AddTransient<FunAsrTranscriptFinalizer>();
builder.Services.AddTransient<PyannoteTranscriptFinalizer>();
builder.Services.AddSingleton<ISpeechRecognitionPipelineFactory, ConfiguredSpeechRecognitionPipelineFactory>();
builder.Services.AddSingleton<MeetingRecordingCoordinator>();
builder.Services.AddHostedService<TemporaryRecordingCleanupHostedService>();
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
#if WINDOWS
builder.Services.AddHostedService<GlobalHotkeyService>();
#endif
var app = builder.Build();
app.MapGet("/health", () => Results.Ok(new
@@ -7,6 +54,138 @@ app.MapGet("/health", () => Results.Ok(new
status = "ok"
}));
app.Run();
app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
app.MapPost("/recording/toggle", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.ToggleAsync(cancellationToken)));
app.MapPost("/recording/start", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StartAsync(cancellationToken)));
app.MapPost("/recording/stop", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StopAsync(cancellationToken)));
app.MapPost("/meetings/current/summary/run", async (
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
CancellationToken cancellationToken) =>
{
if (coordinator.CurrentArtifacts is null)
{
return Results.Conflict(new { error = "No meeting session has been started yet." });
}
return Results.Ok(await summaryPipeline.RunAsync(coordinator.CurrentArtifacts, cancellationToken));
});
app.MapPost("/meetings/summary/retry", async (
SummaryRetryRequest request,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(request.SummaryPath))
{
return Results.BadRequest(new { error = "A summary path is required." });
}
var artifacts = await artifactResolver.ResolveBySummaryPathAsync(request.SummaryPath, cancellationToken);
if (artifacts is null)
{
return Results.NotFound(new { error = $"No meeting note links to summary '{request.SummaryPath}'." });
}
return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken));
});
app.MapGet("/meetings/summary/retry", async (
string summaryPath,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(summaryPath))
{
return Results.BadRequest(new { error = "A summary path is required." });
}
var artifacts = await artifactResolver.ResolveBySummaryPathAsync(summaryPath, cancellationToken);
if (artifacts is null)
{
return Results.NotFound(new { error = $"No meeting note links to summary '{summaryPath}'." });
}
return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken));
});
app.MapPost("/asr/transcribe-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(request.Path))
{
return Results.BadRequest(new { error = "A WAV file path is required." });
}
try
{
return Results.Ok(await diagnostics.TranscribeWavAsync(request.Path, cancellationToken));
}
catch (FileNotFoundException exception)
{
return Results.NotFound(new { error = exception.Message });
}
catch (InvalidDataException exception)
{
return Results.BadRequest(new { error = exception.Message });
}
catch (Exception exception)
{
return Results.Problem(
detail: exception.Message,
statusCode: StatusCodes.Status502BadGateway,
title: "ASR backend failed while processing the WAV file.");
}
});
app.MapPost("/asr/diarize-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(request.Path))
{
return Results.BadRequest(new { error = "A WAV file path is required." });
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(request.Path));
if (!File.Exists(fullPath))
{
return Results.NotFound(new { error = $"WAV file was not found at '{fullPath}'." });
}
try
{
return Results.Ok(await diagnostics.DiarizeWavAsync(
fullPath,
new SpeechRecognitionPipelineOptions(request.NumSpeakers),
cancellationToken));
}
catch (Exception exception)
{
return Results.Problem(
detail: exception.Message,
statusCode: StatusCodes.Status502BadGateway,
title: "ASR diarization backend failed while processing the WAV file.");
}
});
try
{
app.Run();
}
catch (TimeoutException exception)
{
Console.Error.WriteLine(
$"Meeting Assistant failed to start because a required operation timed out: {exception.Message}");
Environment.ExitCode = 1;
}
public partial class Program;
public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null);
public sealed record SummaryRetryRequest(string SummaryPath);
+18
View File
@@ -0,0 +1,18 @@
namespace MeetingAssistant.Recording;
public sealed record AudioChunk(byte[] Pcm, int SampleRate, int Channels)
{
public TimeSpan Duration
{
get
{
var bytesPerSampleFrame = Channels * sizeof(short);
if (bytesPerSampleFrame <= 0 || SampleRate <= 0)
{
return TimeSpan.Zero;
}
return TimeSpan.FromSeconds((double)Pcm.Length / bytesPerSampleFrame / SampleRate);
}
}
}
@@ -0,0 +1,155 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
namespace MeetingAssistant.Recording;
public sealed class CompositeMeetingAudioSource : IMeetingAudioSource
{
private readonly IMeetingAudioSource microphone;
private readonly IMeetingAudioSource systemAudio;
private readonly ILogger<CompositeMeetingAudioSource> logger;
public CompositeMeetingAudioSource(
IMeetingAudioSource microphone,
IMeetingAudioSource systemAudio,
ILogger<CompositeMeetingAudioSource> logger)
{
this.microphone = microphone;
this.systemAudio = systemAudio;
this.logger = logger;
}
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded<SourceChunk>();
var microphoneTask = PumpAsync(AudioSourceKind.Microphone, microphone, channel.Writer, cancellationToken);
var systemTask = PumpAsync(AudioSourceKind.System, systemAudio, channel.Writer, cancellationToken);
_ = Task.WhenAll(microphoneTask, systemTask)
.ContinueWith(
task =>
{
if (task.Exception is not null)
{
channel.Writer.TryComplete(task.Exception);
return;
}
channel.Writer.TryComplete();
},
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
AudioChunk? pendingMicrophone = null;
AudioChunk? pendingSystem = null;
await foreach (var sourceChunk in channel.Reader.ReadAllAsync(cancellationToken))
{
if (sourceChunk.Kind == AudioSourceKind.Microphone)
{
pendingMicrophone = sourceChunk.Chunk;
}
else
{
pendingSystem = sourceChunk.Chunk;
}
if (pendingMicrophone is not null && pendingSystem is not null)
{
yield return Mix(pendingMicrophone, pendingSystem);
pendingMicrophone = null;
pendingSystem = null;
continue;
}
using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(50));
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token);
try
{
if (await channel.Reader.WaitToReadAsync(linked.Token))
{
continue;
}
}
catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
{
}
if (pendingMicrophone is not null)
{
yield return pendingMicrophone;
pendingMicrophone = null;
}
if (pendingSystem is not null)
{
yield return pendingSystem;
pendingSystem = null;
}
}
if (pendingMicrophone is not null)
{
yield return pendingMicrophone;
}
if (pendingSystem is not null)
{
yield return pendingSystem;
}
}
private static async Task PumpAsync(
AudioSourceKind kind,
IMeetingAudioSource source,
ChannelWriter<SourceChunk> writer,
CancellationToken cancellationToken)
{
await foreach (var chunk in source.CaptureAsync(cancellationToken))
{
await writer.WriteAsync(new SourceChunk(kind, chunk), cancellationToken);
}
}
private AudioChunk Mix(AudioChunk left, AudioChunk right)
{
if (left.SampleRate != right.SampleRate || left.Channels != right.Channels)
{
logger.LogWarning(
"Cannot mix audio chunks with different formats. Using microphone format {MicrophoneSampleRate}/{MicrophoneChannels} and dropping system chunk format {SystemSampleRate}/{SystemChannels}.",
left.SampleRate,
left.Channels,
right.SampleRate,
right.Channels);
return left;
}
var mixed = new byte[Math.Max(left.Pcm.Length, right.Pcm.Length)];
var sampleCount = mixed.Length / sizeof(short);
for (var sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++)
{
var byteIndex = sampleIndex * sizeof(short);
var leftSample = ReadSample(left.Pcm, byteIndex);
var rightSample = ReadSample(right.Pcm, byteIndex);
var sample = Math.Clamp(leftSample + rightSample, short.MinValue, short.MaxValue);
BitConverter.GetBytes((short)sample).CopyTo(mixed, byteIndex);
}
return new AudioChunk(mixed, left.SampleRate, left.Channels);
}
private static int ReadSample(byte[] pcm, int byteIndex)
{
return byteIndex + 1 < pcm.Length ? BitConverter.ToInt16(pcm, byteIndex) : 0;
}
private sealed record SourceChunk(AudioSourceKind Kind, AudioChunk Chunk);
private enum AudioSourceKind
{
Microphone,
System
}
}
@@ -0,0 +1,6 @@
namespace MeetingAssistant.Recording;
public interface IMeetingAudioSource
{
IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken);
}
@@ -0,0 +1,19 @@
namespace MeetingAssistant.Recording;
public interface IRecordedAudioStore
{
Task<IRecordedAudioSink> CreateSessionAsync(CancellationToken cancellationToken);
Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken);
}
public interface IRecordedAudioSink : IAsyncDisposable
{
string AudioPath { get; }
Task AppendAsync(AudioChunk chunk, CancellationToken cancellationToken);
Task CompleteAsync(CancellationToken cancellationToken);
Task DeleteAsync(CancellationToken cancellationToken);
}
@@ -0,0 +1,24 @@
using MeetingAssistant.Transcription;
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Recording;
public interface ITranscriptStore
{
Task<TranscriptSession> CreateSessionAsync(CancellationToken cancellationToken);
Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken);
Task ReplaceAsync(
TranscriptSession session,
IReadOnlyList<TranscriptionSegment> replacementSegments,
CancellationToken cancellationToken);
Task UpdateMetadataAsync(
TranscriptSession session,
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
CancellationToken cancellationToken);
}
public sealed record TranscriptSession(string TranscriptPath);
@@ -0,0 +1,449 @@
using System.Threading.Channels;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Recording;
public sealed class MeetingRecordingCoordinator
{
private readonly IMeetingAudioSource audioSource;
private readonly ISpeechRecognitionPipelineFactory speechRecognitionPipelineFactory;
private readonly ITranscriptStore transcriptStore;
private readonly IMeetingNoteStore meetingNoteStore;
private readonly IMeetingNoteOpener meetingNoteOpener;
private readonly IMeetingArtifactStore meetingArtifactStore;
private readonly IMeetingMetadataProvider meetingMetadataProvider;
private readonly IRecordedAudioStore recordedAudioStore;
private readonly IMeetingSummaryPipeline summaryPipeline;
private readonly MeetingAssistantOptions options;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private RecordingRun? currentRun;
private TranscriptSession? currentSession;
private MeetingNote? currentMeetingNote;
private MeetingSessionArtifacts? currentArtifacts;
public MeetingRecordingCoordinator(
IMeetingAudioSource audioSource,
ISpeechRecognitionPipelineFactory speechRecognitionPipelineFactory,
ITranscriptStore transcriptStore,
IMeetingNoteStore meetingNoteStore,
IMeetingNoteOpener meetingNoteOpener,
IMeetingArtifactStore meetingArtifactStore,
IRecordedAudioStore recordedAudioStore,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
ILogger<MeetingRecordingCoordinator> logger,
IMeetingMetadataProvider? meetingMetadataProvider = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
this.transcriptStore = transcriptStore;
this.meetingNoteStore = meetingNoteStore;
this.meetingNoteOpener = meetingNoteOpener;
this.meetingArtifactStore = meetingArtifactStore;
this.meetingMetadataProvider = meetingMetadataProvider ?? new NoopMeetingMetadataProvider();
this.recordedAudioStore = recordedAudioStore;
this.summaryPipeline = summaryPipeline;
this.options = options.Value;
this.logger = logger;
}
public RecordingStatus CurrentStatus => new(
IsRecording,
currentArtifacts?.TranscriptPath ?? currentSession?.TranscriptPath,
currentArtifacts?.MeetingNotePath ?? currentMeetingNote?.Path,
currentArtifacts?.AssistantContextPath,
currentArtifacts?.SummaryPath);
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
private bool IsRecording => currentRun is { IsCaptureStopping: false };
public async Task<RecordingStatus> ToggleAsync(CancellationToken cancellationToken)
{
return IsRecording
? await StopAsync(cancellationToken)
: await StartAsync(cancellationToken);
}
public async Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
if (IsRecording)
{
return CurrentStatus;
}
currentSession = await transcriptStore.CreateSessionAsync(cancellationToken);
var recordedAudio = await recordedAudioStore.CreateSessionAsync(cancellationToken);
var startedAt = DateTimeOffset.Now;
var assistantContextPath = GetAssistantContextPath(startedAt);
var summaryPath = GetSummaryPath(startedAt);
var meetingMetadata = await meetingMetadataProvider.GetCurrentMeetingAsync(startedAt, cancellationToken);
var title = string.IsNullOrWhiteSpace(meetingMetadata?.Title)
? $"Meeting {startedAt:yyyy-MM-dd HH:mm}"
: meetingMetadata.Title;
var attendees = meetingMetadata?.Attendees ?? [];
var agenda = meetingMetadata?.Agenda ?? "";
var scheduledEnd = meetingMetadata?.ScheduledEnd;
currentMeetingNote = await meetingNoteStore.SaveAsync(
MeetingNoteTemplate.Create(
title: title,
startTime: startedAt,
attendees: attendees,
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath),
cancellationToken);
currentArtifacts = new MeetingSessionArtifacts(
currentMeetingNote.Path,
currentSession.TranscriptPath,
assistantContextPath,
summaryPath);
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, agenda, scheduledEnd, cancellationToken);
await transcriptStore.UpdateMetadataAsync(
currentSession,
currentArtifacts,
currentMeetingNote,
cancellationToken);
await meetingNoteOpener.OpenAsync(currentMeetingNote.Path, cancellationToken);
var captureCancellation = new CancellationTokenSource();
var transcriptionCancellation = new CancellationTokenSource();
var pipeline = speechRecognitionPipelineFactory.Create();
var run = new RecordingRun(captureCancellation, transcriptionCancellation, recordedAudio, pipeline);
run.Task = Task.Run(() => RecordAsync(currentSession, run), CancellationToken.None);
currentRun = run;
logger.LogInformation("Meeting recording started");
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
RecordingRun run;
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is null || currentRun.IsCaptureStopping)
{
return CurrentStatus;
}
currentRun.StopCapture();
run = currentRun;
}
finally
{
gate.Release();
}
try
{
await run.Task.WaitAsync(options.Recording.StopProcessingTimeout, cancellationToken);
}
catch (OperationCanceledException)
{
}
catch (TimeoutException)
{
logger.LogWarning("Timed out while draining transcription after recording stop; forcing transcription cancellation");
run.CancelTranscription();
await run.Task.WaitAsync(cancellationToken);
}
await CompleteRunAsync(run);
logger.LogInformation("Meeting recording stopped");
return CurrentStatus;
}
private async Task RecordAsync(
TranscriptSession session,
RecordingRun run)
{
await run.Pipeline.InitializeAsync(run.TranscriptionCancellation);
var liveTranscriptTask = Task.Run(
() => StoreLiveTranscriptAsync(session, run.Pipeline, run.TranscriptionCancellation),
CancellationToken.None);
var captureTask = Task.Run(
() => CaptureAudioAsync(run.Pipeline, run.RecordedAudio, run.CaptureCancellation),
CancellationToken.None);
try
{
await captureTask;
await run.Pipeline.CompleteAsync(run.TranscriptionCancellation);
await liveTranscriptTask;
var artifacts = currentArtifacts;
if (artifacts is not null && HasConfiguredFinalizer())
{
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
AssistantContextState.SpeakerRecognition,
run.TranscriptionCancellation);
}
await RewriteTranscriptWithFinishedSegmentsAsync(session, run.Pipeline, run.RecordedAudio.AudioPath, run.TranscriptionCancellation);
var completedMeetingNote = await CompleteMeetingNoteAsync(run.TranscriptionCancellation);
if (artifacts is not null && completedMeetingNote is not null)
{
await transcriptStore.UpdateMetadataAsync(
session,
artifacts,
completedMeetingNote,
run.TranscriptionCancellation);
}
if (artifacts is not null)
{
await RunSummaryAsync(artifacts, run.TranscriptionCancellation);
}
}
catch (OperationCanceledException) when (run.TranscriptionCancellation.IsCancellationRequested)
{
}
catch (Exception exception) when (run.TranscriptionCancellation.IsCancellationRequested)
{
logger.LogInformation(exception, "Meeting recording stopped while the speech recognition pipeline was processing audio");
}
catch (Exception exception)
{
logger.LogError(exception, "Meeting recording failed");
}
finally
{
await run.RecordedAudio.DeleteAsync(CancellationToken.None);
await run.Pipeline.DisposeAsync();
await CompleteRunAsync(run);
}
}
private async Task CaptureAudioAsync(
ISpeechRecognitionPipeline pipeline,
IRecordedAudioSink recordedAudio,
CancellationToken cancellationToken)
{
try
{
await foreach (var chunk in audioSource.CaptureAsync(cancellationToken))
{
await recordedAudio.AppendAsync(chunk, cancellationToken);
await pipeline.WriteAsync(chunk, cancellationToken);
}
await recordedAudio.CompleteAsync(cancellationToken);
await pipeline.CompleteAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
await recordedAudio.CompleteAsync(CancellationToken.None);
await pipeline.CompleteAsync(CancellationToken.None);
}
}
private async Task StoreLiveTranscriptAsync(
TranscriptSession session,
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
{
await transcriptStore.AppendAsync(session, segment, cancellationToken);
}
}
private async Task RewriteTranscriptWithFinishedSegmentsAsync(
TranscriptSession session,
ISpeechRecognitionPipeline pipeline,
string audioPath,
CancellationToken cancellationToken)
{
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(
audioPath,
await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken),
cancellationToken);
if (finishedSegments.Count == 0)
{
return;
}
await transcriptStore.ReplaceAsync(session, finishedSegments, cancellationToken);
}
private async Task<MeetingNote?> CompleteMeetingNoteAsync(CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
{
return null;
}
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
meetingNote.Frontmatter.EndTime = DateTimeOffset.Now;
currentMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
return currentMeetingNote;
}
private async Task RunSummaryAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken)
{
try
{
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
AssistantContextState.Summarizing,
cancellationToken);
var result = await summaryPipeline.RunAsync(artifacts, cancellationToken);
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
result.Succeeded ? AssistantContextState.Finished : AssistantContextState.Error,
CancellationToken.None);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogError(exception, "Meeting summary pipeline failed unexpectedly");
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
AssistantContextState.Error,
CancellationToken.None);
}
}
private bool HasConfiguredFinalizer()
{
return options.Recording.TranscriptionProvider switch
{
"funasr" => options.FunAsr.Diarization.Enabled,
"whisper-local" => options.WhisperLocal.Diarization.Enabled,
_ => false
};
}
private async Task<SpeechRecognitionPipelineOptions> BuildSpeechRecognitionPipelineOptionsAsync(
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
{
return SpeechRecognitionPipelineOptions.Default;
}
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee));
return attendeeCount > 1
? new SpeechRecognitionPipelineOptions(attendeeCount)
: SpeechRecognitionPipelineOptions.Default;
}
private string GetAssistantContextPath(DateTimeOffset startedAt)
{
return GetMeetingArtifactPath(options.Vault.AssistantContextFolder, startedAt, "assistant-context");
}
private string GetSummaryPath(DateTimeOffset startedAt)
{
return GetMeetingArtifactPath(options.Vault.SummariesFolder, startedAt, "summary");
}
private static string GetMeetingArtifactPath(string configuredFolder, DateTimeOffset startedAt, string artifactName)
{
var folder = VaultPath.Resolve(configuredFolder);
return Path.Combine(folder, $"{startedAt:yyyyMMdd-HHmmss}-{artifactName}.md");
}
private async Task CompleteRunAsync(RecordingRun run)
{
if (!run.TryComplete())
{
return;
}
await gate.WaitAsync(CancellationToken.None);
try
{
if (ReferenceEquals(currentRun, run))
{
currentRun = null;
}
}
finally
{
gate.Release();
run.Dispose();
}
}
private sealed class RecordingRun : IDisposable
{
private int completed;
public RecordingRun(
CancellationTokenSource captureCancellation,
CancellationTokenSource transcriptionCancellation,
IRecordedAudioSink recordedAudio,
ISpeechRecognitionPipeline pipeline)
{
CaptureCancellationSource = captureCancellation;
TranscriptionCancellationSource = transcriptionCancellation;
RecordedAudio = recordedAudio;
Pipeline = pipeline;
}
public CancellationTokenSource CaptureCancellationSource { get; }
public CancellationTokenSource TranscriptionCancellationSource { get; }
public IRecordedAudioSink RecordedAudio { get; }
public ISpeechRecognitionPipeline Pipeline { get; }
public CancellationToken CaptureCancellation => CaptureCancellationSource.Token;
public CancellationToken TranscriptionCancellation => TranscriptionCancellationSource.Token;
public Task Task { get; set; } = Task.CompletedTask;
public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested;
public void StopCapture()
{
CaptureCancellationSource.Cancel();
}
public void CancelTranscription()
{
TranscriptionCancellationSource.Cancel();
}
public bool TryComplete()
{
return Interlocked.Exchange(ref completed, 1) == 0;
}
public void Dispose()
{
CaptureCancellationSource.Dispose();
TranscriptionCancellationSource.Dispose();
}
}
}
public sealed record RecordingStatus(
bool IsRecording,
string? TranscriptPath,
string? MeetingNotePath,
string? AssistantContextPath,
string? SummaryPath);
@@ -0,0 +1,86 @@
using System.Threading.Channels;
using Microsoft.Extensions.Options;
using NAudio.CoreAudioApi;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
private readonly MeetingAssistantOptions options;
public MicrophoneAudioSource(IOptions<MeetingAssistantOptions> options)
{
this.options = options.Value;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureAsync(new WasapiCapture(), cancellationToken);
}
private IAsyncEnumerable<AudioChunk> CaptureAsync(IWaveIn capture, CancellationToken cancellationToken)
{
capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
return CaptureWith(capture, cancellationToken);
}
internal static async IAsyncEnumerable<AudioChunk> CaptureWith(
IWaveIn capture,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded<AudioChunk>();
capture.DataAvailable += (_, args) =>
{
var buffer = new byte[args.BytesRecorded];
Buffer.BlockCopy(args.Buffer, 0, buffer, 0, args.BytesRecorded);
channel.Writer.TryWrite(new AudioChunk(buffer, capture.WaveFormat.SampleRate, capture.WaveFormat.Channels));
};
capture.RecordingStopped += (_, args) =>
{
if (args.Exception is not null)
{
channel.Writer.TryComplete(args.Exception);
return;
}
channel.Writer.TryComplete();
};
try
{
using var registration = cancellationToken.Register(capture.StopRecording);
capture.StartRecording();
await foreach (var chunk in channel.Reader.ReadAllAsync(cancellationToken))
{
yield return chunk;
}
}
finally
{
capture.Dispose();
}
}
}
public sealed class SystemAudioSource : IMeetingAudioSource
{
private readonly MeetingAssistantOptions options;
public SystemAudioSource(IOptions<MeetingAssistantOptions> options)
{
this.options = options.Value;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
var capture = new WasapiLoopbackCapture
{
WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels)
};
return MicrophoneAudioSource.CaptureWith(capture, cancellationToken);
}
}
@@ -0,0 +1,141 @@
using Microsoft.Extensions.Options;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<TemporaryRecordedAudioStore> logger;
public TemporaryRecordedAudioStore(
IOptions<MeetingAssistantOptions> options,
ILogger<TemporaryRecordedAudioStore> logger)
{
this.options = options.Value;
this.logger = logger;
}
public Task<IRecordedAudioSink> CreateSessionAsync(CancellationToken cancellationToken)
{
var folder = GetTemporaryRecordingsFolder();
Directory.CreateDirectory(folder);
var path = Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Guid.NewGuid():N}-recording.wav");
var format = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
logger.LogInformation("Created temporary meeting recording file {RecordingPath}", path);
return Task.FromResult<IRecordedAudioSink>(new TemporaryRecordedAudioSink(path, format, logger));
}
public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken)
{
var folder = GetTemporaryRecordingsFolder();
if (!Directory.Exists(folder))
{
return Task.CompletedTask;
}
foreach (var path in Directory.EnumerateFiles(folder, "*.wav", SearchOption.TopDirectoryOnly))
{
cancellationToken.ThrowIfCancellationRequested();
DeleteFile(path);
}
return Task.CompletedTask;
}
private string GetTemporaryRecordingsFolder()
{
return VaultPath.Resolve(options.Recording.TemporaryRecordingsFolder);
}
private void DeleteFile(string path)
{
try
{
File.Delete(path);
logger.LogInformation("Deleted temporary meeting recording file {RecordingPath}", path);
}
catch (FileNotFoundException)
{
}
catch (DirectoryNotFoundException)
{
}
catch (IOException exception)
{
logger.LogWarning(exception, "Could not delete temporary meeting recording file {RecordingPath}", path);
}
catch (UnauthorizedAccessException exception)
{
logger.LogWarning(exception, "Could not delete temporary meeting recording file {RecordingPath}", path);
}
}
private sealed class TemporaryRecordedAudioSink : IRecordedAudioSink
{
private readonly WaveFileWriter writer;
private readonly ILogger logger;
private bool completed;
private bool disposed;
public TemporaryRecordedAudioSink(string audioPath, WaveFormat format, ILogger logger)
{
AudioPath = audioPath;
this.logger = logger;
writer = new WaveFileWriter(audioPath, format);
}
public string AudioPath { get; }
public Task AppendAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
writer.Write(chunk.Pcm, 0, chunk.Pcm.Length);
return Task.CompletedTask;
}
public Task CompleteAsync(CancellationToken cancellationToken)
{
if (completed)
{
return Task.CompletedTask;
}
cancellationToken.ThrowIfCancellationRequested();
writer.Flush();
completed = true;
return Task.CompletedTask;
}
public ValueTask DisposeAsync()
{
if (disposed)
{
return ValueTask.CompletedTask;
}
writer.Dispose();
disposed = true;
return ValueTask.CompletedTask;
}
public async Task DeleteAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
await DisposeAsync();
try
{
File.Delete(AudioPath);
logger.LogInformation("Deleted temporary meeting recording file {RecordingPath}", AudioPath);
}
catch (FileNotFoundException)
{
}
catch (DirectoryNotFoundException)
{
}
}
}
}
@@ -0,0 +1,26 @@
namespace MeetingAssistant.Recording;
public sealed class TemporaryRecordingCleanupHostedService : IHostedService
{
private readonly IRecordedAudioStore recordedAudioStore;
private readonly ILogger<TemporaryRecordingCleanupHostedService> logger;
public TemporaryRecordingCleanupHostedService(
IRecordedAudioStore recordedAudioStore,
ILogger<TemporaryRecordingCleanupHostedService> logger)
{
this.recordedAudioStore = recordedAudioStore;
this.logger = logger;
}
public async Task StartAsync(CancellationToken cancellationToken)
{
logger.LogInformation("Deleting stale temporary meeting recordings");
await recordedAudioStore.DeleteStaleRecordingsAsync(cancellationToken);
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
@@ -0,0 +1,10 @@
namespace MeetingAssistant.Recording;
public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource
{
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
throw new PlatformNotSupportedException(
"Meeting audio capture is only implemented for the Windows target in this version.");
}
}
@@ -0,0 +1,113 @@
using MeetingAssistant.Transcription;
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Recording;
public sealed class VaultTranscriptStore : ITranscriptStore
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<VaultTranscriptStore> logger;
public VaultTranscriptStore(IOptions<MeetingAssistantOptions> options, ILogger<VaultTranscriptStore> logger)
{
this.options = options.Value;
this.logger = logger;
}
public async Task<TranscriptSession> CreateSessionAsync(CancellationToken cancellationToken)
{
var folder = VaultPath.Resolve(options.Vault.TranscriptsFolder);
Directory.CreateDirectory(folder);
var fileName = $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-transcript.md";
var path = Path.Combine(folder, fileName);
await File.WriteAllTextAsync(path, $"# Meeting Transcript{Environment.NewLine}{Environment.NewLine}", cancellationToken);
logger.LogInformation("Created meeting transcript file {TranscriptPath}", path);
return new TranscriptSession(path);
}
public Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken)
{
return AppendToBodyAsync(session.TranscriptPath, FormatSegment(segment), cancellationToken);
}
public Task ReplaceAsync(
TranscriptSession session,
IReadOnlyList<TranscriptionSegment> replacementSegments,
CancellationToken cancellationToken)
{
var existing = File.Exists(session.TranscriptPath)
? File.ReadAllText(session.TranscriptPath)
: "";
var (frontmatter, _) = MeetingArtifactFrontmatterRenderer.Split(existing);
var body = "# Meeting Transcript"
+ Environment.NewLine
+ Environment.NewLine
+ string.Concat(replacementSegments.Select(FormatSegment));
var content = string.IsNullOrWhiteSpace(frontmatter)
? body
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + body;
return File.WriteAllTextAsync(session.TranscriptPath, content, cancellationToken);
}
public async Task UpdateMetadataAsync(
TranscriptSession session,
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
CancellationToken cancellationToken)
{
var body = File.Exists(session.TranscriptPath)
? MeetingArtifactFrontmatterRenderer.Split(await File.ReadAllTextAsync(session.TranscriptPath, cancellationToken)).Body
: "# Meeting Transcript" + Environment.NewLine + Environment.NewLine;
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
artifacts,
meetingNote,
MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Transcript"),
session.TranscriptPath);
await File.WriteAllTextAsync(
session.TranscriptPath,
MeetingArtifactFrontmatterRenderer.Render(frontmatter, body),
cancellationToken);
}
private static string FormatSegment(TranscriptionSegment segment)
{
var speaker = string.IsNullOrWhiteSpace(segment.Speaker) ? "Unknown" : segment.Speaker;
return $"[{segment.Start:hh\\:mm\\:ss}] {speaker}: {segment.Text}{Environment.NewLine}";
}
private static async Task AppendToBodyAsync(
string path,
string text,
CancellationToken cancellationToken)
{
if (!File.Exists(path))
{
await File.AppendAllTextAsync(path, text, cancellationToken);
return;
}
var content = await File.ReadAllTextAsync(path, cancellationToken);
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(content);
if (string.IsNullOrWhiteSpace(frontmatter))
{
await File.AppendAllTextAsync(path, text, cancellationToken);
return;
}
var updated = "---"
+ Environment.NewLine
+ frontmatter
+ Environment.NewLine
+ "---"
+ Environment.NewLine
+ Environment.NewLine
+ body
+ text;
await File.WriteAllTextAsync(path, updated, cancellationToken);
}
}
@@ -0,0 +1,14 @@
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Summary;
public interface IMeetingSummaryPipeline
{
Task<MeetingSummaryRunResult> RunAsync(MeetingSessionArtifacts artifacts, CancellationToken cancellationToken);
}
public sealed record MeetingSummaryRunResult(
string SummaryPath,
string AgentResponse,
bool Succeeded = true,
string? Error = null);
@@ -0,0 +1,618 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Logging;
namespace MeetingAssistant.Summary;
#pragma warning disable MAAI001
public sealed class LiteLlmResponsesChatClient : IChatClient
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly HttpClient httpClient;
private readonly string model;
private readonly bool enableThinking;
private readonly string reasoningEffort;
private readonly int reconnectionAttempts;
private readonly TimeSpan reconnectionDelay;
private readonly LiteLlmResponsesCompactionOptions? compactionOptions;
private readonly ILogger? logger;
public LiteLlmResponsesChatClient(
Uri endpoint,
string apiKey,
string model,
bool enableThinking,
string reasoningEffort,
int reconnectionAttempts,
TimeSpan reconnectionDelay,
LiteLlmResponsesCompactionOptions? compactionOptions = null,
ILogger? logger = null)
: this(
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
apiKey,
model,
enableThinking,
reasoningEffort,
reconnectionAttempts,
reconnectionDelay,
compactionOptions,
logger)
{
}
internal LiteLlmResponsesChatClient(
HttpClient httpClient,
string apiKey,
string model,
bool enableThinking,
string reasoningEffort,
int reconnectionAttempts,
TimeSpan reconnectionDelay,
LiteLlmResponsesCompactionOptions? compactionOptions = null,
ILogger? logger = null)
{
this.httpClient = httpClient;
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
this.model = model;
this.enableThinking = enableThinking;
this.reasoningEffort = reasoningEffort;
this.reconnectionAttempts = Math.Max(0, reconnectionAttempts);
this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero;
this.compactionOptions = compactionOptions;
this.logger = logger;
}
public void Dispose()
{
httpClient.Dispose();
}
public async Task<ChatResponse> GetResponseAsync(
IEnumerable<ChatMessage> messages,
ChatOptions? options = null,
CancellationToken cancellationToken = default)
{
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
var response = ParseResponseJson(responseJson);
LogResponseUsage(response.Usage);
return response;
}
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
IEnumerable<ChatMessage> messages,
ChatOptions? options = null,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false);
foreach (var message in response.Messages)
{
yield return new ChatResponseUpdate(message.Role, message.Contents);
}
}
public object? GetService(Type serviceType, object? serviceKey = null)
{
if (serviceKey is not null)
{
return null;
}
return serviceType == typeof(ChatClientMetadata)
? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model)
: null;
}
public static ChatResponse ParseResponseJson(string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement;
var contents = new List<AIContent>();
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
{
foreach (var item in output.EnumerateArray())
{
var type = GetString(item, "type");
if (type == "message")
{
AddMessageContent(contents, item);
}
else if (type == "function_call")
{
contents.Add(new FunctionCallContent(
GetRequiredString(item, "call_id"),
GetRequiredString(item, "name"),
ParseArguments(GetString(item, "arguments"))));
}
}
}
var message = new ChatMessage
{
Role = ChatRole.Assistant,
Contents = contents
};
return new ChatResponse(message)
{
ResponseId = GetString(root, "id"),
ModelId = GetString(root, "model"),
CreatedAt = GetUnixTimestamp(root, "created_at"),
Usage = ParseUsage(root),
RawRepresentation = responseJson
};
}
private async Task<JsonObject> CreateCompactedPayloadAsync(
IReadOnlyList<ChatMessage> messages,
ChatOptions? options,
CancellationToken cancellationToken)
{
var payload = CreatePayload(messages, options);
var estimate = EstimateTokens(payload);
var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue;
if (compactionOptions?.Enabled != true || estimate < triggerTokens)
{
LogContextWindow(estimate, compacted: false, source: "none");
return payload;
}
LogContextWindow(estimate, compacted: false, source: "threshold");
var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken)
.ConfigureAwait(false);
if (remotePayload is not null)
{
LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact");
return remotePayload;
}
if (compactionOptions.FallbackStrategy is null)
{
return payload;
}
try
{
var compactedMessages = await CompactionProvider.CompactAsync(
compactionOptions.FallbackStrategy,
messages,
logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance,
cancellationToken)
.ConfigureAwait(false);
var fallbackPayload = CreatePayload(compactedMessages, options);
LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework");
return fallbackPayload;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger?.LogWarning(
exception,
"Agent Framework compaction fallback failed; sending un-compacted summary request");
return payload;
}
}
private async Task<JsonObject?> TryCompactWithResponsesEndpointAsync(
JsonObject payload,
int estimatedTokens,
CancellationToken cancellationToken)
{
try
{
var request = new JsonObject
{
["model"] = payload["model"]?.DeepClone(),
["input"] = payload["input"]?.DeepClone(),
["instructions"] = payload["instructions"]?.DeepClone(),
["context_window_tokens"] = compactionOptions!.ContextWindowTokens,
["max_output_tokens"] = compactionOptions.MaxOutputTokens,
["estimated_input_tokens"] = estimatedTokens,
["target_input_tokens"] = compactionOptions.TriggerTokens,
["strategy"] = new JsonArray
{
"collapse_tool_results",
"summarize_older_spans",
"keep_last_4_user_turns",
"drop_oldest_groups"
}
};
using var content = new StringContent(request.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
using var response = await httpClient.PostAsync(
NormalizeRelativePath(compactionOptions.CompactPath),
content,
cancellationToken)
.ConfigureAwait(false);
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
logger?.LogWarning(
"Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}",
(int)response.StatusCode,
response.ReasonPhrase,
responseJson);
return null;
}
return ApplyCompactedInput(payload, responseJson);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction");
return null;
}
}
private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement;
var input = FindCompactedInput(root);
if (input is null)
{
return null;
}
var compactedPayload = (JsonObject)payload.DeepClone();
compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText());
if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String)
{
compactedPayload["instructions"] = instructions.GetString();
}
return compactedPayload;
}
private static JsonElement? FindCompactedInput(JsonElement root)
{
foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" })
{
if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array)
{
return value;
}
}
return null;
}
private JsonObject CreatePayload(IEnumerable<ChatMessage> messages, ChatOptions? options)
{
var instructions = new StringBuilder();
if (!string.IsNullOrWhiteSpace(options?.Instructions))
{
instructions.AppendLine(options.Instructions);
}
var input = new JsonArray();
foreach (var message in messages)
{
AddInputItem(input, instructions, message);
}
var payload = new JsonObject
{
["model"] = options?.ModelId ?? model,
["input"] = input,
["store"] = false,
["tool_choice"] = "auto"
};
if (instructions.Length > 0)
{
payload["instructions"] = instructions.ToString().Trim();
}
if (enableThinking)
{
payload["reasoning"] = new JsonObject
{
["effort"] = reasoningEffort
};
}
var tools = CreateTools(options?.Tools);
if (tools.Count > 0)
{
payload["tools"] = tools;
payload["parallel_tool_calls"] = true;
}
return payload;
}
private async Task<string> PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken)
{
var payloadJson = payload.ToJsonString(JsonOptions);
Exception? lastException = null;
for (var attempt = 0; attempt <= reconnectionAttempts; attempt++)
{
try
{
using var content = new StringContent(payloadJson, Encoding.UTF8, "application/json");
using var response = await httpClient.PostAsync("responses", content, cancellationToken).ConfigureAwait(false);
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return responseJson;
}
var exception = new InvalidOperationException(
$"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts)
{
throw exception;
}
lastException = exception;
}
catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts)
{
lastException = exception;
}
await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false);
}
throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed.");
}
private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken)
{
if (reconnectionDelay > TimeSpan.Zero)
{
await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false);
}
}
private static bool IsRetryableStatusCode(int statusCode)
{
return statusCode is 408 or 409 or 425 or 429
|| statusCode >= 500;
}
private static bool IsRetryableException(Exception exception)
{
return exception is HttpRequestException or TaskCanceledException;
}
private void LogContextWindow(int estimatedTokens, bool compacted, string source)
{
if (compactionOptions is null || logger is null)
{
return;
}
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
var remainingTokens = Math.Max(0, contextWindow - estimatedTokens);
logger.LogInformation(
"Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}",
estimatedTokens,
contextWindow,
remainingTokens,
compacted,
source);
}
private void LogResponseUsage(UsageDetails? usage)
{
if (usage is null || compactionOptions is null || logger is null)
{
return;
}
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
var inputTokens = usage.InputTokenCount ?? 0;
var outputTokens = usage.OutputTokenCount ?? 0;
var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens;
logger.LogInformation(
"Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}",
inputTokens,
outputTokens,
totalTokens,
contextWindow);
}
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
{
var role = message.Role.Value;
var text = message.Text;
if (role == ChatRole.System.Value)
{
if (!string.IsNullOrWhiteSpace(text))
{
instructions.AppendLine(text);
}
return;
}
if (!string.IsNullOrWhiteSpace(text))
{
input.Add(new JsonObject
{
["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user",
["content"] = text
});
}
foreach (var content in message.Contents)
{
if (content is FunctionCallContent call)
{
input.Add(new JsonObject
{
["type"] = "function_call",
["call_id"] = call.CallId,
["name"] = call.Name,
["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions)
});
}
else if (content is FunctionResultContent result)
{
input.Add(new JsonObject
{
["type"] = "function_call_output",
["call_id"] = result.CallId,
["output"] = ResultToString(result.Result)
});
}
}
}
private static JsonArray CreateTools(IList<AITool>? tools)
{
var result = new JsonArray();
if (tools is null)
{
return result;
}
foreach (var tool in tools.OfType<AIFunctionDeclaration>())
{
result.Add(new JsonObject
{
["type"] = "function",
["name"] = tool.Name,
["description"] = tool.Description ?? string.Empty,
["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText())
});
}
return result;
}
private static void AddMessageContent(List<AIContent> contents, JsonElement item)
{
if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array)
{
return;
}
foreach (var content in messageContent.EnumerateArray())
{
if (GetString(content, "type") == "output_text")
{
var text = GetString(content, "text");
if (!string.IsNullOrEmpty(text))
{
contents.Add(new TextContent(text));
}
}
}
}
private static Dictionary<string, object?> ParseArguments(string? arguments)
{
if (string.IsNullOrWhiteSpace(arguments))
{
return [];
}
using var document = JsonDocument.Parse(arguments);
return document.RootElement.EnumerateObject()
.ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value));
}
private static object? ConvertJsonValue(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number when element.TryGetInt64(out var integer) => integer,
JsonValueKind.Number when element.TryGetDouble(out var number) => number,
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.Null => null,
_ => JsonSerializer.Deserialize<object>(element.GetRawText(), JsonOptions)
};
}
private static string ResultToString(object? result)
{
return result switch
{
null => string.Empty,
string text => text,
_ => JsonSerializer.Serialize(result, JsonOptions)
};
}
private static UsageDetails? ParseUsage(JsonElement root)
{
if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object)
{
return null;
}
var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens");
var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens");
var totalTokens = GetInt(usage, "total_tokens");
return new UsageDetails
{
InputTokenCount = inputTokens,
OutputTokenCount = outputTokens,
TotalTokenCount = totalTokens
};
}
private static int EstimateTokens(JsonObject payload)
{
var json = payload.ToJsonString(JsonOptions);
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
}
private static int? GetInt(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value)
? value
: null;
}
private static string GetRequiredString(JsonElement element, string propertyName)
{
return GetString(element, propertyName)
?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'.");
}
private static string? GetString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
? property.GetString()
: null;
}
private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value)
? DateTimeOffset.FromUnixTimeSeconds(value)
: null;
}
private static Uri NormalizeEndpoint(Uri endpoint)
{
var value = endpoint.ToString().TrimEnd('/');
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
{
value += "/v1";
}
return new Uri(value + "/");
}
private static string NormalizeRelativePath(string value)
{
return value.TrimStart('/');
}
}
#pragma warning restore MAAI001
@@ -0,0 +1,32 @@
using Microsoft.Agents.AI.Compaction;
namespace MeetingAssistant.Summary;
#pragma warning disable MAAI001
public sealed class LiteLlmResponsesCompactionOptions
{
public bool Enabled { get; init; }
public int ContextWindowTokens { get; init; }
public int MaxOutputTokens { get; init; }
public double RemainingRatio { get; init; }
public string CompactPath { get; init; } = "responses/compact";
public CompactionStrategy? FallbackStrategy { get; init; }
public int TriggerTokens
{
get
{
var contextWindow = Math.Max(1, ContextWindowTokens);
var outputReserve = Math.Clamp(MaxOutputTokens, 0, contextWindow - 1);
var inputBudget = contextWindow - outputReserve;
var remainingRatio = Math.Clamp(RemainingRatio, 0.01, 0.90);
return Math.Max(1, (int)Math.Floor(inputBudget * (1 - remainingRatio)));
}
}
}
#pragma warning restore MAAI001
@@ -0,0 +1,126 @@
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Summary;
public interface IMeetingSummaryArtifactResolver
{
Task<MeetingSessionArtifacts?> ResolveBySummaryPathAsync(
string summaryPath,
CancellationToken cancellationToken);
}
public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactResolver
{
private readonly MeetingAssistantOptions options;
private readonly IMeetingNoteStore meetingNoteStore;
public MeetingSummaryArtifactResolver(
IOptions<MeetingAssistantOptions> options,
IMeetingNoteStore meetingNoteStore)
{
this.options = options.Value;
this.meetingNoteStore = meetingNoteStore;
}
public async Task<MeetingSessionArtifacts?> ResolveBySummaryPathAsync(
string summaryPath,
CancellationToken cancellationToken)
{
var requestedSummaryPath = ResolveRequestedSummaryPath(summaryPath);
if (requestedSummaryPath is null)
{
return null;
}
var meetingNotesFolder = VaultPath.Resolve(options.Vault.MeetingNotesFolder);
if (!Directory.Exists(meetingNotesFolder))
{
return null;
}
foreach (var meetingNotePath in Directory.EnumerateFiles(meetingNotesFolder, "*.md"))
{
var note = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
var noteSummaryPath = ResolveNoteLink(note.Frontmatter.Summary, note.Path);
if (!PathsEqual(requestedSummaryPath, noteSummaryPath))
{
continue;
}
return new MeetingSessionArtifacts(
note.Path,
ResolveNoteLink(note.Frontmatter.Transcript, note.Path),
ResolveNoteLink(note.Frontmatter.AssistantContext, note.Path),
requestedSummaryPath);
}
return null;
}
private string? ResolveRequestedSummaryPath(string summaryPath)
{
if (string.IsNullOrWhiteSpace(summaryPath))
{
return null;
}
var summariesFolder = VaultPath.Resolve(options.Vault.SummariesFolder);
var requested = Environment.ExpandEnvironmentVariables(summaryPath);
var fullPath = Path.IsPathRooted(requested)
? Path.GetFullPath(requested)
: Path.GetFullPath(Path.Combine(summariesFolder, requested));
return IsWithinDirectory(summariesFolder, fullPath) ? fullPath : null;
}
private static string ResolveNoteLink(string pathOrLink, string sourceNotePath)
{
if (string.IsNullOrWhiteSpace(pathOrLink))
{
return "";
}
var sourceFolder = Path.GetDirectoryName(sourceNotePath) ?? "";
var target = ExtractWikiLinkTarget(pathOrLink) ?? pathOrLink;
target = target.Replace('/', Path.DirectorySeparatorChar);
if (!Path.HasExtension(target))
{
target += ".md";
}
return Path.GetFullPath(Path.IsPathRooted(target)
? target
: Path.Combine(sourceFolder, target));
}
private static string? ExtractWikiLinkTarget(string value)
{
if (!value.StartsWith("[[", StringComparison.Ordinal) ||
!value.EndsWith("]]", StringComparison.Ordinal))
{
return null;
}
var inner = value[2..^2];
var labelSeparator = inner.IndexOf('|', StringComparison.Ordinal);
return labelSeparator < 0 ? inner : inner[..labelSeparator];
}
private static bool PathsEqual(string left, string right)
{
return string.Equals(
Path.GetFullPath(left),
Path.GetFullPath(right),
StringComparison.OrdinalIgnoreCase);
}
private static bool IsWithinDirectory(string directory, string path)
{
var normalizedDirectory = Path.GetFullPath(directory)
.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var normalizedPath = Path.GetFullPath(path);
return normalizedPath.StartsWith(normalizedDirectory, StringComparison.OrdinalIgnoreCase);
}
}
@@ -0,0 +1,83 @@
using System.Text;
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Summary;
public interface IMeetingSummaryFailureWriter
{
Task<MeetingSummaryRunResult> WriteAsync(
MeetingSessionArtifacts artifacts,
Exception exception,
CancellationToken cancellationToken);
}
public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter
{
private readonly MeetingAssistantOptions options;
private readonly IMeetingNoteStore meetingNoteStore;
public MeetingSummaryFailureWriter(
IOptions<MeetingAssistantOptions> options,
IMeetingNoteStore meetingNoteStore)
{
this.options = options.Value;
this.meetingNoteStore = meetingNoteStore;
}
public async Task<MeetingSummaryRunResult> WriteAsync(
MeetingSessionArtifacts artifacts,
Exception exception,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!);
var meetingNote = File.Exists(artifacts.MeetingNotePath)
? await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken)
: new MeetingNote("", new MeetingNoteFrontmatter(), "");
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
artifacts,
meetingNote,
MeetingArtifactFrontmatterRenderer.DefaultTitle(meetingNote, "Meeting Summary"),
artifacts.SummaryPath);
await File.WriteAllTextAsync(
artifacts.SummaryPath,
MeetingArtifactFrontmatterRenderer.Render(
frontmatter,
RenderFailureMarkdown(artifacts, exception)),
cancellationToken);
return new MeetingSummaryRunResult(
artifacts.SummaryPath,
"Summary generation failed. See the summary file for error details.",
Succeeded: false,
Error: exception.Message);
}
private string RenderFailureMarkdown(MeetingSessionArtifacts artifacts, Exception exception)
{
var builder = new StringBuilder();
builder.AppendLine("# Meeting Summary");
builder.AppendLine();
builder.AppendLine("## Summary Generation Failed");
builder.AppendLine();
builder.AppendLine($"Summary generation failed at {DateTimeOffset.Now:O}.");
builder.AppendLine();
builder.AppendLine(MeetingNoteActionLinks.CreateSummaryRetryLink(
options.Api.PublicBaseUrl,
artifacts.SummaryPath));
builder.AppendLine();
builder.AppendLine("## Error");
builder.AppendLine();
builder.AppendLine("```text");
builder.AppendLine(exception.ToString().Replace("```", "` ` `", StringComparison.Ordinal));
builder.AppendLine("```");
builder.AppendLine();
builder.AppendLine("## Meeting Artifacts");
builder.AppendLine();
builder.AppendLine($"- Meeting note: `{artifacts.MeetingNotePath}`");
builder.AppendLine($"- Transcript: `{artifacts.TranscriptPath}`");
builder.AppendLine($"- Assistant context: `{artifacts.AssistantContextPath}`");
builder.AppendLine($"- Summary: `{artifacts.SummaryPath}`");
return builder.ToString();
}
}
@@ -0,0 +1,641 @@
using MeetingAssistant.MeetingNotes;
using System.Diagnostics;
using System.Text.Json;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
public sealed class MeetingSummaryTools
{
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
private readonly MeetingSessionArtifacts artifacts;
private readonly MeetingAssistantOptions options;
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
: this(artifacts, new MeetingAssistantOptions())
{
}
public MeetingSummaryTools(MeetingSessionArtifacts artifacts, MeetingAssistantOptions options)
{
this.artifacts = artifacts;
this.options = options;
}
public Task<string> ReadTranscript(int? @from = null, int? to = null)
{
return ReadIfExistsAsync(artifacts.TranscriptPath, @from, to);
}
public Task<string> ReadMeetingNote(int? @from = null, int? to = null)
{
return ReadIfExistsAsync(artifacts.MeetingNotePath, @from, to);
}
public Task<string> ReadContext(int? @from = null, int? to = null)
{
return ReadIfExistsAsync(artifacts.AssistantContextPath, @from, to);
}
public async Task<string> ReadUserNotes(int? @from = null, int? to = null)
{
if (!File.Exists(artifacts.MeetingNotePath))
{
return "";
}
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
var document = MarkdownDocumentParser.SplitOptional(content);
var body = document.HasFrontmatter ? document.Body.Trim() : content;
return ReadLines(body, @from, to);
}
public Task<string> ReadGlossary(int? @from = null, int? to = null)
{
return ReadIfExistsAsync(VaultPath.Resolve(options.Vault.DictationWordsPath), @from, to);
}
public async Task<string> WriteSummary(string markdown, string? title = null)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!);
var meetingNote = await ReadMeetingNoteAsync();
var summaryTitle = string.IsNullOrWhiteSpace(meetingNote.Frontmatter.Title)
? title
: meetingNote.Frontmatter.Title;
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
artifacts,
meetingNote,
string.IsNullOrWhiteSpace(summaryTitle) ? "Meeting Summary" : summaryTitle,
artifacts.SummaryPath);
await File.WriteAllTextAsync(
artifacts.SummaryPath,
MeetingArtifactFrontmatterRenderer.Render(frontmatter, markdown));
return artifacts.SummaryPath;
}
public async Task<string> WriteContext(
string content,
int? @from = null,
int? to = null,
int? insert = null)
{
var editMode = FileLineEditMode.Create(@from, to, insert);
if (editMode is null)
{
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for overwrite.";
}
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
var existing = File.Exists(artifacts.AssistantContextPath)
? await File.ReadAllTextAsync(artifacts.AssistantContextPath)
: "";
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(existing);
var updatedBody = ApplyLineEdit(body, content, editMode);
var updated = string.IsNullOrWhiteSpace(frontmatter)
? updatedBody
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + updatedBody;
await File.WriteAllTextAsync(artifacts.AssistantContextPath, updated);
return artifacts.AssistantContextPath;
}
public async Task<string> ListProjects()
{
var projects = await GetBoundProjectsAsync();
return string.Join('\n', projects.Select(project => project.Name));
}
public async Task<string> ListProjectFiles(string project)
{
var projectRoot = await ResolveBoundProjectRootAsync(project);
if (projectRoot is null)
{
return "";
}
var files = Directory.EnumerateFiles(projectRoot, "*", SearchOption.AllDirectories)
.Select(path => ToToolPath(Path.GetRelativePath(projectRoot, path)))
.Order(StringComparer.Ordinal);
return string.Join('\n', files);
}
public async Task<string> ReadProjectFile(string project, string path, int? @from = null, int? to = null)
{
var filePath = await ResolveBoundProjectFilePathAsync(project, path);
if (filePath is null || !File.Exists(filePath))
{
return "";
}
return ReadLines(await File.ReadAllTextAsync(filePath), @from, to);
}
public async Task<string> WriteProjectFile(
string project,
string path,
string content,
int? @from = null,
int? to = null,
int? insert = null)
{
var editMode = FileLineEditMode.Create(@from, to, insert);
if (editMode is null)
{
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for overwrite.";
}
var target = ResolveExistingProjectFilePath(project, path);
if (target is null)
{
return "Refused: project does not exist or the path escapes the project folder.";
}
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
await WriteProjectFileContentAsync(target.Path, content, editMode);
return $"{target.Project.Name}/{ToToolPath(path)}";
}
public async Task<string> Search(string keywords, string[]? projects = null)
{
if (string.IsNullOrWhiteSpace(keywords))
{
return "";
}
var selectedProjects = await GetSearchProjectsAsync(projects);
if (selectedProjects.Count == 0)
{
return "";
}
var projectsRoot = GetProjectsRoot();
var result = await RunRipgrepAsync(projectsRoot, selectedProjects, keywords);
return result is not null
? FormatRipgrepJson(result, selectedProjects.Count == 1)
: SearchWithRegexFallback(selectedProjects, keywords, selectedProjects.Count == 1);
}
private static async Task<string> ReadIfExistsAsync(string path, int? from = null, int? to = null)
{
if (!File.Exists(path))
{
return "";
}
var content = await File.ReadAllTextAsync(path);
return ReadLines(content, from, to);
}
private async Task<MeetingNote> ReadMeetingNoteAsync()
{
if (!File.Exists(artifacts.MeetingNotePath))
{
return new MeetingNote("", new MeetingNoteFrontmatter(), "");
}
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
var document = MarkdownDocumentParser.SplitOptional(content);
var frontmatter = new MeetingNoteFrontmatter();
if (document.HasFrontmatter)
{
var note = YamlDeserializer.Deserialize<MeetingNoteFrontmatterYaml>(document.Frontmatter) ?? new MeetingNoteFrontmatterYaml();
frontmatter.Title = note.Title ?? "";
frontmatter.StartTime = ParseDateTime(note.StartTime);
frontmatter.EndTime = ParseDateTime(note.EndTime);
frontmatter.Attendees = note.Attendees ?? [];
frontmatter.Projects = note.Projects ?? [];
frontmatter.Transcript = note.Transcript ?? "";
frontmatter.AssistantContext = note.AssistantContext ?? "";
frontmatter.Summary = note.Summary ?? "";
}
return new MeetingNote(artifacts.MeetingNotePath, frontmatter, await ReadUserNotes());
}
private async Task<List<ProjectFolder>> GetSearchProjectsAsync(string[]? projects)
{
var boundProjects = await GetBoundProjectsAsync();
if (projects is null || projects.Length == 0)
{
return boundProjects;
}
var requested = projects
.Where(project => !string.IsNullOrWhiteSpace(project))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
return boundProjects
.Where(project => requested.Contains(project.Name))
.ToList();
}
private async Task<ProjectFolder?> ResolveBoundProjectAsync(string project)
{
if (string.IsNullOrWhiteSpace(project))
{
return null;
}
return (await GetBoundProjectsAsync())
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
}
private async Task<string?> ResolveBoundProjectRootAsync(string project)
{
return (await ResolveBoundProjectAsync(project))?.Path;
}
private async Task<string?> ResolveBoundProjectFilePathAsync(string project, string path)
{
var projectRoot = await ResolveBoundProjectRootAsync(project);
if (projectRoot is null || string.IsNullOrWhiteSpace(path) || Path.IsPathRooted(path))
{
return null;
}
var fullPath = Path.GetFullPath(Path.Combine(projectRoot, path));
return IsWithinDirectory(projectRoot, fullPath) ? fullPath : null;
}
private ProjectFileTarget? ResolveExistingProjectFilePath(string project, string path)
{
if (string.IsNullOrWhiteSpace(project) ||
string.IsNullOrWhiteSpace(path) ||
Path.IsPathRooted(path))
{
return null;
}
var projectsRoot = GetProjectsRoot();
if (!Directory.Exists(projectsRoot))
{
return null;
}
var projectFolder = Directory.EnumerateDirectories(projectsRoot)
.Select(candidate => new ProjectFolder(Path.GetFileName(candidate), candidate))
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
if (projectFolder is null)
{
return null;
}
var fullPath = Path.GetFullPath(Path.Combine(projectFolder.Path, path));
return IsWithinDirectory(projectFolder.Path, fullPath)
? new ProjectFileTarget(projectFolder, fullPath)
: null;
}
private static async Task WriteProjectFileContentAsync(
string filePath,
string content,
FileLineEditMode editMode)
{
if (editMode.Kind == FileLineEditKind.Overwrite)
{
await File.WriteAllTextAsync(filePath, content);
return;
}
var lines = File.Exists(filePath)
? (await File.ReadAllLinesAsync(filePath)).ToList()
: [];
var replacementLines = SplitLines(content);
if (editMode.Kind == FileLineEditKind.Insert)
{
var insertIndex = Math.Clamp(editMode.Insert!.Value - 1, 0, lines.Count);
lines.InsertRange(insertIndex, replacementLines);
await File.WriteAllTextAsync(filePath, string.Join('\n', lines));
return;
}
var startIndex = Math.Clamp(editMode.From!.Value - 1, 0, lines.Count);
var endIndexExclusive = Math.Clamp(editMode.To!.Value, 0, lines.Count);
if (endIndexExclusive < startIndex)
{
endIndexExclusive = startIndex;
}
lines.RemoveRange(startIndex, endIndexExclusive - startIndex);
lines.InsertRange(startIndex, replacementLines);
await File.WriteAllTextAsync(filePath, string.Join('\n', lines));
}
private static List<string> SplitLines(string content)
{
if (content.Length == 0)
{
return [];
}
return content
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n')
.ToList();
}
private static string ReadLines(string content, int? from = null, int? to = null)
{
if (!from.HasValue && !to.HasValue)
{
return content;
}
var lines = SplitLines(content);
if (lines.Count == 0)
{
return "";
}
var start = Math.Clamp((from ?? 1) - 1, 0, lines.Count - 1);
var end = Math.Clamp((to ?? lines.Count) - 1, 0, lines.Count - 1);
if (end < start)
{
return "";
}
return string.Join('\n', lines.GetRange(start, end - start + 1));
}
private static string ApplyLineEdit(
string existingContent,
string replacementContent,
FileLineEditMode editMode)
{
if (editMode.Kind == FileLineEditKind.Overwrite)
{
return replacementContent;
}
var lines = SplitLines(existingContent);
var replacementLines = SplitLines(replacementContent);
if (editMode.Kind == FileLineEditKind.Insert)
{
var insertIndex = Math.Clamp(editMode.Insert!.Value - 1, 0, lines.Count);
lines.InsertRange(insertIndex, replacementLines);
return string.Join('\n', lines);
}
var startIndex = Math.Clamp(editMode.From!.Value - 1, 0, lines.Count);
var endIndexExclusive = Math.Clamp(editMode.To!.Value, 0, lines.Count);
if (endIndexExclusive < startIndex)
{
endIndexExclusive = startIndex;
}
lines.RemoveRange(startIndex, endIndexExclusive - startIndex);
lines.InsertRange(startIndex, replacementLines);
return string.Join('\n', lines);
}
private async Task<List<ProjectFolder>> GetBoundProjectsAsync()
{
var projectsRoot = GetProjectsRoot();
if (!Directory.Exists(projectsRoot))
{
return [];
}
var projectNames = await ReadMeetingProjectNamesAsync();
if (projectNames.Count == 0)
{
return [];
}
return Directory.EnumerateDirectories(projectsRoot)
.Select(path => new ProjectFolder(Path.GetFileName(path), path))
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
private async Task<HashSet<string>> ReadMeetingProjectNamesAsync()
{
if (!File.Exists(artifacts.MeetingNotePath))
{
return [];
}
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter)
{
return [];
}
var note = YamlDeserializer.Deserialize<ProjectFrontmatter>(document.Frontmatter) ?? new ProjectFrontmatter();
return (note.Projects ?? [])
.Where(project => !string.IsNullOrWhiteSpace(project))
.Select(project => project.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private string GetProjectsRoot()
{
return VaultPath.Resolve(options.Vault.ProjectsFolder);
}
private static async Task<string?> RunRipgrepAsync(
string projectsRoot,
IReadOnlyList<ProjectFolder> projects,
string keywords)
{
try
{
var startInfo = new ProcessStartInfo
{
FileName = "rg",
WorkingDirectory = projectsRoot,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
startInfo.ArgumentList.Add("--json");
startInfo.ArgumentList.Add("--line-number");
startInfo.ArgumentList.Add("--color");
startInfo.ArgumentList.Add("never");
startInfo.ArgumentList.Add("--");
startInfo.ArgumentList.Add(keywords);
foreach (var project in projects)
{
startInfo.ArgumentList.Add(project.Name);
}
using var process = Process.Start(startInfo);
if (process is null)
{
return null;
}
var output = await process.StandardOutput.ReadToEndAsync();
await process.WaitForExitAsync();
return process.ExitCode is 0 or 1 ? output : null;
}
catch
{
return null;
}
}
private static string FormatRipgrepJson(string output, bool singleProject)
{
var matches = new List<string>();
using var reader = new StringReader(output);
string? line;
while ((line = reader.ReadLine()) is not null)
{
using var document = JsonDocument.Parse(line);
var root = document.RootElement;
if (!root.TryGetProperty("type", out var type) ||
type.GetString() != "match" ||
!root.TryGetProperty("data", out var data))
{
continue;
}
var path = data.GetProperty("path").GetProperty("text").GetString() ?? "";
var lineNumber = data.GetProperty("line_number").GetInt32();
var text = data.GetProperty("lines").GetProperty("text").GetString() ?? "";
matches.Add($"{FormatSearchPath(path, singleProject)}:{lineNumber} {text.TrimEnd('\r', '\n')}");
}
return string.Join('\n', matches);
}
private static string SearchWithRegexFallback(IReadOnlyList<ProjectFolder> projects, string keywords, bool singleProject)
{
try
{
var regex = new System.Text.RegularExpressions.Regex(keywords);
var matches = new List<string>();
foreach (var project in projects)
{
foreach (var file in Directory.EnumerateFiles(project.Path, "*", SearchOption.AllDirectories))
{
var lines = File.ReadAllLines(file);
for (var index = 0; index < lines.Length; index++)
{
if (regex.IsMatch(lines[index]))
{
var relative = Path.Combine(project.Name, Path.GetRelativePath(project.Path, file));
matches.Add($"{FormatSearchPath(relative, singleProject)}:{index + 1} {lines[index]}");
}
}
}
}
return string.Join('\n', matches);
}
catch
{
return "";
}
}
private static string FormatSearchPath(string path, bool singleProject)
{
var formatted = ToToolPath(path);
if (!singleProject)
{
return formatted;
}
var slashIndex = formatted.IndexOf('/', StringComparison.Ordinal);
return slashIndex < 0 ? formatted : formatted[(slashIndex + 1)..];
}
private static bool IsWithinDirectory(string directory, string path)
{
var normalizedDirectory = Path.GetFullPath(directory)
.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var normalizedPath = Path.GetFullPath(path);
return normalizedPath.StartsWith(normalizedDirectory, StringComparison.OrdinalIgnoreCase);
}
private static string ToToolPath(string path)
{
return path.Replace(Path.DirectorySeparatorChar.ToString(), "/", StringComparison.Ordinal)
.Replace(Path.AltDirectorySeparatorChar.ToString(), "/", StringComparison.Ordinal);
}
private sealed record ProjectFolder(string Name, string Path);
private sealed record ProjectFileTarget(ProjectFolder Project, string Path);
private sealed record FileLineEditMode(
FileLineEditKind Kind,
int? From = null,
int? To = null,
int? Insert = null)
{
public static FileLineEditMode? Create(int? from, int? to, int? insert)
{
if (insert.HasValue)
{
return from.HasValue || to.HasValue
? null
: new FileLineEditMode(FileLineEditKind.Insert, Insert: insert);
}
if (from.HasValue || to.HasValue)
{
return from.HasValue && to.HasValue && from.Value <= to.Value
? new FileLineEditMode(FileLineEditKind.Replace, From: from, To: to)
: null;
}
return new FileLineEditMode(FileLineEditKind.Overwrite);
}
}
private enum FileLineEditKind
{
Overwrite,
Replace,
Insert
}
private sealed class ProjectFrontmatter
{
[YamlDotNet.Serialization.YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
}
private sealed class MeetingNoteFrontmatterYaml
{
[YamlDotNet.Serialization.YamlMember(Alias = "title")]
public string? Title { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "start_time")]
public string? StartTime { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "end_time")]
public string? EndTime { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "attendees")]
public List<string>? Attendees { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "transcript")]
public string? Transcript { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "assistant_context")]
public string? AssistantContext { get; set; }
[YamlDotNet.Serialization.YamlMember(Alias = "summary")]
public string? Summary { get; set; }
}
private static DateTimeOffset? ParseDateTime(string? value)
{
return DateTimeOffset.TryParse(value, out var parsed)
? parsed
: null;
}
}
@@ -0,0 +1,296 @@
using MeetingAssistant.MeetingNotes;
using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Summary;
#pragma warning disable MAAI001
public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
{
private const string Instructions = """
You are the Meeting Assistant summary agent.
Use the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files.
All read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines.
Then write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary.
Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.
Use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note.
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
Keep the output grounded in the source material and explicitly say when a section has no known items.
""";
private readonly MeetingAssistantOptions options;
private readonly ILoggerFactory loggerFactory;
private readonly ILogger<OpenAiMeetingSummaryAgentPipeline> logger;
private readonly IServiceProvider services;
private readonly IMeetingSummaryFailureWriter failureWriter;
public OpenAiMeetingSummaryAgentPipeline(
IOptions<MeetingAssistantOptions> options,
ILoggerFactory loggerFactory,
ILogger<OpenAiMeetingSummaryAgentPipeline> logger,
IServiceProvider services,
IMeetingSummaryFailureWriter failureWriter)
{
this.options = options.Value;
this.loggerFactory = loggerFactory;
this.logger = logger;
this.services = services;
this.failureWriter = failureWriter;
}
public async Task<MeetingSummaryRunResult> RunAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken)
{
var agentOptions = options.Agent;
var key = ResolveApiKey(agentOptions);
var tools = CreateTools(new MeetingSummaryTools(artifacts, options));
using var compactionSummaryClient = new LiteLlmResponsesChatClient(
new Uri(agentOptions.Endpoint),
key,
agentOptions.Model,
agentOptions.EnableThinking,
ToReasoningEffortValue(agentOptions.ReasoningEffort),
agentOptions.ReconnectionAttempts,
agentOptions.ReconnectionDelay,
compactionOptions: null,
logger);
var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient);
using var chatClient = new LiteLlmResponsesChatClient(
new Uri(agentOptions.Endpoint),
key,
agentOptions.Model,
agentOptions.EnableThinking,
ToReasoningEffortValue(agentOptions.ReasoningEffort),
agentOptions.ReconnectionAttempts,
agentOptions.ReconnectionDelay,
compactionOptions,
logger);
var agent = chatClient
.AsBuilder()
.UseFunctionInvocation()
.Build()
.AsAIAgent(
Instructions,
"meeting_summary",
"Writes the markdown summary note for a captured meeting.",
tools,
loggerFactory: loggerFactory,
services: services);
try
{
var response = await agent.RunAsync(
"Write the summary for this meeting. Use the tools; do not invent missing information.",
session: null,
options: new ChatClientAgentRunOptions(CreateChatOptions(agentOptions)),
cancellationToken: cancellationToken);
return new MeetingSummaryRunResult(artifacts.SummaryPath, response.Text);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogError(
exception,
"Meeting summary generation failed for {SummaryPath}; writing failure summary",
artifacts.SummaryPath);
return await failureWriter.WriteAsync(artifacts, exception, cancellationToken);
}
}
private static List<AITool> CreateTools(MeetingSummaryTools tools)
{
return
[
AIFunctionFactory.Create(
tools.ReadMeetingNote,
"read_meetingnote",
"Read the meeting note, including YAML frontmatter and user-authored body notes. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."),
AIFunctionFactory.Create(
tools.ReadTranscript,
"read_transcript",
"Read the transcript markdown for the current meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."),
AIFunctionFactory.Create(
tools.ReadContext,
"read_context",
"Read the assistant context markdown for the current meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range."),
AIFunctionFactory.Create(
tools.ReadUserNotes,
"read_usernotes",
"Read the user-authored body notes from the meeting note. With no line arguments, read the whole body. With from and to, read that clamped inclusive 1-based body line range."),
AIFunctionFactory.Create(
tools.WriteContext,
"write_context",
"Write internal assistant notes to the assistant context body. With no line arguments, overwrite the body. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."),
AIFunctionFactory.Create(
tools.ReadGlossary,
"read_glossary",
"Read the glossary or vocabulary context for this meeting. With no line arguments, read the whole file. With from and to, read that clamped inclusive 1-based line range. May be empty."),
AIFunctionFactory.Create(
tools.WriteSummary,
"write_summary",
"Write the finished summary markdown to the configured summary note. Provide title when the meeting note has no title."),
AIFunctionFactory.Create(
tools.ListProjects,
"list_projects",
"List the configured project folders bound to the current meeting note frontmatter."),
AIFunctionFactory.Create(
tools.ListProjectFiles,
"list_projectfiles",
"List markdown and other files in a bound project. The project parameter is a project folder name."),
AIFunctionFactory.Create(
tools.ReadProjectFile,
"read_projectfile",
"Read a bound project file, optionally clamping to from and to inclusive line numbers."),
AIFunctionFactory.Create(
tools.WriteProjectFile,
"write_projectfile",
"Write a file inside an existing project folder. With no line arguments, overwrite or create the file. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."),
AIFunctionFactory.Create(
tools.Search,
"search",
"Run ripgrep syntax keywords against the requested bound projects, or all projects bound to the meeting note.")
];
}
private static ChatOptions CreateChatOptions(AgentOptions options)
{
var chatOptions = new ChatOptions
{
ModelId = options.Model,
AllowMultipleToolCalls = true,
ToolMode = ChatToolMode.Auto
};
if (options.EnableThinking)
{
chatOptions.Reasoning = new ReasoningOptions
{
Effort = ToReasoningEffort(options.ReasoningEffort)
};
}
return chatOptions;
}
private static LiteLlmResponsesCompactionOptions CreateCompactionOptions(
AgentOptions options,
IChatClient? summaryClient)
{
return new LiteLlmResponsesCompactionOptions
{
Enabled = options.EnableCompaction,
ContextWindowTokens = options.ContextWindowTokens,
MaxOutputTokens = options.MaxOutputTokens,
RemainingRatio = options.CompactionRemainingRatio,
CompactPath = options.ResponsesCompactPath,
FallbackStrategy = CreateFallbackCompactionStrategy(
options.ContextWindowTokens,
options.MaxOutputTokens,
options.CompactionRemainingRatio,
summaryClient)
};
}
internal static CompactionStrategy CreateFallbackCompactionStrategyForTests(
int contextWindowTokens,
int maxOutputTokens,
double remainingRatio,
IChatClient? summaryClient)
{
return CreateFallbackCompactionStrategy(
contextWindowTokens,
maxOutputTokens,
remainingRatio,
summaryClient);
}
private static CompactionStrategy CreateFallbackCompactionStrategy(
int contextWindowTokens,
int maxOutputTokens,
double remainingRatio,
IChatClient? summaryClient)
{
var contextWindow = Math.Max(1, contextWindowTokens);
var outputReserve = Math.Clamp(maxOutputTokens, 0, contextWindow - 1);
var inputBudget = contextWindow - outputReserve;
var triggerTokens = Math.Max(1, (int)Math.Floor(inputBudget * (1 - Math.Clamp(remainingRatio, 0.01, 0.90))));
var target = CompactionTriggers.TokensBelow(Math.Max(1, triggerTokens - 1));
var trigger = CompactionTriggers.TokensExceed(triggerTokens);
var strategies = new List<CompactionStrategy>
{
new ToolResultCompactionStrategy(trigger, minimumPreservedGroups: 4, target)
};
if (summaryClient is not null)
{
strategies.Add(new SummarizationCompactionStrategy(
summaryClient,
trigger,
minimumPreservedGroups: 4,
target: target));
}
strategies.Add(new SlidingWindowCompactionStrategy(
trigger,
minimumPreservedTurns: 4,
target));
strategies.Add(new TruncationCompactionStrategy(
trigger,
minimumPreservedGroups: 1,
target));
return new PipelineCompactionStrategy(strategies);
}
private static string ToReasoningEffortValue(ReasoningEffortOption effort)
{
return effort switch
{
ReasoningEffortOption.None => "none",
ReasoningEffortOption.Low => "low",
ReasoningEffortOption.High => "high",
ReasoningEffortOption.ExtraHigh => "xhigh",
_ => "medium"
};
}
private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort)
{
return effort switch
{
ReasoningEffortOption.None => ReasoningEffort.None,
ReasoningEffortOption.Low => ReasoningEffort.Low,
ReasoningEffortOption.High => ReasoningEffort.High,
ReasoningEffortOption.ExtraHigh => ReasoningEffort.ExtraHigh,
_ => ReasoningEffort.Medium
};
}
private static string ResolveApiKey(AgentOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Key))
{
return options.Key;
}
var value = Environment.GetEnvironmentVariable(options.KeyEnv);
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
throw new InvalidOperationException(
$"No agent API key configured. Set MeetingAssistant:Agent:Key or environment variable '{options.KeyEnv}'.");
}
}
#pragma warning restore MAAI001
@@ -0,0 +1,119 @@
using MeetingAssistant.Recording;
using NAudio.Wave;
namespace MeetingAssistant.Transcription;
public sealed class AsrDiagnosticService
{
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
public AsrDiagnosticService(ISpeechRecognitionPipelineFactory pipelineFactory)
{
this.pipelineFactory = pipelineFactory;
}
public async Task<AsrDiagnosticResult> TranscribeWavAsync(string path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("A WAV file path is required.", nameof(path));
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
}
await using var pipeline = pipelineFactory.Create();
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
await pipeline.CompleteAsync(cancellationToken);
return new AsrDiagnosticResult(fullPath, await liveTranscriptTask);
}
public async Task<AsrDiagnosticResult> DiarizeWavAsync(
string path,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
var fullPath = ResolveExistingWavPath(path);
await using var pipeline = pipelineFactory.Create();
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
await pipeline.CompleteAsync(cancellationToken);
await liveTranscriptTask;
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(fullPath, options, cancellationToken);
return new AsrDiagnosticResult(fullPath, finishedSegments);
}
private static async Task<IReadOnlyList<TranscriptionSegment>> CollectLiveTranscriptAsync(
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
var segments = new List<TranscriptionSegment>();
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
{
segments.Add(segment);
}
return segments;
}
private static async Task WriteWavToPipelineAsync(
string fullPath,
ISpeechRecognitionPipeline pipeline,
bool paceAudio,
CancellationToken cancellationToken)
{
await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken))
{
await pipeline.WriteAsync(chunk, cancellationToken);
if (paceAudio && chunk.Duration > TimeSpan.Zero)
{
await Task.Delay(chunk.Duration, cancellationToken);
}
}
}
private static string ResolveExistingWavPath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("A WAV file path is required.", nameof(path));
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
}
return fullPath;
}
private static async IAsyncEnumerable<AudioChunk> ReadPcmWavChunks(
string path,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
using var reader = new WaveFileReader(path);
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
throw new InvalidDataException(
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
}
var buffer = new byte[format.AverageBytesPerSecond];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
}
public sealed record AsrDiagnosticResult(string Path, IReadOnlyList<TranscriptionSegment> Segments);
@@ -0,0 +1,23 @@
namespace MeetingAssistant.Transcription;
public sealed class AzureSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
public AzureSpeechRecognitionPipeline(AzureSpeechStreamingTranscriptionProvider transcriptionProvider)
: base(transcriptionProvider)
{
}
internal AzureSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
: base(transcriptionProvider)
{
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(liveSegments);
}
}
@@ -0,0 +1,190 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.CognitiveServices.Speech;
using Microsoft.CognitiveServices.Speech.Audio;
using Microsoft.CognitiveServices.Speech.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<AzureSpeechStreamingTranscriptionProvider> logger;
public AzureSpeechStreamingTranscriptionProvider(
IOptions<MeetingAssistantOptions> options,
ILogger<AzureSpeechStreamingTranscriptionProvider> logger)
{
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
if (!await enumerator.MoveNextAsync())
{
yield break;
}
var firstChunk = enumerator.Current;
using var pushStream = AudioInputStream.CreatePushStream(
AudioStreamFormat.GetWaveFormatPCM(
(uint)firstChunk.SampleRate,
16,
(byte)firstChunk.Channels));
using var audioConfig = AudioConfig.FromStreamInput(pushStream);
var speechConfig = CreateSpeechConfig();
using var recognizer = new ConversationTranscriber(speechConfig, audioConfig);
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
recognizer.Transcribed += (_, args) =>
{
if (args.Result.Reason != ResultReason.RecognizedSpeech
|| string.IsNullOrWhiteSpace(args.Result.Text))
{
return;
}
var start = TimeSpan.FromTicks(args.Result.OffsetInTicks);
var segment = new TranscriptionSegment(
start,
start + args.Result.Duration,
NormalizeSpeakerId(args.Result.SpeakerId),
args.Result.Text.Trim());
logger.LogInformation(
"Azure Speech emitted segment at {Start} from {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
segments.Writer.TryWrite(segment);
};
recognizer.Canceled += (_, args) =>
{
if (args.Reason == CancellationReason.Error)
{
segments.Writer.TryComplete(new InvalidOperationException(
$"Azure Speech recognition canceled: {args.ErrorCode} {args.ErrorDetails}"));
return;
}
segments.Writer.TryComplete();
};
await recognizer.StartTranscribingAsync();
var pumpTask = Task.Run(
() => PumpAudioAsync(
firstChunk,
enumerator,
pushStream,
recognizer,
segments.Writer,
options.AzureSpeech.RecognitionStopTimeout,
cancellationToken),
CancellationToken.None);
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
await pumpTask.WaitAsync(cancellationToken);
}
private SpeechConfig CreateSpeechConfig()
{
var azure = options.AzureSpeech;
var key = ResolveKey(azure);
if (string.IsNullOrWhiteSpace(key))
{
throw new InvalidOperationException(
$"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
}
var speechConfig = string.IsNullOrWhiteSpace(azure.Region)
? SpeechConfig.FromEndpoint(new Uri(azure.Endpoint), key)
: SpeechConfig.FromSubscription(key, azure.Region);
speechConfig.SpeechRecognitionLanguage = azure.Language;
speechConfig.OutputFormat = OutputFormat.Detailed;
speechConfig.SetProperty(
PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
azure.DiarizeIntermediateResults ? "true" : "false");
return speechConfig;
}
private static async Task PumpAudioAsync(
AudioChunk firstChunk,
IAsyncEnumerator<AudioChunk> audio,
PushAudioInputStream pushStream,
ConversationTranscriber recognizer,
ChannelWriter<TranscriptionSegment> segments,
TimeSpan recognitionStopTimeout,
CancellationToken cancellationToken)
{
try
{
WriteChunk(pushStream, firstChunk, firstChunk);
while (await audio.MoveNextAsync())
{
cancellationToken.ThrowIfCancellationRequested();
WriteChunk(pushStream, firstChunk, audio.Current);
}
pushStream.Close();
try
{
var stopTask = recognizer.StopTranscribingAsync();
if (recognitionStopTimeout > TimeSpan.Zero)
{
await stopTask.WaitAsync(recognitionStopTimeout, cancellationToken);
}
else
{
await stopTask.WaitAsync(cancellationToken);
}
}
catch (TimeoutException)
{
// The SDK can outlive short diagnostic streams while waiting for final service events.
}
segments.TryComplete();
}
catch (Exception exception)
{
segments.TryComplete(exception);
}
}
private static void WriteChunk(PushAudioInputStream pushStream, AudioChunk expectedFormat, AudioChunk chunk)
{
if (chunk.SampleRate != expectedFormat.SampleRate || chunk.Channels != expectedFormat.Channels)
{
return;
}
pushStream.Write(chunk.Pcm);
}
private static string NormalizeSpeakerId(string? speakerId)
{
return string.IsNullOrWhiteSpace(speakerId) || speakerId.Equals("Unknown", StringComparison.OrdinalIgnoreCase)
? "Unknown"
: speakerId;
}
private static string ResolveKey(AzureSpeechOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Key))
{
return options.Key;
}
return string.IsNullOrWhiteSpace(options.KeyEnv)
? ""
: Environment.GetEnvironmentVariable(options.KeyEnv) ?? "";
}
}
@@ -0,0 +1,78 @@
using System.Net.WebSockets;
using System.Text;
namespace MeetingAssistant.Transcription;
public sealed class ClientFunAsrWebSocketConnectionFactory : IFunAsrWebSocketConnectionFactory
{
public async Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken)
{
var socket = new ClientWebSocket();
await socket.ConnectAsync(endpoint, cancellationToken);
return new ClientFunAsrWebSocketConnection(socket);
}
}
public sealed class ClientFunAsrWebSocketConnection : IFunAsrWebSocketConnection
{
private readonly ClientWebSocket socket;
public ClientFunAsrWebSocketConnection(ClientWebSocket socket)
{
this.socket = socket;
}
public Task SendTextAsync(string message, CancellationToken cancellationToken)
{
return socket.SendAsync(
Encoding.UTF8.GetBytes(message),
WebSocketMessageType.Text,
endOfMessage: true,
cancellationToken);
}
public async Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken)
{
await socket.SendAsync(message, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken);
}
public async Task<string?> ReceiveTextAsync(CancellationToken cancellationToken)
{
var buffer = new byte[8192];
using var message = new MemoryStream();
while (true)
{
var result = await socket.ReceiveAsync(buffer, cancellationToken);
if (result.MessageType == WebSocketMessageType.Close)
{
return null;
}
if (result.MessageType != WebSocketMessageType.Text)
{
continue;
}
message.Write(buffer, 0, result.Count);
if (result.EndOfMessage)
{
return Encoding.UTF8.GetString(message.ToArray());
}
}
}
public async Task CloseOutputAsync(CancellationToken cancellationToken)
{
if (socket.State is WebSocketState.Open or WebSocketState.CloseReceived)
{
await socket.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "meeting assistant finished sending audio", cancellationToken);
}
}
public ValueTask DisposeAsync()
{
socket.Dispose();
return ValueTask.CompletedTask;
}
}
@@ -0,0 +1,101 @@
using System.Diagnostics;
using System.Text;
namespace MeetingAssistant.Transcription;
public interface ICommandRunner
{
Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null);
}
public sealed class ProcessCommandRunner : ICommandRunner
{
private readonly ILogger<ProcessCommandRunner> logger;
public ProcessCommandRunner(ILogger<ProcessCommandRunner> logger)
{
this.logger = logger;
}
public async Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
var startInfo = new ProcessStartInfo
{
FileName = fileName,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
foreach (var argument in arguments)
{
startInfo.ArgumentList.Add(argument);
}
if (environment is not null)
{
foreach (var (key, value) in environment)
{
startInfo.Environment[key] = value;
}
}
logger.LogInformation("Running command {Command} {Arguments}", fileName, string.Join(' ', arguments));
using var process = Process.Start(startInfo)
?? throw new InvalidOperationException($"Failed to start command '{fileName}'.");
var outputTask = ReadLinesAsync(process.StandardOutput, line => logger.LogInformation("{Command}: {Line}", fileName, line), cancellationToken);
var errorTask = ReadLinesAsync(process.StandardError, line => logger.LogWarning("{Command}: {Line}", fileName, line), cancellationToken);
try
{
await process.WaitForExitAsync(cancellationToken);
}
catch (OperationCanceledException)
{
if (!process.HasExited)
{
logger.LogWarning("Cancelling command {Command}; killing process tree.", fileName);
process.Kill(entireProcessTree: true);
}
throw;
}
var result = new CommandResult(process.ExitCode, await outputTask, await errorTask);
if (result.ExitCode != 0)
{
logger.LogWarning(
"Command {Command} exited with {ExitCode}: {Error}",
fileName,
result.ExitCode,
result.StandardError);
}
return result;
}
private static async Task<string> ReadLinesAsync(
StreamReader reader,
Action<string> log,
CancellationToken cancellationToken)
{
var output = new StringBuilder();
while (await reader.ReadLineAsync(cancellationToken) is { } line)
{
output.AppendLine(line);
log(line);
}
return output.ToString();
}
}
public sealed record CommandResult(int ExitCode, string StandardOutput, string StandardError);
@@ -0,0 +1,29 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class ConfiguredSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
{
private readonly IServiceProvider services;
private readonly MeetingAssistantOptions options;
public ConfiguredSpeechRecognitionPipelineFactory(
IServiceProvider services,
IOptions<MeetingAssistantOptions> options)
{
this.services = services;
this.options = options.Value;
}
public ISpeechRecognitionPipeline Create()
{
return options.Recording.TranscriptionProvider switch
{
"funasr" => ActivatorUtilities.CreateInstance<FunAsrSpeechRecognitionPipeline>(services),
"whisper-local" => ActivatorUtilities.CreateInstance<WhisperLocalSpeechRecognitionPipeline>(services),
"azure-speech" => ActivatorUtilities.CreateInstance<AzureSpeechRecognitionPipeline>(services),
_ => throw new InvalidOperationException(
$"Unsupported speech recognition pipeline '{options.Recording.TranscriptionProvider}'.")
};
}
}
@@ -0,0 +1,190 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public interface IFunAsrBackendLifecycle
{
Task EnsureStartedAsync(CancellationToken cancellationToken);
}
public sealed class FunAsrDockerBackendLifecycle : IFunAsrBackendLifecycle, IAsyncDisposable
{
private readonly ICommandRunner commandRunner;
private readonly IFunAsrBackendReadinessProbe readinessProbe;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrDockerBackendLifecycle> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private bool containerStarted;
private bool started;
private bool disposed;
public FunAsrDockerBackendLifecycle(
ICommandRunner commandRunner,
IFunAsrBackendReadinessProbe readinessProbe,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrDockerBackendLifecycle> logger)
{
this.commandRunner = commandRunner;
this.readinessProbe = readinessProbe;
this.options = options.Value;
this.logger = logger;
}
public async Task EnsureStartedAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(disposed, this);
var backend = options.FunAsr.Backend;
if (!backend.Enabled)
{
return;
}
await gate.WaitAsync(cancellationToken);
try
{
if (started)
{
return;
}
PrepareModelsFolder(backend);
await RunRequiredAsync(backend.DockerCommand, ["pull", backend.Image], cancellationToken);
await commandRunner.RunAsync(
backend.DockerCommand,
["rm", "-f", backend.ContainerName],
cancellationToken);
await RunRequiredAsync(backend.DockerCommand, BuildDockerRunArguments(), cancellationToken);
containerStarted = true;
await readinessProbe.WaitUntilReadyAsync(
new Uri(options.FunAsr.Endpoint),
backend.StartupTimeout,
cancellationToken);
started = true;
logger.LogInformation(
"Started managed FunASR backend container {ContainerName}; endpoint {Endpoint} is ready",
backend.ContainerName,
options.FunAsr.Endpoint);
}
finally
{
gate.Release();
}
}
public async ValueTask DisposeAsync()
{
if (disposed)
{
return;
}
var backend = options.FunAsr.Backend;
if (!backend.Enabled || !backend.StopOnDispose)
{
disposed = true;
gate.Dispose();
return;
}
await gate.WaitAsync();
try
{
if (containerStarted)
{
await commandRunner.RunAsync(
backend.DockerCommand,
["stop", backend.ContainerName],
CancellationToken.None);
containerStarted = false;
started = false;
}
}
finally
{
disposed = true;
gate.Release();
gate.Dispose();
}
}
private async Task RunRequiredAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken)
{
var backend = options.FunAsr.Backend;
using var timeoutSource = backend.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(backend.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
var effectiveToken = linkedSource?.Token ?? cancellationToken;
CommandResult result;
try
{
result = await commandRunner.RunAsync(fileName, arguments, effectiveToken);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"Command '{fileName} {string.Join(' ', arguments)}' timed out after {backend.CommandTimeout}.");
}
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"Command '{fileName} {string.Join(' ', arguments)}' failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
private string[] BuildDockerRunArguments()
{
var backend = options.FunAsr.Backend;
List<string> arguments =
[
"run",
"--rm",
"-d",
];
if (backend.Privileged)
{
arguments.Add("--privileged=true");
}
arguments.AddRange(
[
"--name",
backend.ContainerName,
"-p",
$"{GetEndpointPort()}:{backend.ContainerPort}",
"-v",
$"{VaultPath.Resolve(backend.ModelsFolder)}:/workspace/models",
backend.Image,
"bash",
"-lc",
backend.ServerCommand
]);
return [.. arguments];
}
private static void PrepareModelsFolder(FunAsrBackendOptions backend)
{
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
var hotwordsPath = Path.Combine(modelsFolder, "hotwords.txt");
if (!File.Exists(hotwordsPath))
{
File.WriteAllText(hotwordsPath, "");
}
}
private int GetEndpointPort()
{
return new Uri(options.FunAsr.Endpoint).Port;
}
}
@@ -0,0 +1,182 @@
using System.Text.Json;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrMessageParser
{
private readonly FunAsrOptions options;
public FunAsrMessageParser(FunAsrOptions options)
{
this.options = options;
}
public FunAsrParsedMessage Parse(string message)
{
using var document = JsonDocument.Parse(message);
var root = document.RootElement;
var mode = GetString(root, "mode");
var isFinal = GetBoolean(root, "is_final");
var shouldEmit = options.EmitOnlinePartials || !IsOnlineMode(mode);
var segments = shouldEmit ? ReadSegments(root) : [];
return new FunAsrParsedMessage(IsTerminal: isFinal && !IsOnlineMode(mode), segments);
}
private static List<TranscriptionSegment> ReadSegments(JsonElement root)
{
var sentenceSegments = ReadSentenceSegments(root, "sentence_info");
if (sentenceSegments.Count > 0)
{
return sentenceSegments;
}
var stampSegments = ReadSentenceSegments(root, "stamp_sents");
if (stampSegments.Count > 0)
{
return stampSegments;
}
var text = GetString(root, "text").Trim();
if (string.IsNullOrWhiteSpace(text))
{
return [];
}
var (start, end) = ReadTimestampRange(root);
return [new TranscriptionSegment(start, end, ReadSpeaker(root), text)];
}
private static List<TranscriptionSegment> ReadSentenceSegments(JsonElement root, string propertyName)
{
if (!root.TryGetProperty(propertyName, out var sentences) || sentences.ValueKind != JsonValueKind.Array)
{
return [];
}
var segments = new List<TranscriptionSegment>();
foreach (var sentence in sentences.EnumerateArray())
{
var text = (GetString(sentence, "text") + GetString(sentence, "text_seg") + GetString(sentence, "punc")).Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromMilliseconds(GetDouble(sentence, "start")),
TimeSpan.FromMilliseconds(GetDouble(sentence, "end")),
ReadSpeaker(sentence),
text));
}
return segments;
}
private static (TimeSpan Start, TimeSpan End) ReadTimestampRange(JsonElement root)
{
if (!root.TryGetProperty("timestamp", out var timestamp))
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
if (timestamp.ValueKind == JsonValueKind.String)
{
var value = timestamp.GetString();
if (string.IsNullOrWhiteSpace(value))
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
using var document = JsonDocument.Parse(value);
return ReadTimestampArray(document.RootElement);
}
return ReadTimestampArray(timestamp);
}
private static (TimeSpan Start, TimeSpan End) ReadTimestampArray(JsonElement timestamp)
{
if (timestamp.ValueKind != JsonValueKind.Array)
{
return (TimeSpan.Zero, TimeSpan.Zero);
}
double? start = null;
double? end = null;
foreach (var item in timestamp.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Array)
{
continue;
}
var values = item.EnumerateArray().ToArray();
if (values.Length < 2)
{
continue;
}
start ??= ReadNumber(values[0]);
end = ReadNumber(values[1]);
}
return (
TimeSpan.FromMilliseconds(start ?? 0),
TimeSpan.FromMilliseconds(end ?? start ?? 0));
}
private static string ReadSpeaker(JsonElement element)
{
var speaker = GetString(element, "spk_name");
if (!string.IsNullOrWhiteSpace(speaker))
{
return speaker;
}
speaker = GetString(element, "speaker");
if (!string.IsNullOrWhiteSpace(speaker))
{
return speaker;
}
if (element.TryGetProperty("spk", out var spk))
{
return $"Speaker {spk}";
}
return "Unknown";
}
private static bool IsOnlineMode(string mode)
{
return mode.Equals("online", StringComparison.OrdinalIgnoreCase)
|| mode.Equals("2pass-online", StringComparison.OrdinalIgnoreCase);
}
private static string GetString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static bool GetBoolean(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind is JsonValueKind.True or JsonValueKind.False
&& property.GetBoolean();
}
private static double GetDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property) ? ReadNumber(property) : 0;
}
private static double ReadNumber(JsonElement element)
{
return element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value) ? value : 0;
}
}
public sealed record FunAsrParsedMessage(bool IsTerminal, IReadOnlyList<TranscriptionSegment> Segments);
@@ -0,0 +1,31 @@
namespace MeetingAssistant.Transcription;
public sealed class FunAsrSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
private readonly IFunAsrBackendLifecycle backendLifecycle;
private readonly FunAsrTranscriptFinalizer transcriptFinalizer;
public FunAsrSpeechRecognitionPipeline(
FunAsrStreamingTranscriptionProvider transcriptionProvider,
IFunAsrBackendLifecycle backendLifecycle,
FunAsrTranscriptFinalizer transcriptFinalizer)
: base(transcriptionProvider)
{
this.backendLifecycle = backendLifecycle;
this.transcriptFinalizer = transcriptFinalizer;
}
public override Task WaitUntilReadyAsync(CancellationToken cancellationToken)
{
return backendLifecycle.EnsureStartedAsync(cancellationToken);
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, cancellationToken);
}
}
@@ -0,0 +1,184 @@
using System.Text.Json;
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrStreamingTranscriptionProvider : IStreamingTranscriptionProvider, IAsyncDisposable
{
private readonly IFunAsrWebSocketConnectionFactory connectionFactory;
private readonly IFunAsrBackendLifecycle backendLifecycle;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrStreamingTranscriptionProvider> logger;
public FunAsrStreamingTranscriptionProvider(
IFunAsrWebSocketConnectionFactory connectionFactory,
IFunAsrBackendLifecycle backendLifecycle,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrStreamingTranscriptionProvider> logger)
{
this.connectionFactory = connectionFactory;
this.backendLifecycle = backendLifecycle;
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var segments = Channel.CreateUnbounded<TranscriptionSegment>();
var session = Task.Run(() => RunSessionAsync(audio, segments.Writer, cancellationToken), CancellationToken.None);
await foreach (var segment in segments.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
await session;
}
private async Task RunSessionAsync(
IAsyncEnumerable<AudioChunk> audio,
ChannelWriter<TranscriptionSegment> segments,
CancellationToken cancellationToken)
{
try
{
await backendLifecycle.EnsureStartedAsync(cancellationToken);
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
if (!await enumerator.MoveNextAsync())
{
segments.TryComplete();
return;
}
var firstChunk = enumerator.Current;
var endpoint = new Uri(options.FunAsr.Endpoint);
await using var connection = await connectionFactory.ConnectAsync(endpoint, cancellationToken);
var parser = new FunAsrMessageParser(options.FunAsr);
var finalReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var receiveTask = ReceiveAsync(connection, parser, segments, finalReceived, cancellationToken);
await SendStartAsync(connection, firstChunk, cancellationToken);
await connection.SendBinaryAsync(firstChunk.Pcm, cancellationToken);
while (await enumerator.MoveNextAsync())
{
await connection.SendBinaryAsync(enumerator.Current.Pcm, cancellationToken);
}
await connection.SendTextAsync("""{"is_speaking":false}""", cancellationToken);
try
{
await finalReceived.Task.WaitAsync(options.FunAsr.FinalResultTimeout, cancellationToken);
}
catch (TimeoutException)
{
logger.LogWarning(
"Timed out waiting {Timeout} for the final FunASR result from {Endpoint}",
options.FunAsr.FinalResultTimeout,
endpoint);
}
await connection.CloseOutputAsync(cancellationToken);
await receiveTask;
segments.TryComplete();
}
catch (Exception exception)
{
segments.TryComplete(exception);
}
}
public async ValueTask DisposeAsync()
{
if (backendLifecycle is IAsyncDisposable disposable)
{
await disposable.DisposeAsync();
}
}
private async Task ReceiveAsync(
IFunAsrWebSocketConnection connection,
FunAsrMessageParser parser,
ChannelWriter<TranscriptionSegment> segments,
TaskCompletionSource finalReceived,
CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var message = await connection.ReceiveTextAsync(cancellationToken);
if (message is null)
{
finalReceived.TrySetResult();
return;
}
var parsed = parser.Parse(message);
foreach (var segment in parsed.Segments)
{
await segments.WriteAsync(segment, cancellationToken);
logger.LogInformation(
"FunASR emitted segment at {Start} for {Speaker}: {Text}",
segment.Start,
segment.Speaker,
segment.Text);
}
if (parsed.IsTerminal)
{
finalReceived.TrySetResult();
return;
}
}
}
private async Task SendStartAsync(
IFunAsrWebSocketConnection connection,
AudioChunk firstChunk,
CancellationToken cancellationToken)
{
var message = new Dictionary<string, object?>
{
["mode"] = options.FunAsr.Mode,
["chunk_size"] = NormalizeChunkSize(options.FunAsr.ChunkSize),
["chunk_interval"] = options.FunAsr.ChunkInterval,
["encoder_chunk_look_back"] = options.FunAsr.EncoderChunkLookBack,
["decoder_chunk_look_back"] = options.FunAsr.DecoderChunkLookBack,
["audio_fs"] = firstChunk.SampleRate,
["wav_name"] = options.FunAsr.WavName,
["wav_format"] = "pcm",
["is_speaking"] = true,
["hotwords"] = await LoadHotwordsAsync(cancellationToken),
["itn"] = options.FunAsr.Itn
};
await connection.SendTextAsync(JsonSerializer.Serialize(message), cancellationToken);
}
private async Task<string> LoadHotwordsAsync(CancellationToken cancellationToken)
{
var path = VaultPath.Resolve(options.Vault.DictationWordsPath);
if (!File.Exists(path))
{
return "";
}
var words = await File.ReadAllLinesAsync(path, cancellationToken);
var hotwords = words
.Select(line => line.Trim().TrimStart('-', '*').Trim())
.Where(line => !string.IsNullOrWhiteSpace(line))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToDictionary(word => word, _ => 20);
return hotwords.Count == 0 ? "" : JsonSerializer.Serialize(hotwords);
}
private static int[] NormalizeChunkSize(int[] configuredChunkSize)
{
return configuredChunkSize.Length == 3 ? configuredChunkSize : [5, 10, 5];
}
}
@@ -0,0 +1,207 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class FunAsrTranscriptFinalizer
{
private const string JsonStart = "__MEETING_ASSISTANT_DIARIZATION_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_DIARIZATION_JSON_END__";
private readonly ICommandRunner commandRunner;
private readonly MeetingAssistantOptions options;
private readonly ILogger<FunAsrTranscriptFinalizer> logger;
public FunAsrTranscriptFinalizer(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<FunAsrTranscriptFinalizer> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value;
this.logger = logger;
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
CancellationToken cancellationToken)
{
var diarization = options.FunAsr.Diarization;
if (!diarization.Enabled)
{
return [];
}
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
if (!File.Exists(fullAudioPath))
{
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
}
var result = await RunDiarizationAsync(fullAudioPath, cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"FunASR diarization failed with exit code {result.ExitCode}: {result.StandardError}");
}
var json = ExtractJson(result.StandardOutput);
var segments = ParseSegments(json);
logger.LogInformation("FunASR final diarization produced {SegmentCount} sentence segments", segments.Count);
return segments;
}
private async Task<CommandResult> RunDiarizationAsync(
string fullAudioPath,
CancellationToken cancellationToken)
{
var backend = options.FunAsr.Backend;
var modelsFolder = VaultPath.Resolve(backend.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
using var timeoutSource = options.FunAsr.Diarization.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(options.FunAsr.Diarization.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
return await commandRunner.RunAsync(
backend.DockerCommand,
BuildDockerArguments(fullAudioPath, modelsFolder),
linkedSource?.Token ?? cancellationToken);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"FunASR diarization timed out after {options.FunAsr.Diarization.CommandTimeout}.");
}
}
private string[] BuildDockerArguments(string fullAudioPath, string modelsFolder)
{
var backend = options.FunAsr.Backend;
return
[
"run",
"--rm",
"-v",
$"{fullAudioPath}:/workspace/input.wav:ro",
"-v",
$"{modelsFolder}:/workspace/models",
backend.Image,
"bash",
"-lc",
BuildPythonCommand()
];
}
private string BuildPythonCommand()
{
var diarization = options.FunAsr.Diarization;
var presetSpeakerCount = diarization.PresetSpeakerCount is int value
? $", preset_spk_num={value}"
: "";
return
"cat > /tmp/meeting_assistant_diarize.py <<'PY'\n"
+ "import json\n"
+ "from funasr import AutoModel\n"
+ "model = AutoModel(\n"
+ $" model={JsonSerializer.Serialize(diarization.AsrModel)},\n"
+ $" vad_model={JsonSerializer.Serialize(diarization.VadModel)},\n"
+ $" punc_model={JsonSerializer.Serialize(diarization.PunctuationModel)},\n"
+ $" spk_model={JsonSerializer.Serialize(diarization.SpeakerModel)},\n"
+ " device='cpu',\n"
+ " disable_update=True,\n"
+ ")\n"
+ "res = model.generate(\n"
+ " input='/workspace/input.wav',\n"
+ $" batch_size_s={diarization.BatchSizeSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)},\n"
+ $" batch_size_threshold_s={diarization.BatchSizeThresholdSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture)}"
+ presetSpeakerCount
+ "\n)\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(res, ensure_ascii=False, default=str))\n"
+ $"print('{JsonEnd}')\n"
+ "PY\n"
+ "MODELSCOPE_CACHE=/workspace/models/modelscope python /tmp/meeting_assistant_diarize.py";
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidOperationException("FunASR diarization output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidOperationException("FunASR diarization output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private static IReadOnlyList<TranscriptionSegment> ParseSegments(string json)
{
using var document = JsonDocument.Parse(json);
var segments = new List<TranscriptionSegment>();
foreach (var result in document.RootElement.EnumerateArray())
{
if (!result.TryGetProperty("sentence_info", out var sentences)
|| sentences.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var sentence in sentences.EnumerateArray())
{
var text = ReadString(sentence, "text").Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromMilliseconds(ReadDouble(sentence, "start")),
TimeSpan.FromMilliseconds(ReadDouble(sentence, "end")),
$"Speaker {ReadInt(sentence, "spk")}",
text));
}
}
return segments;
}
private static string ReadString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static double ReadDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetDouble(out var value)
? value
: 0;
}
private static int ReadInt(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetInt32(out var value)
? value
: 0;
}
}
@@ -0,0 +1,84 @@
using System.Net.WebSockets;
namespace MeetingAssistant.Transcription;
public interface IFunAsrBackendReadinessProbe
{
Task WaitUntilReadyAsync(Uri endpoint, TimeSpan timeout, CancellationToken cancellationToken);
}
public sealed class FunAsrWebSocketBackendReadinessProbe : IFunAsrBackendReadinessProbe
{
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(1);
public async Task WaitUntilReadyAsync(
Uri endpoint,
TimeSpan timeout,
CancellationToken cancellationToken)
{
if (timeout <= TimeSpan.Zero)
{
return;
}
using var timeoutSource = new CancellationTokenSource(timeout);
using var linkedSource = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken,
timeoutSource.Token);
while (!linkedSource.IsCancellationRequested)
{
try
{
using var client = new ClientWebSocket();
await client.ConnectAsync(endpoint, linkedSource.Token).ConfigureAwait(false);
await CloseQuietlyAsync(client, linkedSource.Token).ConfigureAwait(false);
return;
}
catch (OperationCanceledException)
{
break;
}
catch (Exception exception) when (IsTransientReadinessFailure(exception))
{
try
{
await Task.Delay(RetryDelay, linkedSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
}
}
throw new TimeoutException(
$"Timed out waiting {timeout} for managed FunASR backend WebSocket readiness at {endpoint}.");
}
private static async Task CloseQuietlyAsync(ClientWebSocket client, CancellationToken cancellationToken)
{
if (client.State != WebSocketState.Open)
{
return;
}
try
{
await client.CloseOutputAsync(
WebSocketCloseStatus.NormalClosure,
"readiness probe complete",
cancellationToken).ConfigureAwait(false);
}
catch (WebSocketException)
{
}
}
private static bool IsTransientReadinessFailure(Exception exception)
{
return exception is WebSocketException
or HttpRequestException
or IOException;
}
}
@@ -0,0 +1,17 @@
namespace MeetingAssistant.Transcription;
public interface IFunAsrWebSocketConnectionFactory
{
Task<IFunAsrWebSocketConnection> ConnectAsync(Uri endpoint, CancellationToken cancellationToken);
}
public interface IFunAsrWebSocketConnection : IAsyncDisposable
{
Task SendTextAsync(string message, CancellationToken cancellationToken);
Task SendBinaryAsync(ReadOnlyMemory<byte> message, CancellationToken cancellationToken);
Task<string?> ReceiveTextAsync(CancellationToken cancellationToken);
Task CloseOutputAsync(CancellationToken cancellationToken);
}
@@ -0,0 +1,31 @@
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public interface ISpeechRecognitionPipeline : IAsyncDisposable
{
Task InitializeAsync(CancellationToken cancellationToken);
Task WaitUntilReadyAsync(CancellationToken cancellationToken);
ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken);
Task CompleteAsync(CancellationToken cancellationToken);
IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(CancellationToken cancellationToken);
Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
string audioPath,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken);
}
public interface ISpeechRecognitionPipelineFactory
{
ISpeechRecognitionPipeline Create();
}
public sealed record SpeechRecognitionPipelineOptions(int? NumSpeakers = null)
{
public static SpeechRecognitionPipelineOptions Default { get; } = new();
}
@@ -0,0 +1,10 @@
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public interface IStreamingTranscriptionProvider
{
IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
CancellationToken cancellationToken);
}
@@ -0,0 +1,420 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class PyannoteTranscriptFinalizer
{
private const string JsonStart = "__MEETING_ASSISTANT_PYANNOTE_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_PYANNOTE_JSON_END__";
private readonly ICommandRunner commandRunner;
private readonly MeetingAssistantOptions options;
private readonly ILogger<PyannoteTranscriptFinalizer> logger;
public PyannoteTranscriptFinalizer(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<PyannoteTranscriptFinalizer> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value;
this.logger = logger;
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
return await FinalizeAsync(
audioPath,
liveSegments,
options.WhisperLocal.Diarization,
pipelineOptions,
cancellationToken);
}
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
if (!diarization.Enabled || liveSegments.Count == 0)
{
return [];
}
var fullAudioPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(audioPath));
if (!File.Exists(fullAudioPath))
{
throw new FileNotFoundException($"Recorded audio was not found at '{fullAudioPath}'.", fullAudioPath);
}
var token = ResolveToken(diarization);
if (string.IsNullOrWhiteSpace(token))
{
logger.LogWarning(
"Pyannote diarization is enabled but no Hugging Face token is configured directly or via {TokenEnv}",
diarization.TokenEnv);
return [];
}
var result = await RunDiarizationAsync(fullAudioPath, token, diarization, pipelineOptions, cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"pyannote diarization failed with exit code {result.ExitCode}: {result.StandardError}");
}
var turns = ParseTurns(ExtractJson(result.StandardOutput));
var segments = diarization.AlignmentMode == PyannoteAlignmentMode.PyannoteTurns
? BuildTurnSegments(liveSegments, turns)
: AssignSpeakers(liveSegments, turns);
logger.LogInformation(
"pyannote diarization produced {TurnCount} speaker turns and built {SegmentCount} transcript segments using {AlignmentMode}",
turns.Count,
segments.Count,
diarization.AlignmentMode);
return segments;
}
private async Task<CommandResult> RunDiarizationAsync(
string fullAudioPath,
string token,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
var modelsFolder = VaultPath.Resolve(diarization.ModelsFolder);
Directory.CreateDirectory(modelsFolder);
using var timeoutSource = diarization.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(diarization.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
if (diarization.BuildImage)
{
await EnsureDockerImageAsync(diarization, modelsFolder, linkedSource?.Token ?? cancellationToken);
}
return await commandRunner.RunAsync(
diarization.DockerCommand,
BuildDockerArguments(diarization, fullAudioPath, modelsFolder, pipelineOptions),
linkedSource?.Token ?? cancellationToken,
new Dictionary<string, string> { ["HF_TOKEN"] = token });
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"pyannote diarization timed out after {diarization.CommandTimeout}.");
}
}
private async Task EnsureDockerImageAsync(
PyannoteDiarizationOptions diarization,
string modelsFolder,
CancellationToken cancellationToken)
{
var inspectResult = await commandRunner.RunAsync(
diarization.DockerCommand,
["image", "inspect", "--format", "{{.Id}}", diarization.Image],
cancellationToken);
if (inspectResult.ExitCode == 0)
{
return;
}
var dockerfilePath = Path.Combine(modelsFolder, "docker", $"{SanitizeFileName(diarization.Image)}.Dockerfile");
Directory.CreateDirectory(Path.GetDirectoryName(dockerfilePath)!);
await File.WriteAllTextAsync(dockerfilePath, BuildDockerfile(diarization), cancellationToken);
var result = await commandRunner.RunAsync(
diarization.DockerCommand,
["build", "-t", diarization.Image, "-f", dockerfilePath, Path.GetDirectoryName(dockerfilePath)!],
cancellationToken);
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"pyannote Docker image build failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
{
return
$"FROM {diarization.BaseImage}\n"
+ "ENV DEBIAN_FRONTEND=noninteractive\n"
+ "RUN apt-get update \\\n"
+ " && apt-get install -y --no-install-recommends ffmpeg libsndfile1 \\\n"
+ " && rm -rf /var/lib/apt/lists/*\n"
+ "RUN python -m pip install --upgrade pip \\\n"
+ " && python -m pip install pyannote.audio soundfile\n";
}
private string[] BuildDockerArguments(
PyannoteDiarizationOptions diarization,
string fullAudioPath,
string modelsFolder,
SpeechRecognitionPipelineOptions pipelineOptions)
{
return
[
"run",
"--rm",
"-e",
"HF_TOKEN",
"-v",
$"{fullAudioPath}:/workspace/input.wav:ro",
"-v",
$"{modelsFolder}:/workspace/cache",
"-e",
"HF_HOME=/workspace/cache/huggingface",
"-e",
"XDG_CACHE_HOME=/workspace/cache",
"-e",
"PIP_CACHE_DIR=/workspace/cache/pip",
"-e",
"TORCH_HOME=/workspace/cache/torch",
diarization.Image,
"sh",
"-lc",
BuildPythonCommand(diarization, pipelineOptions)
];
}
private static string BuildPythonCommand(
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions)
{
var model = diarization.Model;
var annotationProperty = diarization.AnnotationSource == PyannoteAnnotationSource.ExclusiveSpeakerDiarization
? "exclusive_speaker_diarization"
: "speaker_diarization";
return
"cat > /tmp/meeting_assistant_pyannote.py <<'PY'\n"
+ "import json\n"
+ "import os\n"
+ "import torch\n"
+ "from pyannote.audio import Pipeline\n"
+ $"pipeline = Pipeline.from_pretrained({JsonSerializer.Serialize(model)}, token=os.environ.get('HF_TOKEN'))\n"
+ "pipeline.to(torch.device('cpu'))\n"
+ BuildPipelineInvocation(pipelineOptions)
+ $"diarization = getattr(result, {JsonSerializer.Serialize(annotationProperty)}, result)\n"
+ "turns = []\n"
+ "for turn, _, speaker in diarization.itertracks(yield_label=True):\n"
+ " turns.append({'start': float(turn.start), 'end': float(turn.end), 'speaker': str(speaker)})\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(turns, ensure_ascii=False))\n"
+ $"print('{JsonEnd}')\n"
+ "PY\n"
+ "python /tmp/meeting_assistant_pyannote.py";
}
private static string BuildPipelineInvocation(SpeechRecognitionPipelineOptions pipelineOptions)
{
return pipelineOptions.NumSpeakers is > 0
? $"result = pipeline('/workspace/input.wav', num_speakers={pipelineOptions.NumSpeakers.Value})\n"
: "result = pipeline('/workspace/input.wav')\n";
}
private static IReadOnlyList<TranscriptionSegment> BuildTurnSegments(
IReadOnlyList<TranscriptionSegment> liveSegments,
IReadOnlyList<SpeakerTurn> turns)
{
if (turns.Count == 0)
{
return [];
}
var segments = new List<TranscriptionSegment>();
foreach (var turn in turns)
{
var text = BuildTurnText(liveSegments, turn);
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
segments.Add(new TranscriptionSegment(
TimeSpan.FromSeconds(turn.Start),
TimeSpan.FromSeconds(turn.End),
turn.Speaker,
text));
}
return segments.Count == 0
? AssignSpeakers(liveSegments, turns)
: segments;
}
private static string BuildTurnText(IReadOnlyList<TranscriptionSegment> liveSegments, SpeakerTurn turn)
{
var parts = new List<string>();
foreach (var segment in liveSegments)
{
var segmentStart = segment.Start.TotalSeconds;
var segmentEnd = segment.End > segment.Start
? segment.End.TotalSeconds
: segmentStart;
var overlapStart = Math.Max(segmentStart, turn.Start);
var overlapEnd = Math.Min(segmentEnd, turn.End);
if (overlapEnd <= overlapStart)
{
continue;
}
var part = SliceTextByTime(segment.Text, segmentStart, segmentEnd, overlapStart, overlapEnd);
if (!string.IsNullOrWhiteSpace(part))
{
parts.Add(part);
}
}
return string.Join(' ', parts);
}
private static string SliceTextByTime(
string text,
double segmentStart,
double segmentEnd,
double overlapStart,
double overlapEnd)
{
var words = text
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (words.Length == 0)
{
return "";
}
var duration = Math.Max(0.001, segmentEnd - segmentStart);
var startFraction = Math.Clamp((overlapStart - segmentStart) / duration, 0, 1);
var endFraction = Math.Clamp((overlapEnd - segmentStart) / duration, 0, 1);
var startIndex = Math.Clamp((int)Math.Floor(startFraction * words.Length), 0, words.Length - 1);
var endIndex = Math.Clamp((int)Math.Ceiling(endFraction * words.Length), startIndex + 1, words.Length);
return string.Join(' ', words[startIndex..endIndex]);
}
private static IReadOnlyList<TranscriptionSegment> AssignSpeakers(
IReadOnlyList<TranscriptionSegment> liveSegments,
IReadOnlyList<SpeakerTurn> turns)
{
if (turns.Count == 0)
{
return [];
}
return liveSegments
.Select(segment =>
{
var speaker = FindBestSpeaker(segment, turns) ?? segment.Speaker;
return segment with { Speaker = speaker };
})
.ToList();
}
private static string? FindBestSpeaker(TranscriptionSegment segment, IReadOnlyList<SpeakerTurn> turns)
{
var segmentStart = segment.Start.TotalSeconds;
var segmentEnd = segment.End > segment.Start
? segment.End.TotalSeconds
: segmentStart;
var bestOverlap = 0d;
string? bestSpeaker = null;
foreach (var turn in turns)
{
var overlap = Math.Min(segmentEnd, turn.End) - Math.Max(segmentStart, turn.Start);
if (overlap > bestOverlap)
{
bestOverlap = overlap;
bestSpeaker = turn.Speaker;
}
}
return bestSpeaker;
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidOperationException("pyannote output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidOperationException("pyannote output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private static IReadOnlyList<SpeakerTurn> ParseTurns(string json)
{
using var document = JsonDocument.Parse(json);
var turns = new List<SpeakerTurn>();
foreach (var item in document.RootElement.EnumerateArray())
{
var speaker = ReadString(item, "speaker");
if (string.IsNullOrWhiteSpace(speaker))
{
continue;
}
turns.Add(new SpeakerTurn(
ReadDouble(item, "start"),
ReadDouble(item, "end"),
speaker));
}
return turns;
}
private static string ResolveToken(PyannoteDiarizationOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Token))
{
return options.Token;
}
return string.IsNullOrWhiteSpace(options.TokenEnv)
? ""
: Environment.GetEnvironmentVariable(options.TokenEnv) ?? "";
}
private static string ReadString(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.String
? property.GetString() ?? ""
: "";
}
private static double ReadDouble(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var property)
&& property.ValueKind == JsonValueKind.Number
&& property.TryGetDouble(out var value)
? value
: 0;
}
private static string SanitizeFileName(string value)
{
var invalid = Path.GetInvalidFileNameChars();
return string.Concat(value.Select(character => invalid.Contains(character) ? '_' : character));
}
private sealed record SpeakerTurn(double Start, double End, string Speaker);
}
@@ -0,0 +1,77 @@
namespace MeetingAssistant.Transcription;
public sealed class SpeechRecognitionPipelineHostedService : IHostedService
{
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
private readonly ILogger<SpeechRecognitionPipelineHostedService> logger;
private CancellationTokenSource? startupCancellation;
private Task? startupTask;
private ISpeechRecognitionPipeline? startupPipeline;
public SpeechRecognitionPipelineHostedService(
ISpeechRecognitionPipelineFactory pipelineFactory,
ILogger<SpeechRecognitionPipelineHostedService> logger)
{
this.pipelineFactory = pipelineFactory;
this.logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
startupCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
startupPipeline = pipelineFactory.Create();
startupTask = Task.Run(
() => WarmUpPipelineAsync(startupPipeline, startupCancellation.Token),
CancellationToken.None);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken cancellationToken)
{
var cancellation = startupCancellation;
var task = startupTask;
var pipeline = startupPipeline;
if (cancellation is null || task is null || pipeline is null)
{
return;
}
startupCancellation = null;
startupTask = null;
startupPipeline = null;
await cancellation.CancelAsync();
try
{
await task.WaitAsync(cancellationToken);
}
catch (OperationCanceledException)
{
}
finally
{
await pipeline.DisposeAsync();
cancellation.Dispose();
}
}
private async Task WarmUpPipelineAsync(
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
try
{
await pipeline.InitializeAsync(cancellationToken);
await pipeline.WaitUntilReadyAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Speech recognition pipeline warm-up was cancelled during application shutdown");
}
catch (Exception exception)
{
logger.LogError(
exception,
"Speech recognition pipeline warm-up failed; recording can still start and recognition will retry when audio is processed");
}
}
}
@@ -0,0 +1,114 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Transcription;
public abstract class StreamingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
{
private readonly IStreamingTranscriptionProvider transcriptionProvider;
private readonly Channel<AudioChunk> audio = Channel.CreateUnbounded<AudioChunk>();
private readonly Channel<TranscriptionSegment> liveTranscript = Channel.CreateUnbounded<TranscriptionSegment>();
private readonly List<TranscriptionSegment> liveSegments = [];
private Task? transcriptionTask;
private bool initialized;
protected StreamingSpeechRecognitionPipeline(IStreamingTranscriptionProvider transcriptionProvider)
{
this.transcriptionProvider = transcriptionProvider;
}
public virtual Task InitializeAsync(CancellationToken cancellationToken)
{
if (initialized)
{
return Task.CompletedTask;
}
initialized = true;
transcriptionTask = Task.Run(() => TranscribeAsync(cancellationToken), CancellationToken.None);
return Task.CompletedTask;
}
public virtual Task WaitUntilReadyAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
EnsureInitialized();
return audio.Writer.WriteAsync(chunk, cancellationToken);
}
public async Task CompleteAsync(CancellationToken cancellationToken)
{
EnsureInitialized();
audio.Writer.TryComplete();
if (transcriptionTask is not null)
{
await transcriptionTask.WaitAsync(cancellationToken);
}
}
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
EnsureInitialized();
await foreach (var segment in liveTranscript.Reader.ReadAllAsync(cancellationToken))
{
yield return segment;
}
}
public async Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
string audioPath,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
EnsureInitialized();
await CompleteAsync(cancellationToken);
var finalSegments = await BuildFinishedTranscriptAsync(audioPath, liveSegments, options, cancellationToken);
return finalSegments.Count == 0 ? liveSegments : finalSegments;
}
public virtual ValueTask DisposeAsync()
{
audio.Writer.TryComplete();
liveTranscript.Writer.TryComplete();
return ValueTask.CompletedTask;
}
protected abstract Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken);
private async Task TranscribeAsync(CancellationToken cancellationToken)
{
try
{
await foreach (var segment in transcriptionProvider.TranscribeAsync(
audio.Reader.ReadAllAsync(cancellationToken),
cancellationToken))
{
liveSegments.Add(segment);
await liveTranscript.Writer.WriteAsync(segment, cancellationToken);
}
liveTranscript.Writer.TryComplete();
}
catch (Exception exception)
{
liveTranscript.Writer.TryComplete(exception);
}
}
private void EnsureInitialized()
{
if (!initialized)
{
throw new InvalidOperationException("Speech recognition pipeline must be initialized before use.");
}
}
}
@@ -0,0 +1,3 @@
namespace MeetingAssistant.Transcription;
public sealed record TranscriptionSegment(TimeSpan Start, TimeSpan End, string Speaker, string Text);
@@ -0,0 +1,23 @@
namespace MeetingAssistant.Transcription;
public sealed class WhisperLocalSpeechRecognitionPipeline : StreamingSpeechRecognitionPipeline
{
private readonly PyannoteTranscriptFinalizer transcriptFinalizer;
public WhisperLocalSpeechRecognitionPipeline(
WhisperLocalStreamingTranscriptionProvider transcriptionProvider,
PyannoteTranscriptFinalizer transcriptFinalizer)
: base(transcriptionProvider)
{
this.transcriptFinalizer = transcriptFinalizer;
}
protected override Task<IReadOnlyList<TranscriptionSegment>> BuildFinishedTranscriptAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return transcriptFinalizer.FinalizeAsync(audioPath, liveSegments, options, cancellationToken);
}
}
@@ -0,0 +1,168 @@
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
using NAudio.Wave;
using Whisper.net;
namespace MeetingAssistant.Transcription;
public sealed class WhisperLocalStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
private readonly MeetingAssistantOptions options;
private readonly ILogger<WhisperLocalStreamingTranscriptionProvider> logger;
public WhisperLocalStreamingTranscriptionProvider(
IOptions<MeetingAssistantOptions> options,
ILogger<WhisperLocalStreamingTranscriptionProvider> logger)
{
this.options = options.Value;
this.logger = logger;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var modelPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(options.WhisperLocal.ModelPath));
if (!File.Exists(modelPath))
{
throw new FileNotFoundException($"Whisper model was not found at '{modelPath}'.", modelPath);
}
using var factory = WhisperFactory.FromPath(modelPath);
using var processor = factory.CreateBuilder()
.WithLanguage(options.WhisperLocal.Language)
.Build();
var window = new List<AudioChunk>();
var windowDuration = TimeSpan.Zero;
var windowTarget = TimeSpan.FromSeconds(Math.Max(1, options.WhisperLocal.WindowSeconds));
var offset = TimeSpan.Zero;
logger.LogInformation(
"Starting local Whisper streaming transcription with model {ModelPath} and {WindowSeconds}s windows",
modelPath,
windowTarget.TotalSeconds);
await foreach (var chunk in audio.WithCancellation(cancellationToken))
{
window.Add(chunk);
windowDuration += chunk.Duration;
if (windowDuration >= windowTarget)
{
var segments = 0;
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
{
segments++;
logger.LogInformation(
"Whisper emitted segment at {Start}: {Text}",
segment.Start,
segment.Text);
yield return segment;
}
logger.LogInformation(
"Processed Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
offset,
windowDuration,
segments);
offset += windowDuration;
window.Clear();
windowDuration = TimeSpan.Zero;
}
}
if (window.Count > 0)
{
var segments = 0;
await foreach (var segment in ProcessWindow(processor, window, offset, cancellationToken))
{
segments++;
logger.LogInformation(
"Whisper emitted segment at {Start}: {Text}",
segment.Start,
segment.Text);
yield return segment;
}
logger.LogInformation(
"Processed final Whisper window at offset {Offset} with duration {Duration} and {SegmentCount} segment(s)",
offset,
windowDuration,
segments);
}
}
private async IAsyncEnumerable<TranscriptionSegment> ProcessWindow(
WhisperProcessor processor,
IReadOnlyList<AudioChunk> chunks,
TimeSpan offset,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var stream = CreateWaveStream(chunks);
await foreach (var result in processor.ProcessAsync(stream, cancellationToken))
{
var text = result.Text?.Trim();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
if (IsControlMarker(text))
{
logger.LogInformation("Whisper emitted control marker {Marker}; skipping transcript output", text);
continue;
}
yield return new TranscriptionSegment(offset + result.Start, offset + result.End, "Unknown", text);
}
}
private static bool IsControlMarker(string text)
{
return text is "[BLANK_AUDIO]" or "[MUSIC]" or "[NO_SPEECH]" or "[SILENCE]";
}
private MemoryStream CreateWaveStream(IReadOnlyList<AudioChunk> chunks)
{
var first = chunks[0];
var stream = new MemoryStream();
var pcm = new MemoryStream();
foreach (var chunk in chunks)
{
if (chunk.SampleRate != first.SampleRate || chunk.Channels != first.Channels)
{
logger.LogWarning("Dropping audio chunk with mismatched format while creating Whisper window");
continue;
}
pcm.Write(chunk.Pcm, 0, chunk.Pcm.Length);
}
WriteWaveHeader(stream, first.SampleRate, first.Channels, (int)pcm.Length);
pcm.Position = 0;
pcm.CopyTo(stream);
stream.Position = 0;
return stream;
}
private static void WriteWaveHeader(Stream stream, int sampleRate, int channels, int dataLength)
{
using var writer = new BinaryWriter(stream, System.Text.Encoding.ASCII, leaveOpen: true);
writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"));
writer.Write(36 + dataLength);
writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE"));
writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt "));
writer.Write(16);
writer.Write((short)1);
writer.Write((short)channels);
writer.Write(sampleRate);
writer.Write(sampleRate * channels * sizeof(short));
writer.Write((short)(channels * sizeof(short)));
writer.Write((short)16);
writer.Write(System.Text.Encoding.ASCII.GetBytes("data"));
writer.Write(dataLength);
}
}
+17
View File
@@ -0,0 +1,17 @@
namespace MeetingAssistant;
public static class VaultPath
{
public static string Resolve(string path)
{
var expanded = Environment.ExpandEnvironmentVariables(path);
if (expanded.StartsWith("~", StringComparison.Ordinal))
{
expanded = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
expanded[1..].TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
}
return Path.GetFullPath(expanded);
}
}
@@ -1,4 +1,9 @@
{
"MeetingAssistant": {
"WhisperLocal": {
"WindowSeconds": 3
}
},
"Logging": {
"LogLevel": {
"Default": "Information",
+98
View File
@@ -1,4 +1,102 @@
{
"MeetingAssistant": {
"Hotkey": {
"Toggle": "Ctrl+Alt+M"
},
"Vault": {
"TranscriptsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Transcripts",
"MeetingNotesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Notes",
"AssistantContextFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Assistant Context",
"SummariesFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Summaries",
"ProjectsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Projects",
"DictationWordsPath": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\dictation-words.md"
},
"Recording": {
"TranscriptionProvider": "funasr",
"SampleRate": 16000,
"Channels": 1,
"StopProcessingTimeout": "00:10:00",
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
},
"FunAsr": {
"Endpoint": "ws://127.0.0.1:10095",
"Mode": "2pass",
"ChunkSize": [5, 10, 5],
"ChunkInterval": 10,
"EncoderChunkLookBack": 4,
"DecoderChunkLookBack": 1,
"Itn": true,
"EmitOnlinePartials": false,
"WavName": "meeting",
"FinalResultTimeout": "00:00:10",
"Backend": {
"Enabled": true,
"DockerCommand": "docker",
"Image": "registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.13",
"ContainerName": "meeting-assistant-funasr",
"ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\FunASR\\models",
"ContainerPort": 10095,
"Privileged": true,
"CommandTimeout": "00:15:00",
"StartupTimeout": "00:10:00",
"StopOnDispose": true,
"ServerCommand": "cd /workspace/FunASR/runtime && bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --itn-dir thuduj12/fst_itn_zh --certfile 0 --hotword /workspace/models/hotwords.txt && tail -f /dev/null"
},
"Diarization": {
"Enabled": true,
"AsrModel": "paraformer-zh",
"VadModel": "fsmn-vad",
"PunctuationModel": "ct-punc",
"SpeakerModel": "cam++",
"BatchSizeSeconds": 300,
"BatchSizeThresholdSeconds": 60,
"CommandTimeout": "00:30:00"
}
},
"WhisperLocal": {
"ModelPath": "models\\ggml-base.bin",
"Language": "auto",
"WindowSeconds": 15,
"Diarization": {
"Enabled": true,
"DockerCommand": "docker",
"BaseImage": "python:3.11-slim",
"Image": "meeting-assistant-pyannote:local",
"BuildImage": true,
"ModelsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Pyannote\\models",
"Model": "pyannote/speaker-diarization-3.1",
"AnnotationSource": "SpeakerDiarization",
"AlignmentMode": "PyannoteTurns",
"TokenEnv": "HF_TOKEN",
"CommandTimeout": "00:30:00"
}
},
"AzureSpeech": {
"Endpoint": "https://germanywestcentral.api.cognitive.microsoft.com/",
"Region": "germanywestcentral",
"Language": "en-US",
"KeyEnv": "AZURE_SPEECH_KEY",
"RecognitionStopTimeout": "00:00:10",
"DiarizeIntermediateResults": true
},
"Agent": {
"Endpoint": "https://litellm.schweigert.cloud",
"KeyEnv": "LITELLM_API_KEY",
"Model": "copilot-gpt-5.5",
"EnableThinking": true,
"ReasoningEffort": "Medium",
"ReconnectionAttempts": 2,
"ReconnectionDelay": "00:00:02",
"ContextWindowTokens": 128000,
"MaxOutputTokens": 8192,
"EnableCompaction": true,
"CompactionRemainingRatio": 0.10,
"ResponsesCompactPath": "responses/compact"
},
"Api": {
"PublicBaseUrl": "http://localhost:5090"
}
},
"Logging": {
"LogLevel": {
"Default": "Information",