Make meeting lifecycle stateful

This commit is contained in:
2026-05-27 12:55:17 +02:00
parent d607b957bb
commit e85274829a
91 changed files with 4076 additions and 479 deletions
+39 -17
View File
@@ -1,27 +1,28 @@
using System.Runtime.InteropServices;
using System.Threading;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Hotkeys;
public sealed class GlobalHotkeyService : BackgroundService
{
private const int HotkeyId = 0x4D41;
private const int HotkeyIdBase = 0x4D41;
private const int WmHotkey = 0x0312;
private const int WmQuit = 0x0012;
private readonly MeetingAssistantOptions options;
private readonly ILaunchProfileOptionsProvider launchProfiles;
private readonly MeetingRecordingCoordinator coordinator;
private readonly ILogger<GlobalHotkeyService> logger;
private readonly Dictionary<int, string> hotkeyProfiles = [];
private TaskCompletionSource? messageLoopCompletion;
private uint messageThreadId;
public GlobalHotkeyService(
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
MeetingRecordingCoordinator coordinator,
ILogger<GlobalHotkeyService> logger)
{
this.options = options.Value;
this.launchProfiles = launchProfiles;
this.coordinator = coordinator;
this.logger = logger;
}
@@ -58,31 +59,49 @@ public sealed class GlobalHotkeyService : BackgroundService
private void RunMessageLoop(CancellationToken stoppingToken)
{
var hotkey = HotkeyDefinition.Parse(options.Hotkey.Toggle);
var hotkeys = launchProfiles.GetHotkeys();
messageThreadId = GetCurrentThreadId();
using var stoppingRegistration = stoppingToken.Register(() => PostQuitToMessageThread());
if (!RegisterHotKey(IntPtr.Zero, HotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey))
for (var index = 0; index < hotkeys.Count; index++)
{
logger.LogWarning("Could not register global hotkey {Hotkey}", options.Hotkey.Toggle);
return;
}
var profileHotkey = hotkeys[index];
var hotkey = HotkeyDefinition.Parse(profileHotkey.Toggle);
var hotkeyId = HotkeyIdBase + index;
if (!RegisterHotKey(IntPtr.Zero, hotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey))
{
logger.LogWarning(
"Could not register global hotkey {Hotkey} for launch profile {LaunchProfile}",
profileHotkey.Toggle,
profileHotkey.ProfileName);
continue;
}
logger.LogInformation("Registered global hotkey {Hotkey}", options.Hotkey.Toggle);
hotkeyProfiles[hotkeyId] = profileHotkey.ProfileName;
logger.LogInformation(
"Registered global hotkey {Hotkey} for launch profile {LaunchProfile}",
profileHotkey.Toggle,
profileHotkey.ProfileName);
}
try
{
while (!stoppingToken.IsCancellationRequested && GetMessage(out var message, IntPtr.Zero, 0, 0) > 0)
{
if (message.Message == WmHotkey)
if (message.Message == WmHotkey && hotkeyProfiles.TryGetValue((int)message.WParam, out var profileName))
{
_ = Task.Run(ToggleRecordingAsync, CancellationToken.None);
_ = Task.Run(() => ToggleRecordingAsync(profileName), CancellationToken.None);
}
}
}
finally
{
UnregisterHotKey(IntPtr.Zero, HotkeyId);
foreach (var hotkeyId in hotkeyProfiles.Keys)
{
UnregisterHotKey(IntPtr.Zero, hotkeyId);
}
hotkeyProfiles.Clear();
messageThreadId = 0;
}
}
@@ -93,12 +112,15 @@ public sealed class GlobalHotkeyService : BackgroundService
return base.StopAsync(cancellationToken);
}
private async Task ToggleRecordingAsync()
private async Task ToggleRecordingAsync(string profileName)
{
try
{
var status = await coordinator.ToggleAsync(CancellationToken.None);
logger.LogInformation("Hotkey toggled recording. IsRecording={IsRecording}", status.IsRecording);
var status = await coordinator.ToggleAsync(profileName, CancellationToken.None);
logger.LogInformation(
"Hotkey toggled recording for launch profile {LaunchProfile}. IsRecording={IsRecording}",
profileName,
status.IsRecording);
}
catch (Exception exception)
{
@@ -0,0 +1,153 @@
using Microsoft.Extensions.Configuration;
namespace MeetingAssistant.LaunchProfiles;
public interface ILaunchProfileOptionsProvider
{
LaunchProfile GetRequiredProfile(string? name);
IReadOnlyList<LaunchProfileHotkey> GetHotkeys();
}
public sealed record LaunchProfile(string Name, MeetingAssistantOptions Options);
public sealed record LaunchProfileHotkey(string ProfileName, string Toggle);
public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
{
public const string DefaultProfileName = "default";
private readonly IConfiguration configuration;
public ConfigurationLaunchProfileOptionsProvider(IConfiguration configuration)
{
this.configuration = configuration;
}
public LaunchProfile GetRequiredProfile(string? name)
{
var profileName = NormalizeProfileName(name);
var options = BindDefaultOptions();
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
return new LaunchProfile(DefaultProfileName, options);
}
var profileSection = GetProfilesSection().GetSection(profileName);
if (!profileSection.Exists())
{
throw new KeyNotFoundException($"Launch profile '{profileName}' is not configured.");
}
profileSection.Bind(options);
ApplyArrayOverrides(profileSection, options);
return new LaunchProfile(profileName, options);
}
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
{
var hotkeys = GetProfileNames()
.Select(profileName =>
{
var profile = GetRequiredProfile(profileName);
return new LaunchProfileHotkey(profile.Name, profile.Options.Hotkey.Toggle);
})
.Where(hotkey => !string.IsNullOrWhiteSpace(hotkey.Toggle))
.ToList();
var duplicate = hotkeys
.GroupBy(hotkey => hotkey.Toggle.Trim(), StringComparer.OrdinalIgnoreCase)
.FirstOrDefault(group => group.Count() > 1);
if (duplicate is not null)
{
throw new InvalidOperationException(
$"Launch profile hotkey '{duplicate.Key}' is configured more than once for profiles {string.Join(", ", duplicate.Select(hotkey => hotkey.ProfileName))}.");
}
return hotkeys;
}
private IReadOnlyList<string> GetProfileNames()
{
return new[] { DefaultProfileName }
.Concat(GetProfilesSection()
.GetChildren()
.Select(section => section.Key)
.Where(key => !key.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase)))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private MeetingAssistantOptions BindDefaultOptions()
{
var options = new MeetingAssistantOptions();
GetRootSection().Bind(options);
return options;
}
private IConfigurationSection GetRootSection()
{
return configuration.GetSection("MeetingAssistant");
}
private IConfigurationSection GetProfilesSection()
{
return GetRootSection().GetSection("LaunchProfiles");
}
private static string NormalizeProfileName(string? name)
{
return string.IsNullOrWhiteSpace(name) ||
name.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase)
? DefaultProfileName
: name.Trim();
}
private static void ApplyArrayOverrides(
IConfigurationSection profileSection,
MeetingAssistantOptions options)
{
var autoDetectLanguages = ReadArrayOverride(profileSection.GetSection("AzureSpeech:AutoDetectLanguages"));
if (autoDetectLanguages is not null)
{
options.AzureSpeech.AutoDetectLanguages = autoDetectLanguages;
}
var funAsrChunkSize = ReadIntArrayOverride(profileSection.GetSection("FunAsr:ChunkSize"));
if (funAsrChunkSize is not null)
{
options.FunAsr.ChunkSize = funAsrChunkSize;
}
}
private static string[]? ReadArrayOverride(IConfigurationSection section)
{
if (!section.Exists())
{
return null;
}
return section
.GetChildren()
.OrderBy(child => int.TryParse(child.Key, out var index) ? index : int.MaxValue)
.Select(child => child.Value)
.Where(value => !string.IsNullOrWhiteSpace(value))
.Select(value => value!)
.ToArray();
}
private static int[]? ReadIntArrayOverride(IConfigurationSection section)
{
if (!section.Exists())
{
return null;
}
return section
.GetChildren()
.OrderBy(child => int.TryParse(child.Key, out var index) ? index : int.MaxValue)
.Select(child => int.TryParse(child.Value, out var value) ? value : (int?)null)
.Where(value => value.HasValue)
.Select(value => value!.Value)
.ToArray();
}
}
+1
View File
@@ -12,6 +12,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.6.2" />
<PackageReference Include="DiffPlex" Version="1.9.0" />
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.50.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.8" />
<PackageReference Include="NAudio" Version="2.3.0" />
+5 -1
View File
@@ -104,7 +104,7 @@ public sealed class AzureSpeechOptions
{
public string Endpoint { get; set; } = "";
public string Region { get; set; } = "germanywestcentral";
public string Region { get; set; } = "westeurope";
public string Language { get; set; } = "auto";
@@ -120,6 +120,8 @@ public sealed class AzureSpeechOptions
public bool DiarizeIntermediateResults { get; set; } = true;
public string? PostProcessingOption { get; set; }
public double PhraseListWeight { get; set; } = 1.5;
}
@@ -232,6 +234,8 @@ public sealed class SpeakerIdentificationOptions
public TimeSpan MergeRecentIdentityAge { get; set; } = TimeSpan.FromDays(14);
public TimeSpan MatchTimeout { get; set; } = TimeSpan.FromMinutes(3);
public AzureSpeechOptions AzureSpeech { get; set; } = new();
}
@@ -4,16 +4,23 @@ public interface IMeetingArtifactStore
{
Task CreateAssistantContextAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken);
Task UpdateAssistantContextMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken);
Task UpdateAssistantContextMeetingAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
CancellationToken cancellationToken);
Task UpdateAssistantContextStateAsync(
MeetingSessionArtifacts artifacts,
AssistantContextState state,
@@ -22,6 +29,7 @@ public interface IMeetingArtifactStore
public enum AssistantContextState
{
CollectingMetadata,
Transcribing,
SpeakerRecognition,
Summarizing,
@@ -4,5 +4,13 @@ public interface IMeetingNoteStore
{
Task<MeetingNote> SaveAsync(MeetingNote note, CancellationToken cancellationToken);
Task<MeetingNote> SaveAsync(
MeetingNote note,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return SaveAsync(note, cancellationToken);
}
Task<MeetingNote> ReadAsync(string path, CancellationToken cancellationToken);
}
@@ -17,6 +17,7 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
public async Task CreateAssistantContextAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken)
@@ -25,7 +26,8 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
var content = Render(
artifacts,
AssistantContextState.Transcribing,
meetingNote,
AssistantContextState.CollectingMetadata,
agenda,
scheduledEnd,
"# Assistant Context" + Environment.NewLine + Environment.NewLine + "## Live Context" + Environment.NewLine);
@@ -45,12 +47,13 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
: new MarkdownDocument(false, "", "## Live Context" + Environment.NewLine);
var body = existing.Body;
var existingFrontmatter = ReadFrontmatter(existing.Frontmatter);
var metadata = existingFrontmatter.ToMeetingArtifactMetadata();
var agenda = existingFrontmatter.Agenda ?? "";
var scheduledEnd = ParseDateTime(existingFrontmatter.ScheduledEnd);
await File.WriteAllTextAsync(
artifacts.AssistantContextPath,
Render(artifacts, state, agenda, scheduledEnd, body),
Render(artifacts, metadata, state, agenda, scheduledEnd, body),
cancellationToken);
logger.LogInformation(
"Updated assistant context note {AssistantContextPath} state to {State}",
@@ -60,6 +63,7 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
public async Task UpdateAssistantContextMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
string agenda,
DateTimeOffset? scheduledEnd,
CancellationToken cancellationToken)
@@ -73,29 +77,68 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
await File.WriteAllTextAsync(
artifacts.AssistantContextPath,
Render(artifacts, state, agenda, scheduledEnd, existing.Body),
Render(artifacts, meetingNote, state, agenda, scheduledEnd, existing.Body),
cancellationToken);
logger.LogInformation(
"Updated assistant context note {AssistantContextPath} calendar metadata",
artifacts.AssistantContextPath);
}
public async Task UpdateAssistantContextMeetingAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
var existing = File.Exists(artifacts.AssistantContextPath)
? MarkdownDocumentParser.SplitOptional(await File.ReadAllTextAsync(artifacts.AssistantContextPath, cancellationToken))
: new MarkdownDocument(false, "", "## Live Context" + Environment.NewLine);
var existingFrontmatter = ReadFrontmatter(existing.Frontmatter);
var state = ParseState(existingFrontmatter.State);
var agenda = existingFrontmatter.Agenda ?? "";
var scheduledEnd = ParseDateTime(existingFrontmatter.ScheduledEnd);
await File.WriteAllTextAsync(
artifacts.AssistantContextPath,
Render(artifacts, meetingNote, state, agenda, scheduledEnd, existing.Body),
cancellationToken);
logger.LogInformation(
"Updated assistant context note {AssistantContextPath} meeting metadata",
artifacts.AssistantContextPath);
}
private static string Render(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
AssistantContextState state,
string? agenda,
DateTimeOffset? scheduledEnd,
string body)
{
var frontmatter = new MeetingArtifactFrontmatter
{
Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, artifacts.AssistantContextPath, "Meeting Note"),
Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, artifacts.AssistantContextPath, "Transcript"),
Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, artifacts.AssistantContextPath, "Summary"),
State = ToYamlState(state),
Agenda = agenda ?? "",
ScheduledEnd = scheduledEnd
};
var metadata = new AssistantContextMeetingMetadata(
meetingNote.Frontmatter.Title,
meetingNote.Frontmatter.StartTime,
meetingNote.Frontmatter.EndTime);
return Render(artifacts, metadata, state, agenda, scheduledEnd, body);
}
private static string Render(
MeetingSessionArtifacts artifacts,
AssistantContextMeetingMetadata metadata,
AssistantContextState state,
string? agenda,
DateTimeOffset? scheduledEnd,
string body)
{
var frontmatter = MeetingArtifactFrontmatterRenderer.Create(
artifacts,
metadata.Title,
metadata.StartTime,
metadata.EndTime,
artifacts.AssistantContextPath,
ToYamlState(state));
frontmatter.Agenda = agenda ?? "";
frontmatter.ScheduledEnd = scheduledEnd;
return MeetingArtifactFrontmatterRenderer.Render(
frontmatter,
@@ -125,23 +168,46 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
[YamlMember(Alias = "state")]
public string? State { get; set; }
[YamlMember(Alias = "title")]
public string? Title { get; set; }
[YamlMember(Alias = "start_time")]
public string? StartTime { get; set; }
[YamlMember(Alias = "end_time")]
public string? EndTime { get; set; }
[YamlMember(Alias = "agenda")]
public string? Agenda { get; set; }
[YamlMember(Alias = "scheduled_end")]
public string? ScheduledEnd { get; set; }
public AssistantContextMeetingMetadata ToMeetingArtifactMetadata()
{
return new AssistantContextMeetingMetadata(
Title ?? "",
ParseDateTime(StartTime),
ParseDateTime(EndTime));
}
}
private sealed record AssistantContextMeetingMetadata(
string Title,
DateTimeOffset? StartTime,
DateTimeOffset? EndTime);
private static AssistantContextState ParseState(string? value)
{
return value?.Trim().ToLowerInvariant() switch
{
"transcribing" => AssistantContextState.Transcribing,
"collecting metadata" => AssistantContextState.CollectingMetadata,
"speaker recognition" => AssistantContextState.SpeakerRecognition,
"summarizing" => AssistantContextState.Summarizing,
"finished" => AssistantContextState.Finished,
"error" => AssistantContextState.Error,
_ => AssistantContextState.Transcribing
_ => AssistantContextState.CollectingMetadata
};
}
@@ -149,6 +215,7 @@ public sealed class MarkdownMeetingArtifactStore : IMeetingArtifactStore
{
return state switch
{
AssistantContextState.CollectingMetadata => "collecting metadata",
AssistantContextState.Transcribing => "transcribing",
AssistantContextState.SpeakerRecognition => "speaker recognition",
AssistantContextState.Summarizing => "summarizing",
@@ -22,12 +22,20 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
}
public async Task<MeetingNote> SaveAsync(MeetingNote note, CancellationToken cancellationToken)
{
return await SaveAsync(note, options, cancellationToken);
}
public async Task<MeetingNote> SaveAsync(
MeetingNote note,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var folder = VaultPath.Resolve(options.Vault, options.Vault.MeetingNotesFolder);
Directory.CreateDirectory(folder);
var path = string.IsNullOrWhiteSpace(note.Path)
? Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Slugify(note.Frontmatter.Title)}.md")
? Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss-fffffff}-{Slugify(note.Frontmatter.Title)}.md")
: note.Path;
var frontmatter = PrepareFrontmatter(note.Frontmatter, path);
var content = Render(frontmatter, note.UserNotes);
@@ -36,10 +36,10 @@ public static class MeetingArtifactFrontmatterRenderer
AppendScalar(builder, "title", frontmatter.Title);
AppendDateTime(builder, "start_time", frontmatter.StartTime);
AppendDateTime(builder, "end_time", frontmatter.EndTime);
AppendQuoted(builder, "meeting", frontmatter.Meeting);
AppendQuoted(builder, "transcript", frontmatter.Transcript);
AppendQuoted(builder, "assistant_context", frontmatter.AssistantContext);
AppendQuoted(builder, "summary", frontmatter.Summary);
AppendQuotedIfNotEmpty(builder, "meeting", frontmatter.Meeting);
AppendQuotedIfNotEmpty(builder, "transcript", frontmatter.Transcript);
AppendQuotedIfNotEmpty(builder, "assistant_context", frontmatter.AssistantContext);
AppendQuotedIfNotEmpty(builder, "summary", frontmatter.Summary);
if (!string.IsNullOrWhiteSpace(frontmatter.State))
{
AppendScalar(builder, "state", frontmatter.State);
@@ -73,16 +73,33 @@ public static class MeetingArtifactFrontmatterRenderer
string title,
string sourceNotePath,
string? state = null)
{
return Create(
artifacts,
title,
meetingNote.Frontmatter.StartTime,
meetingNote.Frontmatter.EndTime,
sourceNotePath,
state);
}
public static MeetingArtifactFrontmatter Create(
MeetingSessionArtifacts artifacts,
string title,
DateTimeOffset? startTime,
DateTimeOffset? endTime,
string sourceNotePath,
string? state = null)
{
return new MeetingArtifactFrontmatter
{
Title = title,
StartTime = meetingNote.Frontmatter.StartTime,
EndTime = meetingNote.Frontmatter.EndTime,
Meeting = ObsidianLink.ToWikiLink(artifacts.MeetingNotePath, sourceNotePath, "Meeting Note"),
Transcript = ObsidianLink.ToWikiLink(artifacts.TranscriptPath, sourceNotePath, "Transcript"),
AssistantContext = ObsidianLink.ToWikiLink(artifacts.AssistantContextPath, sourceNotePath, "Assistant Context"),
Summary = ObsidianLink.ToWikiLink(artifacts.SummaryPath, sourceNotePath, "Summary"),
StartTime = startTime,
EndTime = endTime,
Meeting = LinkUnlessSelf(artifacts.MeetingNotePath, sourceNotePath, "Meeting Note"),
Transcript = LinkUnlessSelf(artifacts.TranscriptPath, sourceNotePath, "Transcript"),
AssistantContext = LinkUnlessSelf(artifacts.AssistantContextPath, sourceNotePath, "Assistant Context"),
Summary = LinkUnlessSelf(artifacts.SummaryPath, sourceNotePath, "Summary"),
State = state
};
}
@@ -108,6 +125,16 @@ public static class MeetingArtifactFrontmatterRenderer
builder.AppendLine(EscapeQuoted(value));
}
private static void AppendQuotedIfNotEmpty(StringBuilder builder, string key, string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
AppendQuoted(builder, key, value);
}
private static void AppendDateTime(StringBuilder builder, string key, DateTimeOffset? value)
{
builder.Append(key);
@@ -145,4 +172,19 @@ public static class MeetingArtifactFrontmatterRenderer
: value;
}
private static string LinkUnlessSelf(string targetPath, string sourceNotePath, string label)
{
return IsSamePath(targetPath, sourceNotePath)
? ""
: ObsidianLink.ToWikiLink(targetPath, sourceNotePath, label);
}
private static bool IsSamePath(string first, string second)
{
return string.Equals(
Path.GetFullPath(first),
Path.GetFullPath(second),
StringComparison.OrdinalIgnoreCase);
}
}
@@ -0,0 +1,11 @@
namespace MeetingAssistant.MeetingNotes;
internal static class MeetingAttendeeNames
{
public static string NormalizeDisplayName(string attendee)
{
var trimmed = attendee.Trim();
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
}
}
+229 -26
View File
@@ -1,5 +1,6 @@
using MeetingAssistant;
using MeetingAssistant.Hotkeys;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Recording;
using MeetingAssistant.Speakers;
@@ -10,6 +11,7 @@ using Microsoft.Extensions.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
#if WINDOWS
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
@@ -40,6 +42,7 @@ builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpea
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
builder.Services.AddSingleton<IMeetingSummaryInstructionBuilder, MeetingSummaryInstructionBuilder>();
@@ -54,6 +57,9 @@ builder.Services.AddTransient<WhisperLocalStreamingTranscriptionProvider>();
builder.Services.AddTransient<AzureSpeechStreamingTranscriptionProvider>();
builder.Services.AddTransient<FunAsrTranscriptFinalizer>();
builder.Services.AddTransient<PyannoteTranscriptFinalizer>();
builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, AzureSpeechRecognitionPipelineBuilder>();
builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, FunAsrSpeechRecognitionPipelineBuilder>();
builder.Services.AddSingleton<ISpeechRecognitionPipelineBuilder, WhisperLocalSpeechRecognitionPipelineBuilder>();
builder.Services.AddSingleton<ISpeechRecognitionPipelineFactory, ConfiguredSpeechRecognitionPipelineFactory>();
builder.Services.AddSingleton<MeetingRecordingCoordinator>();
builder.Services.AddHostedService<TemporaryRecordingCleanupHostedService>();
@@ -72,17 +78,54 @@ app.MapGet("/health", () => Results.Ok(new
}));
app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
app.MapGet("/profiles/{launchProfile}/recording/status", (
string launchProfile,
MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
app.MapGet("/diagnostics/outlook/current-meeting", async (
IMeetingMetadataProvider metadataProvider,
IOptions<MeetingAssistantOptions> options,
DateTimeOffset? startedAt,
double? timeoutSeconds,
CancellationToken cancellationToken) =>
await DiagnoseOutlookMeetingAsync(
null,
metadataProvider,
options.Value,
null,
startedAt,
timeoutSeconds,
cancellationToken));
app.MapGet("/profiles/{launchProfile}/diagnostics/outlook/current-meeting", async (
string launchProfile,
IMeetingMetadataProvider metadataProvider,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
DateTimeOffset? startedAt,
double? timeoutSeconds,
CancellationToken cancellationToken) =>
await DiagnoseOutlookMeetingAsync(
launchProfile,
metadataProvider,
options.Value,
launchProfiles,
startedAt,
timeoutSeconds,
cancellationToken));
static async Task<IResult> DiagnoseOutlookMeetingAsync(
string? launchProfile,
IMeetingMetadataProvider metadataProvider,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
DateTimeOffset? startedAt,
double? timeoutSeconds,
CancellationToken cancellationToken)
{
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
var lookupStartedAt = startedAt ?? DateTimeOffset.Now;
var timeout = timeoutSeconds is > 0
? TimeSpan.FromSeconds(timeoutSeconds.Value)
: options.Value.Recording.MetadataLookupTimeout;
: profileOptions.Recording.MetadataLookupTimeout;
if (metadataProvider is IMeetingMetadataDiagnosticProvider diagnostics)
{
return Results.Ok(await diagnostics.DiagnoseCurrentMeetingAsync(
@@ -102,78 +145,224 @@ app.MapGet("/diagnostics/outlook/current-meeting", async (
stopwatch.ElapsedMilliseconds,
metadata,
metadata is null ? "No meeting metadata provider result was available." : null));
});
}
static MeetingAssistantOptions ResolveProfileOptions(
string? launchProfile,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles)
{
if (launchProfiles is not null)
{
return launchProfiles.GetRequiredProfile(launchProfile).Options;
}
if (string.IsNullOrWhiteSpace(launchProfile) ||
launchProfile.Equals(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
return defaultOptions;
}
throw new InvalidOperationException(
$"Launch profile '{launchProfile}' was requested, but no launch profile provider is registered.");
}
app.MapPost("/diagnostics/speaker-identities/merge", async (
SpeakerIdentityMergeDiagnosticRequest request,
ISpeakerIdentityMergeService mergeService,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await MergeSpeakerIdentitiesAsync(null, request, mergeService, options.Value, null, cancellationToken));
app.MapPost("/profiles/{launchProfile}/diagnostics/speaker-identities/merge", async (
string launchProfile,
SpeakerIdentityMergeDiagnosticRequest request,
ISpeakerIdentityMergeService mergeService,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await MergeSpeakerIdentitiesAsync(launchProfile, request, mergeService, options.Value, launchProfiles, cancellationToken));
static async Task<IResult> MergeSpeakerIdentitiesAsync(
string? launchProfile,
SpeakerIdentityMergeDiagnosticRequest request,
ISpeakerIdentityMergeService mergeService,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
CancellationToken cancellationToken)
{
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
var recentAge = request.RecentDays is > 0
? TimeSpan.FromDays(request.RecentDays.Value)
: options.Value.SpeakerIdentification.MergeRecentIdentityAge;
: profileOptions.SpeakerIdentification.MergeRecentIdentityAge;
return Results.Ok(await mergeService.MergeRecentIdentitiesAsync(recentAge, cancellationToken));
});
}
app.MapPost("/recording/toggle", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.ToggleAsync(cancellationToken)));
app.MapPost("/profiles/{launchProfile}/recording/toggle", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.ToggleAsync(launchProfile, cancellationToken)));
app.MapPost("/recording/start", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StartAsync(cancellationToken)));
app.MapPost("/profiles/{launchProfile}/recording/start", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StartAsync(launchProfile, cancellationToken)));
app.MapPost("/recording/stop", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StopAsync(cancellationToken)));
app.MapPost("/profiles/{launchProfile}/recording/stop", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StopAsync(cancellationToken)));
app.MapPost("/meetings/current/summary/run", async (
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RunCurrentSummaryAsync(
null,
coordinator,
summaryPipeline,
options.Value,
null,
cancellationToken));
app.MapPost("/profiles/{launchProfile}/meetings/current/summary/run", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RunCurrentSummaryAsync(
launchProfile,
coordinator,
summaryPipeline,
options.Value,
launchProfiles,
cancellationToken));
static async Task<IResult> RunCurrentSummaryAsync(
string? launchProfile,
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
CancellationToken cancellationToken)
{
if (coordinator.CurrentArtifacts is null)
{
return Results.Conflict(new { error = "No meeting session has been started yet." });
}
return Results.Ok(await summaryPipeline.RunAsync(coordinator.CurrentArtifacts, cancellationToken));
});
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
return Results.Ok(await summaryPipeline.RunAsync(coordinator.CurrentArtifacts, profileOptions, cancellationToken));
}
app.MapPost("/meetings/summary/retry", async (
SummaryRetryRequest request,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(request.SummaryPath))
{
return Results.BadRequest(new { error = "A summary path is required." });
}
var artifacts = await artifactResolver.ResolveBySummaryPathAsync(request.SummaryPath, cancellationToken);
if (artifacts is null)
{
return Results.NotFound(new { error = $"No meeting note links to summary '{request.SummaryPath}'." });
}
return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken));
});
await RetrySummaryAsync(
null,
request.SummaryPath,
artifactResolver,
summaryPipeline,
options.Value,
null,
cancellationToken));
app.MapPost("/profiles/{launchProfile}/meetings/summary/retry", async (
string launchProfile,
SummaryRetryRequest request,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetrySummaryAsync(
launchProfile,
request.SummaryPath,
artifactResolver,
summaryPipeline,
options.Value,
launchProfiles,
cancellationToken));
app.MapGet("/meetings/summary/retry", async (
string summaryPath,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RetrySummaryAsync(
null,
summaryPath,
artifactResolver,
summaryPipeline,
options.Value,
null,
cancellationToken));
app.MapGet("/profiles/{launchProfile}/meetings/summary/retry", async (
string launchProfile,
string summaryPath,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetrySummaryAsync(
launchProfile,
summaryPath,
artifactResolver,
summaryPipeline,
options.Value,
launchProfiles,
cancellationToken));
static async Task<IResult> RetrySummaryAsync(
string? launchProfile,
string? summaryPath,
IMeetingSummaryArtifactResolver artifactResolver,
IMeetingSummaryPipeline summaryPipeline,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(summaryPath))
{
return Results.BadRequest(new { error = "A summary path is required." });
}
var artifacts = await artifactResolver.ResolveBySummaryPathAsync(summaryPath, cancellationToken);
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
var artifacts = await artifactResolver.ResolveBySummaryPathAsync(summaryPath, profileOptions, cancellationToken);
if (artifacts is null)
{
return Results.NotFound(new { error = $"No meeting note links to summary '{summaryPath}'." });
}
return Results.Ok(await summaryPipeline.RunAsync(artifacts, cancellationToken));
});
return Results.Ok(await summaryPipeline.RunAsync(artifacts, profileOptions, cancellationToken));
}
app.MapPost("/asr/transcribe-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
await TranscribeFileAsync(null, request, diagnostics, cancellationToken));
app.MapPost("/profiles/{launchProfile}/asr/transcribe-file", async (
string launchProfile,
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
await TranscribeFileAsync(launchProfile, request, diagnostics, cancellationToken));
static async Task<IResult> TranscribeFileAsync(
string? launchProfile,
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(request.Path))
{
@@ -182,7 +371,7 @@ app.MapPost("/asr/transcribe-file", async (
try
{
return Results.Ok(await diagnostics.TranscribeWavAsync(request.Path, cancellationToken));
return Results.Ok(await diagnostics.TranscribeWavAsync(request.Path, launchProfile, cancellationToken));
}
catch (FileNotFoundException exception)
{
@@ -199,11 +388,24 @@ app.MapPost("/asr/transcribe-file", async (
statusCode: StatusCodes.Status502BadGateway,
title: "ASR backend failed while processing the WAV file.");
}
});
}
app.MapPost("/asr/diarize-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
await DiarizeFileAsync(null, request, diagnostics, cancellationToken));
app.MapPost("/profiles/{launchProfile}/asr/diarize-file", async (
string launchProfile,
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken) =>
await DiarizeFileAsync(launchProfile, request, diagnostics, cancellationToken));
static async Task<IResult> DiarizeFileAsync(
string? launchProfile,
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(request.Path))
{
@@ -221,6 +423,7 @@ app.MapPost("/asr/diarize-file", async (
return Results.Ok(await diagnostics.DiarizeWavAsync(
fullPath,
new SpeechRecognitionPipelineOptions(request.NumSpeakers),
launchProfile,
cancellationToken));
}
catch (Exception exception)
@@ -230,7 +433,7 @@ app.MapPost("/asr/diarize-file", async (
statusCode: StatusCodes.Status502BadGateway,
title: "ASR diarization backend failed while processing the WAV file.");
}
});
}
try
{
@@ -4,6 +4,13 @@ public interface IRecordedAudioStore
{
Task<IRecordedAudioSink> CreateSessionAsync(CancellationToken cancellationToken);
Task<IRecordedAudioSink> CreateSessionAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return CreateSessionAsync(cancellationToken);
}
Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken);
}
@@ -7,6 +7,13 @@ public interface ITranscriptStore
{
Task<TranscriptSession> CreateSessionAsync(CancellationToken cancellationToken);
Task<TranscriptSession> CreateSessionAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return CreateSessionAsync(cancellationToken);
}
Task AppendAsync(TranscriptSession session, TranscriptionSegment segment, CancellationToken cancellationToken);
Task ReplaceAsync(
@@ -1,5 +1,6 @@
using System.Threading.Channels;
using System.Collections.Concurrent;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Summary;
@@ -20,7 +21,9 @@ public sealed class MeetingRecordingCoordinator
private readonly IRecordedAudioStore recordedAudioStore;
private readonly IMeetingSummaryPipeline summaryPipeline;
private readonly ISpeakerIdentificationService? speakerIdentificationService;
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
private readonly IDictationWordStore dictationWordStore;
private readonly ILaunchProfileOptionsProvider? launchProfiles;
private readonly MeetingAssistantOptions options;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
@@ -42,7 +45,9 @@ public sealed class MeetingRecordingCoordinator
ILogger<MeetingRecordingCoordinator> logger,
IMeetingMetadataProvider? meetingMetadataProvider = null,
ISpeakerIdentificationService? speakerIdentificationService = null,
IDictationWordStore? dictationWordStore = null)
IDictationWordStore? dictationWordStore = null,
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
ILaunchProfileOptionsProvider? launchProfiles = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -55,6 +60,8 @@ public sealed class MeetingRecordingCoordinator
this.summaryPipeline = summaryPipeline;
this.dictationWordStore = dictationWordStore ?? new EmptyDictationWordStore();
this.speakerIdentificationService = speakerIdentificationService;
this.attendeeCanonicalizer = attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance;
this.launchProfiles = launchProfiles;
this.options = options.Value;
this.logger = logger;
}
@@ -71,13 +78,27 @@ public sealed class MeetingRecordingCoordinator
private bool IsRecording => currentRun is { IsCaptureStopping: false };
public async Task<RecordingStatus> ToggleAsync(CancellationToken cancellationToken)
{
return await ToggleAsync(null, cancellationToken);
}
public async Task<RecordingStatus> ToggleAsync(
string? launchProfileName,
CancellationToken cancellationToken)
{
return IsRecording
? await StopAsync(cancellationToken)
: await StartAsync(cancellationToken);
: await StartAsync(launchProfileName, cancellationToken);
}
public async Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
{
return await StartAsync(null, cancellationToken);
}
public async Task<RecordingStatus> StartAsync(
string? launchProfileName,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
@@ -87,11 +108,12 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
currentSession = await transcriptStore.CreateSessionAsync(cancellationToken);
var recordedAudio = await recordedAudioStore.CreateSessionAsync(cancellationToken);
var runOptions = ResolveOptions(launchProfileName);
currentSession = await transcriptStore.CreateSessionAsync(runOptions, cancellationToken);
var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken);
var startedAt = DateTimeOffset.Now;
var assistantContextPath = GetAssistantContextPath(startedAt);
var summaryPath = GetSummaryPath(startedAt);
var assistantContextPath = GetAssistantContextPath(startedAt, runOptions);
var summaryPath = GetSummaryPath(startedAt, runOptions);
currentMeetingNote = await meetingNoteStore.SaveAsync(
MeetingNoteTemplate.Create(
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
@@ -100,16 +122,14 @@ public sealed class MeetingRecordingCoordinator
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath),
runOptions,
cancellationToken);
currentArtifacts = new MeetingSessionArtifacts(
currentMeetingNote.Path,
currentSession.TranscriptPath,
assistantContextPath,
summaryPath);
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, "", null, cancellationToken);
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, currentArtifacts, currentMeetingNote.Path),
CancellationToken.None);
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, currentMeetingNote, "", null, cancellationToken);
await transcriptStore.UpdateMetadataAsync(
currentSession,
currentArtifacts,
@@ -118,18 +138,28 @@ public sealed class MeetingRecordingCoordinator
await OpenMeetingNoteAsync(currentMeetingNote.Path, cancellationToken);
var captureCancellation = new CancellationTokenSource();
var transcriptionCancellation = new CancellationTokenSource();
var pipeline = speechRecognitionPipelineFactory.Create();
var pipelineOptions = await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken);
var pipeline = speechRecognitionPipelineFactory.Create(launchProfileName);
var pipelineOptions = await BuildSpeechRecognitionPipelineOptionsAsync(
runOptions,
currentMeetingNote.Path,
cancellationToken);
var run = new RecordingRun(
captureCancellation,
transcriptionCancellation,
currentSession,
currentMeetingNote.Path,
currentArtifacts,
recordedAudio,
pipeline,
pipelineOptions,
options.SpeakerIdentification.LiveSampleBufferDuration,
options.SpeakerIdentification.MaxSnippetsPerSpeaker);
run.Task = Task.Run(() => RecordAsync(currentSession, run), CancellationToken.None);
runOptions,
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker);
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
currentRun = run;
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
CancellationToken.None);
logger.LogInformation("Meeting recording started");
return CurrentStatus;
@@ -170,7 +200,7 @@ public sealed class MeetingRecordingCoordinator
try
{
await run.Task.WaitAsync(options.Recording.StopProcessingTimeout, cancellationToken);
await run.Task.WaitAsync(run.Options.Recording.StopProcessingTimeout, cancellationToken);
}
catch (OperationCanceledException)
{
@@ -187,19 +217,17 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
private async Task RecordAsync(
TranscriptSession session,
RecordingRun run)
private async Task RecordAsync(RecordingRun run)
{
await run.Pipeline.InitializeAsync(run.PipelineOptions, run.TranscriptionCancellation);
var liveTranscriptTask = Task.Run(
() => StoreLiveTranscriptAsync(session, run, run.TranscriptionCancellation),
() => StoreLiveTranscriptAsync(run, run.TranscriptionCancellation),
CancellationToken.None);
var captureTask = Task.Run(
() => CaptureAudioAsync(run),
CancellationToken.None);
var liveIdentificationTask = Task.Run(
() => IdentifyLiveSpeakersAsync(session, run, run.TranscriptionCancellation),
() => IdentifyLiveSpeakersAsync(run, run.TranscriptionCancellation),
CancellationToken.None);
try
@@ -210,30 +238,34 @@ public sealed class MeetingRecordingCoordinator
run.CancelLiveIdentification();
await liveIdentificationTask;
await run.RecordedAudio.DisposeAsync();
var artifacts = currentArtifacts;
if (artifacts is not null && HasConfiguredFinalizer())
var artifacts = run.Artifacts;
if (HasConfiguredFinalizer(run.Options))
{
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
await TransitionMeetingAsync(
run,
AssistantContextState.SpeakerRecognition,
run.TranscriptionCancellation);
}
await RewriteTranscriptWithFinishedSegmentsAsync(session, run.Pipeline, run, run.TranscriptionCancellation);
var completedMeetingNote = await CompleteMeetingNoteAsync(run.TranscriptionCancellation);
if (artifacts is not null && completedMeetingNote is not null)
await RewriteTranscriptWithFinishedSegmentsAsync(run, run.TranscriptionCancellation);
var completedMeetingNote = await CompleteMeetingNoteAsync(
run.MeetingNotePath,
run.Options,
run.TranscriptionCancellation);
if (completedMeetingNote is not null)
{
await transcriptStore.UpdateMetadataAsync(
session,
run.Session,
artifacts,
completedMeetingNote,
run.TranscriptionCancellation);
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
artifacts,
completedMeetingNote,
run.TranscriptionCancellation);
}
if (artifacts is not null)
{
await RunSummaryAsync(artifacts, run.TranscriptionCancellation);
}
await RunSummaryAsync(run, run.TranscriptionCancellation);
}
catch (OperationCanceledException) when (run.TranscriptionCancellation.IsCancellationRequested)
{
@@ -298,7 +330,6 @@ public sealed class MeetingRecordingCoordinator
}
private async Task StoreLiveTranscriptAsync(
TranscriptSession session,
RecordingRun run,
CancellationToken cancellationToken)
{
@@ -307,22 +338,21 @@ public sealed class MeetingRecordingCoordinator
run.TryAddSpeakerSample(segment);
var relabeledSegment = run.Relabel(segment);
run.AddLiveSegment(relabeledSegment);
await transcriptStore.AppendAsync(session, relabeledSegment, cancellationToken);
await transcriptStore.AppendAsync(run.Session, relabeledSegment, cancellationToken);
}
}
private async Task IdentifyLiveSpeakersAsync(
TranscriptSession session,
RecordingRun run,
CancellationToken cancellationToken)
{
if (speakerIdentificationService is null || !options.SpeakerIdentification.Enabled)
if (speakerIdentificationService is null || !run.Options.SpeakerIdentification.Enabled)
{
return;
}
var initialDelay = options.SpeakerIdentification.InitialDelay;
var interval = options.SpeakerIdentification.Interval;
var initialDelay = run.Options.SpeakerIdentification.InitialDelay;
var interval = run.Options.SpeakerIdentification.Interval;
try
{
if (initialDelay > TimeSpan.Zero)
@@ -332,7 +362,7 @@ public sealed class MeetingRecordingCoordinator
while (!run.LiveIdentificationCancellation.IsCancellationRequested)
{
await IdentifyLiveSpeakersOnceAsync(session, run, run.LiveIdentificationCancellation);
await IdentifyLiveSpeakersOnceAsync(run, run.LiveIdentificationCancellation);
await Task.Delay(interval > TimeSpan.Zero ? interval : TimeSpan.FromMinutes(1), run.LiveIdentificationCancellation);
}
}
@@ -342,11 +372,10 @@ public sealed class MeetingRecordingCoordinator
}
private async Task IdentifyLiveSpeakersOnceAsync(
TranscriptSession session,
RecordingRun run,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
if (string.IsNullOrWhiteSpace(run.MeetingNotePath))
{
return;
}
@@ -360,7 +389,7 @@ public sealed class MeetingRecordingCoordinator
try
{
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
if (checkpoint is null)
{
@@ -368,14 +397,23 @@ public sealed class MeetingRecordingCoordinator
}
var result = await speakerIdentificationService!.IdentifyKnownSpeakersAsync(
new SpeakerIdentificationRequest("", meetingNote, segments, samples),
new SpeakerIdentificationRequest(
"",
meetingNote,
segments,
samples,
run.GetSpeakerMappingsSnapshot()),
cancellationToken);
run.MarkLiveIdentificationAttempted(checkpoint);
await AddIdentifiedSpeakersToMeetingAttendeesAsync(result.AttendeeMatches, cancellationToken);
await AddIdentifiedSpeakersToMeetingAttendeesAsync(
result.AttendeeMatches,
run.MeetingNotePath,
run.Options,
cancellationToken);
if (run.AddSpeakerMappings(result.SpeakerMappings))
{
await transcriptStore.ReplaceAsync(
session,
run.Session,
run.GetRelabeledLiveSegmentsSnapshot(),
cancellationToken);
}
@@ -392,52 +430,75 @@ public sealed class MeetingRecordingCoordinator
private async Task ApplyMeetingMetadataWhenAvailableAsync(
DateTimeOffset startedAt,
MeetingSessionArtifacts artifacts,
string meetingNotePath)
RecordingRun run)
{
var metadata = await GetMeetingMetadataAsync(
startedAt,
options.Recording.BackgroundMetadataLookupTimeout,
CancellationToken.None);
if (metadata is null)
{
return;
}
await gate.WaitAsync(CancellationToken.None);
try
{
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, CancellationToken.None);
ApplyMeetingMetadata(meetingNote, metadata);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path)
var metadata = await GetMeetingMetadataAsync(
startedAt,
run.Options.Recording.BackgroundMetadataLookupTimeout,
CancellationToken.None);
if (metadata is null)
{
currentMeetingNote = meetingNote;
return;
}
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
artifacts,
metadata.Agenda,
metadata.ScheduledEnd,
CancellationToken.None);
logger.LogInformation(
"Applied Outlook meeting metadata to {MeetingNotePath}",
meetingNotePath);
await gate.WaitAsync(CancellationToken.None);
try
{
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
await ApplyMeetingMetadataAsync(meetingNote, metadata, CancellationToken.None);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path)
{
currentMeetingNote = meetingNote;
}
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
run.Artifacts,
meetingNote,
metadata.Agenda,
metadata.ScheduledEnd,
CancellationToken.None);
logger.LogInformation(
"Applied Outlook meeting metadata to {MeetingNotePath}",
run.MeetingNotePath);
}
finally
{
gate.Release();
}
}
catch (Exception exception)
{
logger.LogWarning(
exception,
"Could not apply Outlook meeting metadata to {MeetingNotePath}",
meetingNotePath);
run.MeetingNotePath);
}
finally
{
gate.Release();
try
{
await TransitionMeetingAsync(
run,
AssistantContextState.Transcribing,
CancellationToken.None);
}
catch (Exception exception)
{
logger.LogWarning(
exception,
"Could not update assistant context state to transcribing for {MeetingNotePath}",
run.MeetingNotePath);
}
}
}
private static void ApplyMeetingMetadata(MeetingNote meetingNote, MeetingMetadata metadata)
private async Task ApplyMeetingMetadataAsync(
MeetingNote meetingNote,
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(metadata.Title))
{
@@ -446,7 +507,30 @@ public sealed class MeetingRecordingCoordinator
if (metadata.Attendees.Count > 0)
{
meetingNote.Frontmatter.Attendees = metadata.Attendees.ToList();
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
}
}
private async Task<List<string>> CanonicalizeAttendeesAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
try
{
return (await attendeeCanonicalizer.CanonicalizeAsync(attendees, cancellationToken)).ToList();
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not canonicalize meeting attendees; writing raw attendee names");
return attendees
.Select(NormalizeAttendeeName)
.Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
}
@@ -503,14 +587,12 @@ public sealed class MeetingRecordingCoordinator
}
private async Task RewriteTranscriptWithFinishedSegmentsAsync(
TranscriptSession session,
ISpeechRecognitionPipeline pipeline,
RecordingRun run,
CancellationToken cancellationToken)
{
var finishedSegments = await pipeline.ReadFinishedTranscriptAsync(
var finishedSegments = await run.Pipeline.ReadFinishedTranscriptAsync(
run.RecordedAudio.AudioPath,
await BuildSpeechRecognitionPipelineOptionsAsync(cancellationToken),
await BuildSpeechRecognitionPipelineOptionsAsync(run.Options, run.MeetingNotePath, cancellationToken),
cancellationToken);
if (finishedSegments.Count == 0)
{
@@ -524,28 +606,43 @@ public sealed class MeetingRecordingCoordinator
run.RecordedAudio.AudioPath,
finishedSegments,
run.GetSpeakerSamplesSnapshot(),
run.GetSpeakerMappingsSnapshot(),
run.MeetingNotePath,
run.Options,
cancellationToken);
await transcriptStore.ReplaceAsync(session, finishedSegments, cancellationToken);
await transcriptStore.ReplaceAsync(run.Session, finishedSegments, cancellationToken);
}
private async Task<IReadOnlyList<TranscriptionSegment>> IdentifySpeakersAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> finishedSegments,
IReadOnlyList<SpeakerAudioSample> samples,
IReadOnlyDictionary<string, string> knownSpeakerMappings,
string meetingNotePath,
MeetingAssistantOptions runOptions,
CancellationToken cancellationToken)
{
if (speakerIdentificationService is null || string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
if (speakerIdentificationService is null || string.IsNullOrWhiteSpace(meetingNotePath))
{
return finishedSegments;
}
try
{
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
var result = await speakerIdentificationService.ProcessFinishedTranscriptAsync(
new SpeakerIdentificationRequest(audioPath, meetingNote, finishedSegments, samples),
new SpeakerIdentificationRequest(
audioPath,
meetingNote,
finishedSegments,
samples,
knownSpeakerMappings),
cancellationToken);
await AddIdentifiedSpeakersToMeetingAttendeesAsync(
result.AttendeeMatches,
meetingNotePath,
runOptions,
cancellationToken);
await AddIdentifiedSpeakersToMeetingAttendeesAsync(result.AttendeeMatches, cancellationToken);
return result.Segments;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -561,14 +658,16 @@ public sealed class MeetingRecordingCoordinator
private async Task AddIdentifiedSpeakersToMeetingAttendeesAsync(
IReadOnlyList<SpeakerIdentityAttendeeMatch>? matches,
string meetingNotePath,
MeetingAssistantOptions runOptions,
CancellationToken cancellationToken)
{
if (matches is null || matches.Count == 0 || string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
if (matches is null || matches.Count == 0 || string.IsNullOrWhiteSpace(meetingNotePath))
{
return;
}
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
var existingNames = meetingNote.Frontmatter.Attendees
.Select(NormalizeAttendeeName)
.Where(name => !string.IsNullOrWhiteSpace(name))
@@ -603,45 +702,56 @@ public sealed class MeetingRecordingCoordinator
return;
}
currentMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
var savedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
if (string.Equals(currentMeetingNote?.Path, savedMeetingNote.Path, StringComparison.OrdinalIgnoreCase))
{
currentMeetingNote = savedMeetingNote;
}
logger.LogInformation(
"Added identified speaker(s) to meeting note attendees for {MeetingNotePath}",
currentMeetingNote.Path);
savedMeetingNote.Path);
}
private async Task<MeetingNote?> CompleteMeetingNoteAsync(CancellationToken cancellationToken)
private async Task<MeetingNote?> CompleteMeetingNoteAsync(
string meetingNotePath,
MeetingAssistantOptions runOptions,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
if (string.IsNullOrWhiteSpace(meetingNotePath))
{
return null;
}
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
meetingNote.Frontmatter.EndTime = DateTimeOffset.Now;
currentMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
return currentMeetingNote;
var savedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
if (string.Equals(currentMeetingNote?.Path, savedMeetingNote.Path, StringComparison.OrdinalIgnoreCase))
{
currentMeetingNote = savedMeetingNote;
}
return savedMeetingNote;
}
private static string NormalizeAttendeeName(string attendee)
{
var trimmed = attendee.Trim();
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private async Task RunSummaryAsync(
MeetingSessionArtifacts artifacts,
RecordingRun run,
CancellationToken cancellationToken)
{
try
{
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
await TransitionMeetingAsync(
run,
AssistantContextState.Summarizing,
cancellationToken);
var result = await summaryPipeline.RunAsync(artifacts, cancellationToken);
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
var result = await summaryPipeline.RunAsync(run.Artifacts, run.Options, cancellationToken);
await TransitionMeetingAsync(
run,
result.Succeeded ? AssistantContextState.Finished : AssistantContextState.Error,
CancellationToken.None);
}
@@ -652,47 +762,82 @@ public sealed class MeetingRecordingCoordinator
catch (Exception exception)
{
logger.LogError(exception, "Meeting summary pipeline failed unexpectedly");
await meetingArtifactStore.UpdateAssistantContextStateAsync(
artifacts,
await TransitionMeetingAsync(
run,
AssistantContextState.Error,
CancellationToken.None);
}
}
private bool HasConfiguredFinalizer()
private async Task TransitionMeetingAsync(
RecordingRun run,
AssistantContextState state,
CancellationToken cancellationToken)
{
return options.Recording.TranscriptionProvider switch
if (!run.TryTransitionTo(state))
{
"funasr" => options.FunAsr.Diarization.Enabled,
"whisper-local" => options.WhisperLocal.Diarization.Enabled,
return;
}
await meetingArtifactStore.UpdateAssistantContextStateAsync(
run.Artifacts,
state,
cancellationToken);
}
private bool HasConfiguredFinalizer(MeetingAssistantOptions runOptions)
{
return runOptions.Recording.TranscriptionProvider switch
{
"funasr" => runOptions.FunAsr.Diarization.Enabled,
"whisper-local" => runOptions.WhisperLocal.Diarization.Enabled,
_ => false
};
}
private async Task<SpeechRecognitionPipelineOptions> BuildSpeechRecognitionPipelineOptionsAsync(
MeetingAssistantOptions runOptions,
string? meetingNotePath,
CancellationToken cancellationToken)
{
var dictationWords = await dictationWordStore.ReadWordsAsync(cancellationToken);
if (string.IsNullOrWhiteSpace(currentMeetingNote?.Path))
var dictationWords = await dictationWordStore.ReadWordsAsync(runOptions, cancellationToken);
if (string.IsNullOrWhiteSpace(meetingNotePath))
{
return new SpeechRecognitionPipelineOptions(DictationWords: dictationWords);
}
var meetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
var attendeeCount = meetingNote.Frontmatter.Attendees.Count(attendee => !string.IsNullOrWhiteSpace(attendee));
return attendeeCount > 1
? new SpeechRecognitionPipelineOptions(attendeeCount, dictationWords)
: new SpeechRecognitionPipelineOptions(DictationWords: dictationWords);
}
private string GetAssistantContextPath(DateTimeOffset startedAt)
private MeetingAssistantOptions ResolveOptions(string? launchProfileName)
{
return GetMeetingArtifactPath(options.Vault, options.Vault.AssistantContextFolder, startedAt, "assistant-context");
if (launchProfiles is not null)
{
return launchProfiles.GetRequiredProfile(launchProfileName).Options;
}
if (string.IsNullOrWhiteSpace(launchProfileName) ||
launchProfileName.Equals(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
return options;
}
throw new InvalidOperationException(
$"Launch profile '{launchProfileName}' was requested, but no launch profile provider is registered.");
}
private string GetSummaryPath(DateTimeOffset startedAt)
private string GetAssistantContextPath(DateTimeOffset startedAt, MeetingAssistantOptions runOptions)
{
return GetMeetingArtifactPath(options.Vault, options.Vault.SummariesFolder, startedAt, "summary");
return GetMeetingArtifactPath(runOptions.Vault, runOptions.Vault.AssistantContextFolder, startedAt, "assistant-context");
}
private string GetSummaryPath(DateTimeOffset startedAt, MeetingAssistantOptions runOptions)
{
return GetMeetingArtifactPath(runOptions.Vault, runOptions.Vault.SummariesFolder, startedAt, "summary");
}
private static string GetMeetingArtifactPath(
@@ -702,7 +847,7 @@ public sealed class MeetingRecordingCoordinator
string artifactName)
{
var folder = VaultPath.Resolve(vault, configuredFolder);
return Path.Combine(folder, $"{startedAt:yyyyMMdd-HHmmss}-{artifactName}.md");
return Path.Combine(folder, $"{startedAt:yyyyMMdd-HHmmss-fffffff}-{artifactName}.md");
}
private async Task CompleteRunAsync(RecordingRun run)
@@ -718,6 +863,9 @@ public sealed class MeetingRecordingCoordinator
if (ReferenceEquals(currentRun, run))
{
currentRun = null;
currentSession = null;
currentMeetingNote = null;
currentArtifacts = null;
}
}
finally
@@ -730,6 +878,7 @@ public sealed class MeetingRecordingCoordinator
private sealed class RecordingRun : IDisposable
{
private int completed;
private readonly object stateGate = new();
private readonly object liveSegmentsGate = new();
private readonly object liveIdentificationGate = new();
private readonly List<TranscriptionSegment> liveSegments = [];
@@ -740,18 +889,26 @@ public sealed class MeetingRecordingCoordinator
public RecordingRun(
CancellationTokenSource captureCancellation,
CancellationTokenSource transcriptionCancellation,
TranscriptSession session,
string meetingNotePath,
MeetingSessionArtifacts artifacts,
IRecordedAudioSink recordedAudio,
ISpeechRecognitionPipeline pipeline,
SpeechRecognitionPipelineOptions pipelineOptions,
MeetingAssistantOptions options,
TimeSpan liveSampleBufferDuration,
int maxSpeakerSamples)
{
CaptureCancellationSource = captureCancellation;
TranscriptionCancellationSource = transcriptionCancellation;
LiveIdentificationCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(transcriptionCancellation.Token);
Session = session;
MeetingNotePath = meetingNotePath;
Artifacts = artifacts;
RecordedAudio = recordedAudio;
Pipeline = pipeline;
PipelineOptions = pipelineOptions;
Options = options;
speakerSampleCollector = new SpeakerAudioSampleCollector(liveSampleBufferDuration, maxSpeakerSamples);
}
@@ -763,10 +920,18 @@ public sealed class MeetingRecordingCoordinator
public IRecordedAudioSink RecordedAudio { get; }
public TranscriptSession Session { get; }
public string MeetingNotePath { get; }
public MeetingSessionArtifacts Artifacts { get; }
public ISpeechRecognitionPipeline Pipeline { get; }
public SpeechRecognitionPipelineOptions PipelineOptions { get; }
public MeetingAssistantOptions Options { get; }
public CancellationToken CaptureCancellation => CaptureCancellationSource.Token;
public CancellationToken TranscriptionCancellation => TranscriptionCancellationSource.Token;
@@ -777,6 +942,8 @@ public sealed class MeetingRecordingCoordinator
public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested;
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public void StopCapture()
{
CaptureCancellationSource.Cancel();
@@ -792,6 +959,20 @@ public sealed class MeetingRecordingCoordinator
LiveIdentificationCancellationSource.Cancel();
}
public bool TryTransitionTo(AssistantContextState state)
{
lock (stateGate)
{
if (StateRank(state) <= StateRank(ContextState))
{
return false;
}
ContextState = state;
return true;
}
}
public void AddLiveSegment(TranscriptionSegment segment)
{
lock (liveSegmentsGate)
@@ -883,6 +1064,14 @@ public sealed class MeetingRecordingCoordinator
}
}
public IReadOnlyDictionary<string, string> GetSpeakerMappingsSnapshot()
{
return speakerMappings.ToDictionary(
pair => pair.Key,
pair => pair.Value,
StringComparer.OrdinalIgnoreCase);
}
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
{
return GetSpeakerSamplesSnapshot()
@@ -917,6 +1106,20 @@ public sealed class MeetingRecordingCoordinator
LiveIdentificationCancellationSource.Dispose();
}
private static int StateRank(AssistantContextState state)
{
return state switch
{
AssistantContextState.CollectingMetadata => 0,
AssistantContextState.Transcribing => 1,
AssistantContextState.SpeakerRecognition => 2,
AssistantContextState.Summarizing => 3,
AssistantContextState.Finished => 4,
AssistantContextState.Error => 5,
_ => 5
};
}
public sealed record LiveIdentificationCheckpoint(
IReadOnlyList<string> Speakers,
string AttendeeSignature)
@@ -18,7 +18,14 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
public Task<IRecordedAudioSink> CreateSessionAsync(CancellationToken cancellationToken)
{
var folder = GetTemporaryRecordingsFolder();
return CreateSessionAsync(options, cancellationToken);
}
public Task<IRecordedAudioSink> CreateSessionAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var folder = GetTemporaryRecordingsFolder(options);
Directory.CreateDirectory(folder);
var path = Path.Combine(folder, $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-{Guid.NewGuid():N}-recording.wav");
@@ -30,7 +37,7 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
public Task DeleteStaleRecordingsAsync(CancellationToken cancellationToken)
{
var folder = GetTemporaryRecordingsFolder();
var folder = GetTemporaryRecordingsFolder(options);
if (!Directory.Exists(folder))
{
return Task.CompletedTask;
@@ -45,7 +52,7 @@ public sealed class TemporaryRecordedAudioStore : IRecordedAudioStore
return Task.CompletedTask;
}
private string GetTemporaryRecordingsFolder()
private static string GetTemporaryRecordingsFolder(MeetingAssistantOptions options)
{
return VaultPath.Resolve(options.Recording.TemporaryRecordingsFolder);
}
@@ -16,11 +16,18 @@ public sealed class VaultTranscriptStore : ITranscriptStore
}
public async Task<TranscriptSession> CreateSessionAsync(CancellationToken cancellationToken)
{
return await CreateSessionAsync(options, cancellationToken);
}
public async Task<TranscriptSession> CreateSessionAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var folder = VaultPath.Resolve(options.Vault, options.Vault.TranscriptsFolder);
Directory.CreateDirectory(folder);
var fileName = $"{DateTimeOffset.Now:yyyyMMdd-HHmmss}-transcript.md";
var fileName = $"{DateTimeOffset.Now:yyyyMMdd-HHmmss-fffffff}-transcript.md";
var path = Path.Combine(folder, fileName);
await File.WriteAllTextAsync(path, $"# Meeting Transcript{Environment.NewLine}{Environment.NewLine}", cancellationToken);
logger.LogInformation("Created meeting transcript file {TranscriptPath}", path);
@@ -52,7 +52,15 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
return null;
}
var segments = await diarizationClient.DiarizeAsync(tempPath, cancellationToken);
var segments = await DiarizeWithTimeoutAsync(tempPath, cancellationToken);
if (segments.Count == 0)
{
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} skipped because diarization returned no segments",
request.DiarizedSpeaker);
return null;
}
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} received {SegmentCount} diarized segment(s)",
request.DiarizedSpeaker,
@@ -95,6 +103,31 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
}
}
private async Task<IReadOnlyList<TranscriptionSegment>> DiarizeWithTimeoutAsync(
string wavPath,
CancellationToken cancellationToken)
{
var matchTimeout = options.MatchTimeout;
if (matchTimeout <= TimeSpan.Zero)
{
return await diarizationClient.DiarizeAsync(wavPath, cancellationToken);
}
using var timeout = new CancellationTokenSource(matchTimeout);
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token);
try
{
return await diarizationClient.DiarizeAsync(wavPath, linked.Token);
}
catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
{
logger.LogWarning(
"Speaker identity diarization timed out after {Timeout}",
matchTimeout);
return [];
}
}
private CompositeLayout WriteCompositeWav(string path, SpeakerIdentityMatchRequest request)
{
using var firstReader = OpenFirstReadableWave(request);
@@ -7,7 +7,8 @@ public sealed record SpeakerIdentificationRequest(
string AudioPath,
MeetingNote MeetingNote,
IReadOnlyList<TranscriptionSegment> Segments,
IReadOnlyList<SpeakerAudioSample>? Samples = null);
IReadOnlyList<SpeakerAudioSample>? Samples = null,
IReadOnlyDictionary<string, string>? KnownSpeakerMappings = null);
public sealed record SpeakerAudioSample(
string Speaker,
@@ -32,7 +33,7 @@ public sealed record SpeakerIdentityMatchRequest(
public sealed record SpeakerIdentityMatchCandidate(
int IdentityId,
string? CanonicalName,
int TranscriptionCount,
int ReferenceCount,
IReadOnlyList<byte[]> Snippets);
public sealed record SpeakerIdentityMatch(int IdentityId);
+79 -2
View File
@@ -1,4 +1,5 @@
using Microsoft.EntityFrameworkCore;
using System.ComponentModel.DataAnnotations.Schema;
namespace MeetingAssistant.Speakers;
@@ -8,8 +9,6 @@ public sealed class SpeakerIdentity
public string? CanonicalName { get; set; }
public int TranscriptionCount { get; set; }
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset UpdatedAt { get; set; }
@@ -20,6 +19,11 @@ public sealed class SpeakerIdentity
public List<SpeakerSnippet> Snippets { get; set; } = [];
public List<SpeakerIdentityReference> References { get; set; } = [];
[NotMapped]
public int ReferenceCount => References.Count;
public string? GetDisplayName()
{
if (!string.IsNullOrWhiteSpace(CanonicalName))
@@ -35,6 +39,65 @@ public sealed class SpeakerIdentity
}
}
public sealed class SpeakerIdentityReference
{
public int Id { get; set; }
public int SpeakerIdentityId { get; set; }
public SpeakerIdentity? SpeakerIdentity { get; set; }
public string MeetingNotePath { get; set; } = "";
public string TranscriptPath { get; set; } = "";
public DateTimeOffset CreatedAt { get; set; }
}
public static class SpeakerIdentityReferences
{
public static SpeakerIdentityReference Create(
string meetingNotePath,
string transcriptPath,
DateTimeOffset createdAt)
{
return new SpeakerIdentityReference
{
MeetingNotePath = meetingNotePath,
TranscriptPath = transcriptPath,
CreatedAt = createdAt
};
}
public static void AddIfMissing(
SpeakerIdentity identity,
SpeakerIdentityReference reference,
DateTimeOffset? createdAt = null)
{
if (IsEmpty(reference) || identity.References.Any(existing => IsSame(existing, reference)))
{
return;
}
identity.References.Add(Create(
reference.MeetingNotePath,
reference.TranscriptPath,
createdAt ?? reference.CreatedAt));
}
private static bool IsEmpty(SpeakerIdentityReference reference)
{
return string.IsNullOrWhiteSpace(reference.MeetingNotePath) &&
string.IsNullOrWhiteSpace(reference.TranscriptPath);
}
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
}
}
public sealed class SpeakerCandidateName
{
public int Id { get; set; }
@@ -85,10 +148,15 @@ public sealed class SpeakerIdentityDbContext : DbContext
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<SpeakerIdentity>(entity =>
{
entity.Property<int>("TranscriptionCount")
.HasDefaultValue(0);
entity.HasMany(identity => identity.CandidateNames)
.WithOne(candidate => candidate.SpeakerIdentity)
.HasForeignKey(candidate => candidate.SpeakerIdentityId)
@@ -103,6 +171,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
.WithOne(snippet => snippet.SpeakerIdentity)
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.References)
.WithOne(reference => reference.SpeakerIdentity)
.HasForeignKey(reference => reference.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<SpeakerCandidateName>()
@@ -112,5 +185,9 @@ public sealed class SpeakerIdentityDbContext : DbContext
modelBuilder.Entity<SpeakerAlias>()
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
.IsUnique();
modelBuilder.Entity<SpeakerIdentityReference>()
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
.IsUnique();
}
}
@@ -0,0 +1,124 @@
using Microsoft.EntityFrameworkCore;
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Speakers;
public interface ISpeakerIdentityAttendeeCanonicalizer
{
Task<IReadOnlyList<string>> CanonicalizeAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken);
}
public sealed class SpeakerIdentityAttendeeCanonicalizer : ISpeakerIdentityAttendeeCanonicalizer
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
public SpeakerIdentityAttendeeCanonicalizer(IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory)
{
this.dbContextFactory = dbContextFactory;
}
public async Task<IReadOnlyList<string>> CanonicalizeAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
var attendeeEntries = attendees
.Select(attendee => new
{
Raw = attendee.Trim(),
DisplayName = NormalizeName(attendee)
})
.Where(entry => !string.IsNullOrWhiteSpace(entry.Raw) && !string.IsNullOrWhiteSpace(entry.DisplayName))
.ToList();
if (attendeeEntries.Count == 0)
{
return [];
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await context.SpeakerIdentities
.Include(identity => identity.Aliases)
.Include(identity => identity.References)
.ToListAsync(cancellationToken);
var identitiesByAcceptedName = identities
.OrderBy(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.ThenByDescending(identity => identity.ReferenceCount)
.ThenByDescending(identity => identity.UpdatedAt)
.ThenBy(identity => identity.Id)
.SelectMany(identity => GetExactAcceptedNames(identity).Select(name => new { Name = name, Identity = identity }))
.GroupBy(entry => entry.Name, StringComparer.OrdinalIgnoreCase)
.ToDictionary(group => group.Key, group => group.First().Identity, StringComparer.OrdinalIgnoreCase);
var result = new List<string>();
var seenNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var seenIdentityIds = new HashSet<int>();
foreach (var attendee in attendeeEntries)
{
if (identitiesByAcceptedName.TryGetValue(attendee.DisplayName!, out var identity))
{
var displayName = identity.GetDisplayName();
if (string.IsNullOrWhiteSpace(displayName) || !seenIdentityIds.Add(identity.Id))
{
continue;
}
if (seenNames.Add(displayName))
{
result.Add(displayName);
}
continue;
}
if (seenNames.Add(attendee.Raw))
{
result.Add(attendee.Raw);
}
}
return result;
}
private static IEnumerable<string> GetExactAcceptedNames(SpeakerIdentity identity)
{
return new[] { identity.CanonicalName }
.Concat(identity.Aliases.Select(alias => alias.Name))
.Select(NormalizeName)
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!);
}
private static string? NormalizeName(string? name)
{
if (string.IsNullOrWhiteSpace(name))
{
return null;
}
return MeetingAttendeeNames.NormalizeDisplayName(name);
}
}
public sealed class PassthroughSpeakerIdentityAttendeeCanonicalizer : ISpeakerIdentityAttendeeCanonicalizer
{
public static PassthroughSpeakerIdentityAttendeeCanonicalizer Instance { get; } = new();
private PassthroughSpeakerIdentityAttendeeCanonicalizer()
{
}
public Task<IReadOnlyList<string>> CanonicalizeAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
var distinctAttendees = attendees
.Select(attendee => attendee.Trim())
.Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return Task.FromResult<IReadOnlyList<string>>(distinctAttendees);
}
}
@@ -47,7 +47,8 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.OrderByDescending(identity => identity.TranscriptionCount)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
@@ -107,7 +108,14 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
continue;
}
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
MergeIdentities(target, source);
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
target.References,
targetName,
sourceName,
cancellationToken);
context.SpeakerIdentities.Remove(source);
identities.Remove(source);
mergedPairs++;
@@ -138,7 +146,7 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
candidates.Select(identity => new SpeakerIdentityMatchCandidate(
identity.Id,
identity.GetDisplayName(),
identity.TranscriptionCount,
identity.ReferenceCount,
identity.Snippets
.OrderBy(snippet => snippet.CreatedAt)
.Select(snippet => snippet.WavBytes)
@@ -166,8 +174,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
AddAlias(target, candidate.Name);
}
target.TranscriptionCount += source.TranscriptionCount;
target.UpdatedAt = DateTimeOffset.UtcNow;
foreach (var reference in source.References)
{
SpeakerIdentityReferences.AddIfMissing(target, reference);
}
var retainedSnippets = target.Snippets
.Concat(source.Snippets)
.OrderBy(snippet => StableSnippetKey(snippet.WavBytes))
@@ -27,6 +27,25 @@ internal static class SpeakerIdentitySchema
ON "SpeakerAliases" ("SpeakerIdentityId", "Name");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE TABLE IF NOT EXISTS "SpeakerIdentityReferences" (
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerIdentityReferences" PRIMARY KEY AUTOINCREMENT,
"SpeakerIdentityId" INTEGER NOT NULL,
"MeetingNotePath" TEXT NOT NULL,
"TranscriptPath" TEXT NOT NULL,
"CreatedAt" TEXT NOT NULL,
CONSTRAINT "FK_SpeakerIdentityReferences_SpeakerIdentities_SpeakerIdentityId"
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
);
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerIdentityReferences_SpeakerIdentityId_MeetingNotePath_TranscriptPath"
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
""",
cancellationToken);
}
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
@@ -1,3 +1,4 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
@@ -54,9 +55,31 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
var knownSpeakerMappings = request.KnownSpeakerMappings ??
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var knownDiarizedSpeakers = knownSpeakerMappings.Keys
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var alreadyIdentifiedNames = knownSpeakerMappings.Values
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
{
alreadyIdentifiedNames.Add(speaker.Trim());
}
var matchedAcceptedNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var name in alreadyIdentifiedNames)
{
matchedAcceptedNames.Add(name);
}
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
var samplesBySpeaker = request.Samples?
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
@@ -73,6 +96,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.OrderBy(group => group.Min(segment => segment.Start)))
{
var speaker = group.Key;
if (knownDiarizedSpeakers.Contains(speaker) || alreadyIdentifiedNames.Contains(speaker))
{
continue;
}
if (speakerMappings.ContainsKey(speaker))
{
continue;
@@ -90,7 +118,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
continue;
}
var match = await FindMatchAsync(context, attendees, speaker, snippet, cancellationToken);
var match = await FindMatchAsync(
context,
attendees,
alreadyIdentifiedNames,
speaker,
snippet,
cancellationToken);
if (match is null)
{
unmatchedSpeakers.Add((speaker, snippet));
@@ -106,7 +140,19 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
if (mode == SpeakerIdentityProcessingMode.Final)
{
var previousCanonicalName = identity.CanonicalName;
AddMeetingReference(identity, meetingReference);
UpdateMatchedIdentity(identity, attendees, snippet);
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
identity.References,
speaker,
identity.CanonicalName,
cancellationToken);
}
foreach (var acceptedName in GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
@@ -117,6 +163,12 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
if (!string.IsNullOrWhiteSpace(speakerName))
{
speakerMappings[speaker] = speakerName;
alreadyIdentifiedNames.Add(speakerName);
foreach (var acceptedName in GetAcceptedNames(identity))
{
alreadyIdentifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
speakerName,
GetAcceptedNames(identity).ToList()));
@@ -125,7 +177,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
if (mode == SpeakerIdentityProcessingMode.Final)
{
await LearnUnmatchedSpeakersAsync(context, attendees, matchedAcceptedNames, unmatchedSpeakers, cancellationToken);
await LearnUnmatchedSpeakersAsync(
context,
attendees,
matchedAcceptedNames,
unmatchedSpeakers,
meetingReference,
cancellationToken);
}
await context.SaveChangesAsync(cancellationToken);
@@ -141,6 +199,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
private async Task<SpeakerIdentityMatch?> FindMatchAsync(
SpeakerIdentityDbContext context,
IReadOnlyList<string> attendees,
IReadOnlySet<string> alreadyIdentifiedNames,
string speaker,
byte[] snippet,
CancellationToken cancellationToken)
@@ -150,7 +209,8 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var identities = await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.Aliases)
.OrderByDescending(identity => identity.TranscriptionCount)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
identities = identities
@@ -161,8 +221,9 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
.Where(candidate => !MatchesAcceptedNames(candidate.Identity, alreadyIdentifiedNames))
.OrderByDescending(candidate => candidate.IsAttendee)
.ThenByDescending(candidate => candidate.Identity.TranscriptionCount)
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
.ThenBy(candidate => candidate.Identity.Id)
.Take(maxCandidates)
.Select(candidate => candidate.Identity)
@@ -176,7 +237,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
batch.Select(identity => new SpeakerIdentityMatchCandidate(
identity.Id,
identity.CanonicalName,
identity.TranscriptionCount,
identity.ReferenceCount,
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
.ToList());
var match = await matcher.MatchAsync(request, cancellationToken);
@@ -189,6 +250,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
return null;
}
private static bool MatchesAcceptedNames(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return GetAcceptedNames(identity).Any(names.Contains);
}
private static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
@@ -206,6 +274,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Aliases)
.Include(identity => identity.Snippets)
.Include(identity => identity.References)
.SingleOrDefaultAsync(identity => identity.Id == identityId, cancellationToken);
}
@@ -214,7 +283,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
IReadOnlyList<string> attendees,
byte[] snippet)
{
identity.TranscriptionCount++;
identity.UpdatedAt = DateTimeOffset.UtcNow;
if (string.IsNullOrWhiteSpace(identity.CanonicalName) && attendees.Count > 0)
@@ -265,6 +333,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
IReadOnlyList<string> attendees,
IEnumerable<string> matchedCanonicalNames,
IReadOnlyList<(string Speaker, byte[] Snippet)> unmatchedSpeakers,
SpeakerIdentityReference meetingReference,
CancellationToken cancellationToken)
{
var remainingCandidates = attendees
@@ -276,11 +345,13 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
return;
}
foreach (var (_, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
{
var now = DateTimeOffset.UtcNow;
context.SpeakerIdentities.Add(new SpeakerIdentity
var canonicalName = remainingCandidates.Count == 1 ? remainingCandidates[0] : null;
var identity = new SpeakerIdentity
{
CanonicalName = canonicalName,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = remainingCandidates
@@ -293,8 +364,24 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
WavBytes = snippet,
CreatedAt = now
}
],
References =
[
SpeakerIdentityReferences.Create(
meetingReference.MeetingNotePath,
meetingReference.TranscriptPath,
now)
]
});
};
context.SpeakerIdentities.Add(identity);
if (!string.IsNullOrWhiteSpace(canonicalName))
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
identity.References,
speaker,
canonicalName,
cancellationToken);
}
}
await Task.CompletedTask;
@@ -366,9 +453,30 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
private static string NormalizeAttendee(string attendee)
{
var trimmed = attendee.Trim();
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(
meetingNote.Path,
meetingNote.Frontmatter.Transcript,
timestamp);
}
private static void AddMeetingReference(
SpeakerIdentity identity,
SpeakerIdentityReference reference)
{
SpeakerIdentityReferences.AddIfMissing(identity, reference, DateTimeOffset.UtcNow);
}
private enum SpeakerIdentityProcessingMode
@@ -0,0 +1,59 @@
namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityTranscriptAudit
{
public static Task AppendIdentifiedAsync(
IEnumerable<SpeakerIdentityReference> references,
string speakerLabel,
string name,
CancellationToken cancellationToken)
{
return AppendAsync(
references,
$"{DateTimeOffset.Now:yyyy-MM-dd} {speakerLabel} was identified as {name}",
cancellationToken);
}
public static Task AppendMergedAsync(
IEnumerable<SpeakerIdentityReference> references,
string name1,
string name2,
CancellationToken cancellationToken)
{
return AppendAsync(
references,
$"{DateTimeOffset.Now:yyyy-MM-dd} {name1} and {name2} were merged",
cancellationToken);
}
private static async Task AppendAsync(
IEnumerable<SpeakerIdentityReference> references,
string line,
CancellationToken cancellationToken)
{
foreach (var transcriptPath in references
.Select(reference => reference.TranscriptPath)
.Where(path => !string.IsNullOrWhiteSpace(path))
.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (!File.Exists(transcriptPath))
{
continue;
}
var existing = await File.ReadAllTextAsync(transcriptPath, cancellationToken);
if (existing.Contains(line, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var separator = existing.EndsWith(Environment.NewLine, StringComparison.Ordinal)
? ""
: Environment.NewLine;
await File.AppendAllTextAsync(
transcriptPath,
$"{separator}{line}{Environment.NewLine}",
cancellationToken);
}
}
}
@@ -0,0 +1,74 @@
using MeetingAssistant.MeetingNotes;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
public sealed record BoundMeetingProject(string Name, string Path);
public sealed class BoundMeetingProjectResolver
{
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
private readonly MeetingAssistantOptions options;
public BoundMeetingProjectResolver(MeetingAssistantOptions options)
{
this.options = options;
}
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken = default)
{
if (!Directory.Exists(ProjectsRoot))
{
return [];
}
var projectNames = await ReadMeetingProjectNamesAsync(artifacts.MeetingNotePath, cancellationToken);
if (projectNames.Count == 0)
{
return [];
}
return Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static async Task<HashSet<string>> ReadMeetingProjectNamesAsync(
string meetingNotePath,
CancellationToken cancellationToken)
{
if (!File.Exists(meetingNotePath))
{
return [];
}
var content = await File.ReadAllTextAsync(meetingNotePath, cancellationToken);
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter)
{
return [];
}
var frontmatter = YamlDeserializer.Deserialize<ProjectFrontmatter>(document.Frontmatter)
?? new ProjectFrontmatter();
return (frontmatter.Projects ?? [])
.Where(project => !string.IsNullOrWhiteSpace(project))
.Select(project => project.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private sealed class ProjectFrontmatter
{
[YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
}
}
@@ -7,4 +7,12 @@ public interface IMeetingSummaryInstructionBuilder
Task<string> BuildAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken);
Task<string> BuildAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return BuildAsync(artifacts, cancellationToken);
}
}
@@ -5,6 +5,14 @@ namespace MeetingAssistant.Summary;
public interface IMeetingSummaryPipeline
{
Task<MeetingSummaryRunResult> RunAsync(MeetingSessionArtifacts artifacts, CancellationToken cancellationToken);
Task<MeetingSummaryRunResult> RunAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return RunAsync(artifacts, cancellationToken);
}
}
public sealed record MeetingSummaryRunResult(
@@ -8,6 +8,14 @@ public interface IMeetingSummaryArtifactResolver
Task<MeetingSessionArtifacts?> ResolveBySummaryPathAsync(
string summaryPath,
CancellationToken cancellationToken);
Task<MeetingSessionArtifacts?> ResolveBySummaryPathAsync(
string summaryPath,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return ResolveBySummaryPathAsync(summaryPath, cancellationToken);
}
}
public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactResolver
@@ -27,7 +35,15 @@ public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactReso
string summaryPath,
CancellationToken cancellationToken)
{
var requestedSummaryPath = ResolveRequestedSummaryPath(summaryPath);
return await ResolveBySummaryPathAsync(summaryPath, options, cancellationToken);
}
public async Task<MeetingSessionArtifacts?> ResolveBySummaryPathAsync(
string summaryPath,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var requestedSummaryPath = ResolveRequestedSummaryPath(summaryPath, options);
if (requestedSummaryPath is null)
{
return null;
@@ -58,7 +74,9 @@ public sealed class MeetingSummaryArtifactResolver : IMeetingSummaryArtifactReso
return null;
}
private string? ResolveRequestedSummaryPath(string summaryPath)
private static string? ResolveRequestedSummaryPath(
string summaryPath,
MeetingAssistantOptions options)
{
if (string.IsNullOrWhiteSpace(summaryPath))
{
@@ -10,6 +10,15 @@ public interface IMeetingSummaryFailureWriter
MeetingSessionArtifacts artifacts,
Exception exception,
CancellationToken cancellationToken);
Task<MeetingSummaryRunResult> WriteAsync(
MeetingSessionArtifacts artifacts,
Exception exception,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return WriteAsync(artifacts, exception, cancellationToken);
}
}
public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter
@@ -29,6 +38,15 @@ public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter
MeetingSessionArtifacts artifacts,
Exception exception,
CancellationToken cancellationToken)
{
return await WriteAsync(artifacts, exception, options, cancellationToken);
}
public async Task<MeetingSummaryRunResult> WriteAsync(
MeetingSessionArtifacts artifacts,
Exception exception,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!);
var meetingNote = File.Exists(artifacts.MeetingNotePath)
@@ -43,7 +61,7 @@ public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter
artifacts.SummaryPath,
MeetingArtifactFrontmatterRenderer.Render(
frontmatter,
RenderFailureMarkdown(artifacts, exception)),
RenderFailureMarkdown(artifacts, exception, options)),
cancellationToken);
return new MeetingSummaryRunResult(
@@ -53,7 +71,10 @@ public sealed class MeetingSummaryFailureWriter : IMeetingSummaryFailureWriter
Error: exception.Message);
}
private string RenderFailureMarkdown(MeetingSessionArtifacts artifacts, Exception exception)
private static string RenderFailureMarkdown(
MeetingSessionArtifacts artifacts,
Exception exception,
MeetingAssistantOptions options)
{
var builder = new StringBuilder();
builder.AppendLine("# Meeting Summary");
@@ -1,6 +1,5 @@
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
@@ -21,25 +20,31 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
Keep the output grounded in the source material and explicitly say when a section has no known items.
""";
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
private readonly MeetingAssistantOptions options;
private readonly BoundMeetingProjectResolver projectResolver;
public MeetingSummaryInstructionBuilder(IOptions<MeetingAssistantOptions> options)
{
this.options = options.Value;
projectResolver = new BoundMeetingProjectResolver(this.options);
}
public async Task<string> BuildAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken)
{
return await BuildAsync(artifacts, options, cancellationToken);
}
public async Task<string> BuildAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt)
? DefaultInitialPrompt
: options.Agent.InitialPrompt.Trim();
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, cancellationToken);
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, options, cancellationToken);
return string.IsNullOrWhiteSpace(projectInstructions)
? instructions
: instructions.TrimEnd() + "\n\n" + projectInstructions;
@@ -47,9 +52,10 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
private async Task<string> BuildProjectInstructionsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, cancellationToken);
var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, options, cancellationToken);
if (projects.Count == 0)
{
return "";
@@ -62,30 +68,16 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
private async Task<List<ProjectInstructions>> GetBoundProjectsWithInstructionsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var projectNames = await ReadMeetingProjectNamesAsync(artifacts.MeetingNotePath, cancellationToken);
if (projectNames.Count == 0)
{
return [];
}
var projectsRoot = VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
if (!Directory.Exists(projectsRoot))
{
return [];
}
var resolver = ReferenceEquals(options, this.options)
? projectResolver
: new BoundMeetingProjectResolver(options);
var projects = new List<ProjectInstructions>();
foreach (var projectDirectory in Directory.EnumerateDirectories(projectsRoot).Order(StringComparer.OrdinalIgnoreCase))
foreach (var project in await resolver.GetBoundProjectsAsync(artifacts, cancellationToken))
{
var projectName = Path.GetFileName(projectDirectory);
if (!projectNames.Contains(projectName))
{
continue;
}
var agentsPath = Path.Combine(projectDirectory, "AGENTS.md");
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
if (!File.Exists(agentsPath))
{
continue;
@@ -94,42 +86,12 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
var content = await File.ReadAllTextAsync(agentsPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(content))
{
projects.Add(new ProjectInstructions(projectName, content));
projects.Add(new ProjectInstructions(project.Name, content));
}
}
return projects;
}
private static async Task<HashSet<string>> ReadMeetingProjectNamesAsync(
string meetingNotePath,
CancellationToken cancellationToken)
{
if (!File.Exists(meetingNotePath))
{
return [];
}
var content = await File.ReadAllTextAsync(meetingNotePath, cancellationToken);
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter)
{
return [];
}
var frontmatter = YamlDeserializer.Deserialize<ProjectFrontmatter>(document.Frontmatter)
?? new ProjectFrontmatter();
return (frontmatter.Projects ?? [])
.Where(project => !string.IsNullOrWhiteSpace(project))
.Select(project => project.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private sealed record ProjectInstructions(string Name, string Instructions);
private sealed class ProjectFrontmatter
{
[YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
}
}
+36 -57
View File
@@ -15,6 +15,8 @@ public sealed class MeetingSummaryTools
private readonly MeetingSessionArtifacts artifacts;
private readonly MeetingAssistantOptions options;
private readonly IDictationWordStore? dictationWordStore;
private readonly SummaryAgentWriteAudit? writeAudit;
private readonly BoundMeetingProjectResolver projectResolver;
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
: this(artifacts, new MeetingAssistantOptions(), null)
@@ -24,11 +26,14 @@ public sealed class MeetingSummaryTools
public MeetingSummaryTools(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
IDictationWordStore? dictationWordStore = null)
IDictationWordStore? dictationWordStore = null,
SummaryAgentWriteAudit? writeAudit = null)
{
this.artifacts = artifacts;
this.options = options;
this.dictationWordStore = dictationWordStore;
this.writeAudit = writeAudit;
projectResolver = new BoundMeetingProjectResolver(options);
}
public Task<string> ReadTranscript(int? @from = null, int? to = null)
@@ -76,7 +81,16 @@ public sealed class MeetingSummaryTools
return "Refused: word must not be empty.";
}
var words = await dictationWordStore.AddWordAsync(word, CancellationToken.None);
if (writeAudit is not null)
{
IReadOnlyList<string>? capturedWords = null;
await writeAudit.CaptureFileWriteAsync(
VaultPath.Resolve(options.Vault, options.Vault.DictationWordsPath),
async () => capturedWords = await dictationWordStore.AddWordAsync(word, options, CancellationToken.None));
return $"Added '{word.Trim()}'. Dictionary now contains {capturedWords?.Count ?? 0} word(s).";
}
var words = await dictationWordStore.AddWordAsync(word, options, CancellationToken.None);
return $"Added '{word.Trim()}'. Dictionary now contains {words.Count} word(s).";
}
@@ -176,7 +190,17 @@ public sealed class MeetingSummaryTools
}
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
await WriteProjectFileContentAsync(target.Path, content, editMode);
if (writeAudit is not null)
{
await writeAudit.CaptureFileWriteAsync(
target.Path,
() => WriteProjectFileContentAsync(target.Path, content, editMode));
}
else
{
await WriteProjectFileContentAsync(target.Path, content, editMode);
}
return $"{target.Project.Name}/{ToToolPath(path)}";
}
@@ -237,7 +261,7 @@ public sealed class MeetingSummaryTools
return new MeetingNote(artifacts.MeetingNotePath, frontmatter, await ReadUserNotes());
}
private async Task<List<ProjectFolder>> GetSearchProjectsAsync(string[]? projects)
private async Task<List<BoundMeetingProject>> GetSearchProjectsAsync(string[]? projects)
{
var boundProjects = await GetBoundProjectsAsync();
if (projects is null || projects.Length == 0)
@@ -253,7 +277,7 @@ public sealed class MeetingSummaryTools
.ToList();
}
private async Task<ProjectFolder?> ResolveBoundProjectAsync(string project)
private async Task<BoundMeetingProject?> ResolveBoundProjectAsync(string project)
{
if (string.IsNullOrWhiteSpace(project))
{
@@ -297,7 +321,7 @@ public sealed class MeetingSummaryTools
}
var projectFolder = Directory.EnumerateDirectories(projectsRoot)
.Select(candidate => new ProjectFolder(Path.GetFileName(candidate), candidate))
.Select(candidate => new BoundMeetingProject(Path.GetFileName(candidate), candidate))
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
if (projectFolder is null)
{
@@ -414,56 +438,19 @@ public sealed class MeetingSummaryTools
return string.Join('\n', lines);
}
private async Task<List<ProjectFolder>> GetBoundProjectsAsync()
private Task<List<BoundMeetingProject>> GetBoundProjectsAsync()
{
var projectsRoot = GetProjectsRoot();
if (!Directory.Exists(projectsRoot))
{
return [];
}
var projectNames = await ReadMeetingProjectNamesAsync();
if (projectNames.Count == 0)
{
return [];
}
return Directory.EnumerateDirectories(projectsRoot)
.Select(path => new ProjectFolder(Path.GetFileName(path), path))
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
private async Task<HashSet<string>> ReadMeetingProjectNamesAsync()
{
if (!File.Exists(artifacts.MeetingNotePath))
{
return [];
}
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter)
{
return [];
}
var note = YamlDeserializer.Deserialize<ProjectFrontmatter>(document.Frontmatter) ?? new ProjectFrontmatter();
return (note.Projects ?? [])
.Where(project => !string.IsNullOrWhiteSpace(project))
.Select(project => project.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
return projectResolver.GetBoundProjectsAsync(artifacts);
}
private string GetProjectsRoot()
{
return VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
return projectResolver.ProjectsRoot;
}
private static async Task<string?> RunRipgrepAsync(
string projectsRoot,
IReadOnlyList<ProjectFolder> projects,
IReadOnlyList<BoundMeetingProject> projects,
string keywords)
{
try
@@ -528,7 +515,7 @@ public sealed class MeetingSummaryTools
return string.Join('\n', matches);
}
private static string SearchWithRegexFallback(IReadOnlyList<ProjectFolder> projects, string keywords, bool singleProject)
private static string SearchWithRegexFallback(IReadOnlyList<BoundMeetingProject> projects, string keywords, bool singleProject)
{
try
{
@@ -584,9 +571,7 @@ public sealed class MeetingSummaryTools
.Replace(Path.AltDirectorySeparatorChar.ToString(), "/", StringComparison.Ordinal);
}
private sealed record ProjectFolder(string Name, string Path);
private sealed record ProjectFileTarget(ProjectFolder Project, string Path);
private sealed record ProjectFileTarget(BoundMeetingProject Project, string Path);
private sealed record FileLineEditMode(
FileLineEditKind Kind,
@@ -621,12 +606,6 @@ public sealed class MeetingSummaryTools
Insert
}
private sealed class ProjectFrontmatter
{
[YamlDotNet.Serialization.YamlMember(Alias = "projects")]
public List<string>? Projects { get; set; }
}
private sealed class MeetingNoteFrontmatterYaml
{
[YamlDotNet.Serialization.YamlMember(Alias = "title")]
@@ -39,11 +39,20 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
public async Task<MeetingSummaryRunResult> RunAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken)
{
return await RunAsync(artifacts, options, cancellationToken);
}
public async Task<MeetingSummaryRunResult> RunAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var agentOptions = options.Agent;
var key = ResolveApiKey(agentOptions);
var tools = CreateTools(new MeetingSummaryTools(artifacts, options, dictationWordStore));
var instructions = await instructionBuilder.BuildAsync(artifacts, cancellationToken);
var writeAudit = new SummaryAgentWriteAudit(artifacts);
var tools = CreateTools(new MeetingSummaryTools(artifacts, options, dictationWordStore, writeAudit));
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
using var compactionSummaryClient = new LiteLlmResponsesChatClient(
new Uri(agentOptions.Endpoint),
key,
@@ -85,6 +94,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
options: new ChatClientAgentRunOptions(CreateChatOptions(agentOptions)),
cancellationToken: cancellationToken);
await writeAudit.AppendToAssistantContextAsync(cancellationToken);
return new MeetingSummaryRunResult(artifacts.SummaryPath, response.Text);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -97,7 +107,9 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
exception,
"Meeting summary generation failed for {SummaryPath}; writing failure summary",
artifacts.SummaryPath);
return await failureWriter.WriteAsync(artifacts, exception, cancellationToken);
var result = await failureWriter.WriteAsync(artifacts, exception, options, cancellationToken);
await writeAudit.AppendToAssistantContextAsync(cancellationToken);
return result;
}
}
@@ -0,0 +1,117 @@
using DiffPlex.DiffBuilder;
using DiffPlex.DiffBuilder.Model;
using MeetingAssistant.MeetingNotes;
using System.Text;
namespace MeetingAssistant.Summary;
public sealed class SummaryAgentWriteAudit
{
private readonly MeetingSessionArtifacts artifacts;
private readonly List<FileChange> changes = [];
public SummaryAgentWriteAudit(MeetingSessionArtifacts artifacts)
{
this.artifacts = artifacts;
}
public async Task CaptureFileWriteAsync(
string path,
Func<Task> writeOperation,
CancellationToken cancellationToken = default)
{
var fullPath = Path.GetFullPath(path);
var before = File.Exists(fullPath)
? await File.ReadAllTextAsync(fullPath, cancellationToken)
: "";
await writeOperation();
var after = File.Exists(fullPath)
? await File.ReadAllTextAsync(fullPath, cancellationToken)
: "";
Capture(fullPath, before, after, cancellationToken);
}
public void Capture(
string path,
string before,
string after,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var fullPath = Path.GetFullPath(path);
if (IsOwnedArtifact(fullPath) || string.Equals(before, after, StringComparison.Ordinal))
{
return;
}
var diffLines = CreateDiffLines(before, after);
if (diffLines.Count == 0)
{
return;
}
changes.Add(new FileChange(fullPath, diffLines));
}
public async Task AppendToAssistantContextAsync(CancellationToken cancellationToken)
{
if (changes.Count == 0)
{
return;
}
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
var existing = File.Exists(artifacts.AssistantContextPath)
? await File.ReadAllTextAsync(artifacts.AssistantContextPath, cancellationToken)
: "";
var builder = new StringBuilder();
if (!string.IsNullOrEmpty(existing) && !existing.EndsWith(Environment.NewLine, StringComparison.Ordinal))
{
builder.AppendLine();
}
builder.AppendLine();
builder.AppendLine("## Summary Agent External File Changes");
builder.AppendLine();
foreach (var change in changes)
{
builder.AppendLine($"### {change.Path}");
builder.AppendLine();
builder.AppendLine("```diff");
foreach (var line in change.DiffLines)
{
builder.AppendLine(line);
}
builder.AppendLine("```");
builder.AppendLine();
}
await File.AppendAllTextAsync(artifacts.AssistantContextPath, builder.ToString(), cancellationToken);
}
private bool IsOwnedArtifact(string fullPath)
{
return IsSamePath(fullPath, artifacts.SummaryPath) ||
IsSamePath(fullPath, artifacts.AssistantContextPath);
}
private static bool IsSamePath(string first, string second)
{
return string.Equals(
Path.GetFullPath(first),
Path.GetFullPath(second),
StringComparison.OrdinalIgnoreCase);
}
private static List<string> CreateDiffLines(string before, string after)
{
var model = InlineDiffBuilder.Diff(before, after);
return model.Lines
.Where(line => line.Type is ChangeType.Deleted or ChangeType.Inserted)
.Select(line => $"{(line.Type == ChangeType.Inserted ? "+" : "-")} {line.Text}")
.ToList();
}
private sealed record FileChange(string Path, IReadOnlyList<string> DiffLines);
}
@@ -12,20 +12,18 @@ public sealed class AsrDiagnosticService
this.pipelineFactory = pipelineFactory;
}
public async Task<AsrDiagnosticResult> TranscribeWavAsync(string path, CancellationToken cancellationToken)
public Task<AsrDiagnosticResult> TranscribeWavAsync(string path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("A WAV file path is required.", nameof(path));
}
return TranscribeWavAsync(path, launchProfileName: null, cancellationToken);
}
var fullPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
if (!File.Exists(fullPath))
{
throw new FileNotFoundException($"WAV file was not found at '{fullPath}'.", fullPath);
}
await using var pipeline = pipelineFactory.Create();
public async Task<AsrDiagnosticResult> TranscribeWavAsync(
string path,
string? launchProfileName,
CancellationToken cancellationToken)
{
var fullPath = ResolveExistingWavPath(path);
await using var pipeline = pipelineFactory.Create(launchProfileName);
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
@@ -39,7 +37,17 @@ public sealed class AsrDiagnosticService
CancellationToken cancellationToken)
{
var fullPath = ResolveExistingWavPath(path);
await using var pipeline = pipelineFactory.Create();
return await DiarizeWavAsync(fullPath, options, launchProfileName: null, cancellationToken);
}
public async Task<AsrDiagnosticResult> DiarizeWavAsync(
string path,
SpeechRecognitionPipelineOptions options,
string? launchProfileName,
CancellationToken cancellationToken)
{
var fullPath = ResolveExistingWavPath(path);
await using var pipeline = pipelineFactory.Create(launchProfileName);
await pipeline.InitializeAsync(cancellationToken);
var liveTranscriptTask = CollectLiveTranscriptAsync(pipeline, cancellationToken);
await WriteWavToPipelineAsync(fullPath, pipeline, paceAudio: true, cancellationToken);
@@ -99,7 +107,9 @@ public sealed class AsrDiagnosticService
string path,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
using var reader = new WaveFileReader(path);
var reader = new WaveFileReader(path);
try
{
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
@@ -113,6 +123,18 @@ public sealed class AsrDiagnosticService
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
finally
{
try
{
reader.Dispose();
}
catch (NotSupportedException)
{
// NAudio can throw while disposing reader streams from async iterators; diagnostics should not fail after reading all chunks.
}
}
}
}
@@ -0,0 +1,18 @@
namespace MeetingAssistant.Transcription;
public sealed class AzureSpeechRecognitionPipelineBuilder :
SpeechRecognitionPipelineBuilder,
ISpeechRecognitionPipelineBuilder
{
public AzureSpeechRecognitionPipelineBuilder(IServiceProvider services)
: base(services)
{
}
public string ProviderName => "azure-speech";
public ISpeechRecognitionPipeline Create(MeetingAssistantOptions options)
{
return new AzureSpeechRecognitionPipeline(CreateProfiled<AzureSpeechStreamingTranscriptionProvider>(options));
}
}
@@ -25,7 +25,9 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
SpeechRecognitionPipelineOptions pipelineOptions,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await using var enumerator = audio.GetAsyncEnumerator(cancellationToken);
var enumerator = audio.GetAsyncEnumerator(cancellationToken);
try
{
if (!await enumerator.MoveNextAsync())
{
yield break;
@@ -94,6 +96,18 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
}
await pumpTask.WaitAsync(cancellationToken);
}
finally
{
try
{
await enumerator.DisposeAsync();
}
catch (NotSupportedException)
{
// Some diagnostic async enumerable sources expose DisposeAsync but throw after the stream has been read.
}
}
}
internal ConversationTranscriber CreateTranscriber(SpeechConfig speechConfig, AudioConfig audioConfig)
@@ -150,7 +164,7 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
$"Azure Speech key is not configured. Set AzureSpeech:Key or environment variable {azure.KeyEnv}.");
}
var speechConfig = SpeechConfig.FromEndpoint(GetEndpoint(azure), key);
var speechConfig = CreateSpeechConfig(azure, key);
var autoDetectLanguages = GetAutoDetectLanguages(azure);
if (autoDetectLanguages.Count == 0)
{
@@ -167,6 +181,13 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
speechConfig.SetProperty(
PropertyId.SpeechServiceResponse_DiarizeIntermediateResults,
azure.DiarizeIntermediateResults ? "true" : "false");
if (!string.IsNullOrWhiteSpace(azure.PostProcessingOption))
{
logger.LogWarning(
"Azure Speech post-processing option {PostProcessingOption} is configured but skipped because the live Azure backend uses ConversationTranscriber; Speech SDK post-processing is documented for SpeechRecognizer.",
azure.PostProcessingOption.Trim());
}
return speechConfig;
}
@@ -185,10 +206,17 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
}
internal static Uri GetEndpoint(AzureSpeechOptions options)
{
return string.IsNullOrWhiteSpace(options.Endpoint)
? throw new InvalidOperationException("Azure Speech endpoint must be configured.")
: new Uri(options.Endpoint);
}
internal static SpeechConfig CreateSpeechConfig(AzureSpeechOptions options, string key)
{
if (!string.IsNullOrWhiteSpace(options.Endpoint))
{
return new Uri(options.Endpoint);
return SpeechConfig.FromEndpoint(GetEndpoint(options), key);
}
if (string.IsNullOrWhiteSpace(options.Region))
@@ -196,7 +224,7 @@ public sealed class AzureSpeechStreamingTranscriptionProvider : IStreamingTransc
throw new InvalidOperationException("Azure Speech region or endpoint must be configured.");
}
return new Uri($"wss://{options.Region}.stt.speech.microsoft.com/speech/universal/v2");
return SpeechConfig.FromSubscription(key, options.Region.Trim());
}
internal static string NormalizeLanguageIdMode(string? value)
@@ -1,29 +1,75 @@
using MeetingAssistant.LaunchProfiles;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public sealed class ConfiguredSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
{
private readonly IServiceProvider services;
private readonly IReadOnlyDictionary<string, ISpeechRecognitionPipelineBuilder> builders;
private readonly MeetingAssistantOptions options;
private readonly ILaunchProfileOptionsProvider? launchProfiles;
public ConfiguredSpeechRecognitionPipelineFactory(
IServiceProvider services,
IEnumerable<ISpeechRecognitionPipelineBuilder> builders,
IOptions<MeetingAssistantOptions> options)
: this(builders, options, launchProfiles: null)
{
}
public ConfiguredSpeechRecognitionPipelineFactory(
IEnumerable<ISpeechRecognitionPipelineBuilder> builders,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider? launchProfiles)
{
this.services = services;
this.options = options.Value;
this.launchProfiles = launchProfiles;
this.builders = builders
.GroupBy(builder => builder.ProviderName, StringComparer.OrdinalIgnoreCase)
.ToDictionary(
group => group.Key,
group => group.Count() == 1
? group.Single()
: throw new InvalidOperationException(
$"Speech recognition pipeline provider '{group.Key}' is registered more than once."),
StringComparer.OrdinalIgnoreCase);
if (this.builders.Count == 0)
{
throw new InvalidOperationException("No speech recognition pipeline builders are registered.");
}
}
public ISpeechRecognitionPipeline Create()
{
return options.Recording.TranscriptionProvider switch
{
"funasr" => ActivatorUtilities.CreateInstance<FunAsrSpeechRecognitionPipeline>(services),
"whisper-local" => ActivatorUtilities.CreateInstance<WhisperLocalSpeechRecognitionPipeline>(services),
"azure-speech" => ActivatorUtilities.CreateInstance<AzureSpeechRecognitionPipeline>(services),
_ => throw new InvalidOperationException(
$"Unsupported speech recognition pipeline '{options.Recording.TranscriptionProvider}'.")
};
return Create(null);
}
public ISpeechRecognitionPipeline Create(string? launchProfileName)
{
var profileOptions = ResolveOptions(launchProfileName);
if (!builders.TryGetValue(profileOptions.Recording.TranscriptionProvider, out var builder))
{
throw new InvalidOperationException(
$"Unsupported speech recognition pipeline '{profileOptions.Recording.TranscriptionProvider}'.");
}
return builder.Create(profileOptions);
}
private MeetingAssistantOptions ResolveOptions(string? launchProfileName)
{
if (launchProfiles is not null)
{
return launchProfiles.GetRequiredProfile(launchProfileName).Options;
}
if (string.IsNullOrWhiteSpace(launchProfileName) ||
launchProfileName.Equals(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
return options;
}
throw new InvalidOperationException(
$"Launch profile '{launchProfileName}' was requested, but no launch profile provider is registered.");
}
}
@@ -0,0 +1,22 @@
namespace MeetingAssistant.Transcription;
public sealed class FunAsrSpeechRecognitionPipelineBuilder :
SpeechRecognitionPipelineBuilder,
ISpeechRecognitionPipelineBuilder
{
public FunAsrSpeechRecognitionPipelineBuilder(IServiceProvider services)
: base(services)
{
}
public string ProviderName => "funasr";
public ISpeechRecognitionPipeline Create(MeetingAssistantOptions options)
{
var backendLifecycle = GetRequiredService<IFunAsrBackendLifecycle>();
return new FunAsrSpeechRecognitionPipeline(
CreateProfiled<FunAsrStreamingTranscriptionProvider>(options),
backendLifecycle,
CreateProfiled<FunAsrTranscriptFinalizer>(options));
}
}
@@ -4,5 +4,20 @@ public interface IDictationWordStore
{
Task<IReadOnlyList<string>> ReadWordsAsync(CancellationToken cancellationToken);
Task<IReadOnlyList<string>> ReadWordsAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return ReadWordsAsync(cancellationToken);
}
Task<IReadOnlyList<string>> AddWordAsync(string word, CancellationToken cancellationToken);
Task<IReadOnlyList<string>> AddWordAsync(
string word,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return AddWordAsync(word, cancellationToken);
}
}
@@ -27,6 +27,11 @@ public interface ISpeechRecognitionPipeline : IAsyncDisposable
public interface ISpeechRecognitionPipelineFactory
{
ISpeechRecognitionPipeline Create();
ISpeechRecognitionPipeline Create(string? launchProfileName)
{
return Create();
}
}
public sealed record SpeechRecognitionPipelineOptions(
@@ -0,0 +1,8 @@
namespace MeetingAssistant.Transcription;
public interface ISpeechRecognitionPipelineBuilder
{
string ProviderName { get; }
ISpeechRecognitionPipeline Create(MeetingAssistantOptions options);
}
@@ -13,7 +13,14 @@ public sealed class MarkdownDictationWordStore : IDictationWordStore
public async Task<IReadOnlyList<string>> ReadWordsAsync(CancellationToken cancellationToken)
{
var path = GetPath();
return await ReadWordsAsync(options, cancellationToken);
}
public async Task<IReadOnlyList<string>> ReadWordsAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var path = GetPath(options);
if (!File.Exists(path))
{
return [];
@@ -25,7 +32,15 @@ public sealed class MarkdownDictationWordStore : IDictationWordStore
public async Task<IReadOnlyList<string>> AddWordAsync(string word, CancellationToken cancellationToken)
{
var path = GetPath();
return await AddWordAsync(word, options, cancellationToken);
}
public async Task<IReadOnlyList<string>> AddWordAsync(
string word,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var path = GetPath(options);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var existing = File.Exists(path)
? await File.ReadAllLinesAsync(path, cancellationToken)
@@ -39,7 +54,7 @@ public sealed class MarkdownDictationWordStore : IDictationWordStore
return words;
}
private string GetPath()
private static string GetPath(MeetingAssistantOptions options)
{
return VaultPath.Resolve(options.Vault, options.Vault.DictationWordsPath);
}
@@ -0,0 +1,25 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Transcription;
public abstract class SpeechRecognitionPipelineBuilder
{
private readonly IServiceProvider services;
protected SpeechRecognitionPipelineBuilder(IServiceProvider services)
{
this.services = services;
}
protected T CreateProfiled<T>(MeetingAssistantOptions options)
{
return ActivatorUtilities.CreateInstance<T>(services, Options.Create(options));
}
protected T GetRequiredService<T>()
where T : notnull
{
return services.GetRequiredService<T>();
}
}
@@ -0,0 +1,20 @@
namespace MeetingAssistant.Transcription;
public sealed class WhisperLocalSpeechRecognitionPipelineBuilder :
SpeechRecognitionPipelineBuilder,
ISpeechRecognitionPipelineBuilder
{
public WhisperLocalSpeechRecognitionPipelineBuilder(IServiceProvider services)
: base(services)
{
}
public string ProviderName => "whisper-local";
public ISpeechRecognitionPipeline Create(MeetingAssistantOptions options)
{
return new WhisperLocalSpeechRecognitionPipeline(
CreateProfiled<WhisperLocalStreamingTranscriptionProvider>(options),
CreateProfiled<PyannoteTranscriptFinalizer>(options));
}
}
+18 -4
View File
@@ -80,16 +80,16 @@
}
},
"AzureSpeech": {
"Endpoint": "wss://germanywestcentral.stt.speech.microsoft.com/speech/universal/v2",
"Region": "germanywestcentral",
"Endpoint": "",
"Region": "westeurope",
"Language": "de-DE",
"AutoDetectLanguages": [
"de-DE"
],
"LanguageIdMode": "Continuous",
"KeyEnv": "AZURE_SPEECH_KEY",
"RecognitionStopTimeout": "00:00:10",
"DiarizeIntermediateResults": true,
"PostProcessingOption": "",
"PhraseListWeight": 1.5
},
"SpeakerIdentification": {
@@ -103,7 +103,8 @@
"MaxSnippetsPerSpeaker": 3,
"SilenceBetweenSnippetsSeconds": 1,
"LiveSampleBufferDuration": "00:10:00",
"MergeRecentIdentityAge": "14.00:00:00"
"MergeRecentIdentityAge": "14.00:00:00",
"MatchTimeout": "00:03:00"
},
"Agent": {
"Endpoint": "https://litellm.schweigert.cloud",
@@ -122,6 +123,19 @@
},
"Api": {
"PublicBaseUrl": "http://localhost:5090"
},
"LaunchProfiles": {
"english": {
"Hotkey": {
"Toggle": "Ctrl+Alt+E"
},
"AzureSpeech": {
"Language": "en-US",
"AutoDetectLanguages": [
"en-US"
]
}
}
}
},
"Logging": {