Author SHA1 Message Date
renovate-bot 8474f902ee chore(deps): update dependency microsoft.aspnetcore.mvc.testing to 10.0.11
PR and Push Build/Test / build-and-test (push) Failing after 10m14s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m0s
2026-08-12 02:48:58 +00:00
119 changed files with 574 additions and 9800 deletions
@@ -1,7 +1,6 @@
using System.Net;
using System.Net.Http.Json;
using MeetingAssistant.Recording;
using MeetingAssistant.Workflow;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
@@ -42,17 +41,6 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
Assert.Equal("ok", body?.Status);
}
[Fact]
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
{
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
var result = await playback.QueueAsync(1, [1, 2]);
Assert.Contains("Windows build", result);
Assert.DoesNotContain("Queued", result);
}
[Fact]
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
{
@@ -58,64 +58,6 @@ public sealed class LaunchProfileOptionsProviderTests
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
}
[Fact]
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
{
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
}
[Fact]
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
}
[Fact]
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
}
[Fact]
public void ContradictorySpeakerSampleDurationsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
Assert.Contains("MaximumSampleDuration", exception.Message);
}
[Fact]
public void DuplicateProfileHotkeysAreRejected()
{
@@ -9,10 +9,10 @@
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="10.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.11" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
</ItemGroup>
<ItemGroup>
@@ -7,37 +7,6 @@ namespace MeetingAssistant.Tests;
public sealed class MeetingSummaryInstructionBuilderTests
{
[Fact]
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
var artifacts = CreateArtifacts(root);
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
{
Agent = new AgentOptions { InitialPrompt = "Custom base." },
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
}));
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Custom base.", instructions);
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha Platform", instructions);
Assert.Contains("Builds meeting automation", instructions);
Assert.Contains("PROJECT.md", instructions);
Assert.Contains("256", instructions);
Assert.Contains("request_project_association", instructions);
Assert.Contains("request_workflow_change", instructions);
Assert.DoesNotContain("Internal notes", instructions);
Assert.DoesNotContain("Do not expose", instructions);
}
[Fact]
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
{
@@ -106,8 +75,7 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Custom summarizer instructions.", instructions);
Assert.Contains("request_project_association", instructions);
Assert.Equal("Custom summarizer instructions.", instructions);
}
[Fact]
@@ -128,12 +96,21 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Base.", instructions);
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha instructions.", instructions);
Assert.Contains("# Beta", instructions);
Assert.Contains("Beta instructions.", instructions);
var expected = """
Base.
---
projects:
# Alpha
Alpha instructions.
# Beta
Beta instructions.
""";
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
}
private static MeetingSessionArtifacts CreateArtifacts(string root)
@@ -50,145 +50,6 @@ public sealed class MicrophoneAudioSourceTests
await cancellation.CancelAsync();
}
[Fact]
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SelectableCaptureDevices(selection);
var prompts = new AcceptingPrompt();
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
var nextChunk = chunks.MoveNextAsync().AsTask();
Assert.False(nextChunk.IsCompleted);
Assert.Null(selection.SelectedDeviceId);
prompts.Response.SetResult(true);
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.Equal("new", selection.SelectedDeviceId);
Assert.False(lifetime.IsCancellationRequested);
}
finally
{
await lifetime.CancelAsync();
}
}
[Fact]
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var abandonedCapture = new DisposableAudioSource();
var factory = new CallbackCaptureFactory(() =>
{
lifetime.Cancel();
return abandonedCapture;
});
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
Assert.False(await chunks.MoveNextAsync());
Assert.True(abandonedCapture.Disposed);
}
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
{
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
}
[Fact]
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SwitchDuringCreationDevices(selection);
var prompts = new AcceptingPrompt();
prompts.Response.SetResult(true);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal("new", selection.SelectedDeviceId);
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.True(devices.AbandonedCapture.Disposed);
}
finally
{
await lifetime.CancelAsync();
}
}
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
private MicrophoneDevice[] available = [new("old", "Built-in")];
private int creations;
private volatile bool switchObserved;
public DisposableAudioSource AbandonedCapture { get; } = new();
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
if (selection.SelectedDeviceId == "new") switchObserved = true;
return Volatile.Read(ref available);
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(GetAvailableMicrophones(), available[0]);
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
// Let the acceptance callback finish before returning the already-opened source.
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
throw new TimeoutException("The new microphone was not selected during capture creation.");
return AbandonedCapture;
}
}
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
{
public bool Disposed { get; private set; }
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
public void Dispose() => Disposed = true;
}
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
{
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
Shown.TrySetResult(microphone);
return Response.Task.WaitAsync(cancellationToken);
}
}
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
}
private static byte[] Pcm16(short sample)
{
return BitConverter.GetBytes(sample);
@@ -1,163 +0,0 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class MicrophoneSwitchMonitorTests
{
[Fact]
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var devices = new MutableDevices();
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var switched = false;
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
devices.Available = [new("usb", "USB microphone")];
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(prompts.WasShown);
Assert.False(switched);
}
[Fact]
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
{
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
var newDevice = new MicrophoneDevice("new", "USB headset");
var devices = new MutableDevices { Available = [oldDevice] };
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts);
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
try
{
devices.Available = [oldDevice, newDevice];
var prompt = await prompts.NextAsync();
Assert.Equal(newDevice, prompt.Device);
Assert.False(switched.Task.IsCompleted);
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
prompt.Response.SetResult(true);
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
[Theory]
[InlineData("declined")]
[InlineData("removed")]
[InlineData("expired")]
[InlineData("ended")]
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
{
var device = new MicrophoneDevice("usb", "USB microphone");
var devices = new MutableDevices();
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts,
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
var switched = false;
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
try
{
devices.Available = [device];
var prompt = await prompts.NextAsync();
if (outcome == "declined")
{
prompt.Response.SetResult(false);
await devices.WaitForPollsAsync();
}
else
{
if (outcome == "removed") devices.Available = [];
if (outcome == "ended") await lifetime.CancelAsync();
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
await lifetime.CancelAsync();
await monitoring;
Assert.False(switched);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
TimeSpan? promptLifetime = null) => new(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
promptLifetime ?? TimeSpan.FromMinutes(1));
private sealed class MutableDevices : IMicrophoneDeviceProvider
{
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
public volatile MicrophoneDevice[] Available = [];
public Action? BeforeRead { get; set; }
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
BeforeRead?.Invoke();
polls.Writer.TryWrite(true);
return Available;
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, Available.FirstOrDefault());
public async Task WaitForPollsAsync()
{
while (polls.Reader.TryRead(out _)) { }
for (var i = 0; i < 3; i++)
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
}
}
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
{
public bool WasShown { get; private set; }
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
WasShown = true;
beforeAcceptance();
return Task.FromResult(true);
}
}
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
{
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
var prompt = new Prompt(microphone, cancellationToken);
requests.Writer.TryWrite(prompt);
return await prompt.Response.Task.WaitAsync(cancellationToken);
}
}
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -25,16 +25,4 @@ public sealed class NotificationActivationArgumentsTests
Assert.Equal("abc123", arguments["promptId"]);
Assert.Equal("continue", arguments["response"]);
}
[Fact]
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
{
Assert.Contains(
MeetingInactivityPromptActions.All,
action => action.Content == "Pause transcription" &&
action.ResponseArgument == "pause");
Assert.Equal(
MeetingInactivityPromptResponse.Pause,
MeetingInactivityPromptActions.ParseResponse("pause"));
}
}
@@ -1,90 +1,10 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class ProjectKnowledgeToolTests
{
[Theory]
[InlineData("name: Incomplete project")]
[InlineData("description: Missing name")]
[InlineData("name: ' '\ndescription: Missing name")]
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
using var catalog = JsonDocument.Parse(await tools.ListProjects());
var project = Assert.Single(catalog.RootElement.EnumerateArray());
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
Assert.Equal("", project.GetProperty("description").GetString());
}
[Fact]
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
var tools = new MeetingSummaryTools(artifacts,
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
$"description: {new string('b', 257)}", from: 3, to: 3);
Assert.StartsWith("Refused:", rejected);
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
"description: Updated project purpose", from: 3, to: 3));
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
Assert.Contains("owner: Ada", updated);
Assert.Contains("User-authored project notes.", updated);
Assert.Contains("Updated project purpose", await tools.ListProjects());
}
[Fact]
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
var catalog = await tools.ListProjects();
using var json = JsonDocument.Parse(catalog);
var projects = json.RootElement.EnumerateArray().ToArray();
Assert.Equal(3, projects.Length);
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
Assert.Equal("", projects[1].GetProperty("description").GetString());
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
Assert.Equal("", projects[2].GetProperty("description").GetString());
Assert.DoesNotContain("Private", catalog);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
{
@@ -128,7 +48,7 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Contains("MeetingAssistant", await tools.ListProjects());
Assert.Equal("MeetingAssistant", await tools.ListProjects());
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
@@ -136,15 +56,15 @@ public sealed class ProjectKnowledgeToolTests
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.Equal(
"README.md:2 Second alpha line",
await tools.Search("alpha"));
}
[Fact]
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
@@ -168,8 +88,7 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
Assert.Equal("MeetingAssistant", await tools.ListProjects());
}
[Fact]
@@ -181,11 +100,8 @@ public sealed class ProjectKnowledgeToolTests
Directory.CreateDirectory(projectRoot);
var projectFile = Path.Combine(projectRoot, "notes.md");
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var tools = new MeetingSummaryTools(
artifacts,
CreateArtifacts(root),
new MeetingAssistantOptions
{
Vault =
@@ -216,13 +132,8 @@ public sealed class ProjectKnowledgeToolTests
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var outsidePath = Path.Combine(projectsRoot, "outside.md");
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
var tools = new MeetingSummaryTools(
artifacts,
CreateArtifacts(root),
new MeetingAssistantOptions
{
Vault =
@@ -233,7 +144,6 @@ public sealed class ProjectKnowledgeToolTests
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
@@ -18,8 +18,31 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
Enabled = true,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote-validation:local",
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
}),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
@@ -31,109 +54,23 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
Assert.True(commandRunner.RunCancellationWasObserved);
}
[Fact]
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(
[
new LaunchProfile("english", namedOptions),
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
]),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await commandRunner.WaitForRunAsync();
await service.StopAsync(CancellationToken.None);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
}
[Fact]
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(options),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromMilliseconds(50));
await service.StopAsync(CancellationToken.None);
Assert.Empty(commandRunner.Commands);
}
private static MeetingAssistantOptions CreateValidationOptions(string image)
{
return new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
DockerCommand = "docker",
Image = image,
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
};
}
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
{
private readonly IReadOnlyList<LaunchProfile> profiles;
private readonly MeetingAssistantOptions options;
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
{
}
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
{
this.profiles = profiles;
this.options = options;
}
public LaunchProfile GetRequiredProfile(string? name)
{
var profileName = string.IsNullOrWhiteSpace(name)
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
: name;
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
}
public IReadOnlyList<LaunchProfile> GetProfiles()
{
return profiles;
return [GetRequiredProfile(null)];
}
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
@@ -2,7 +2,6 @@ using MeetingAssistant;
using MeetingAssistant.Recording;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NAudio.Wave;
@@ -58,49 +57,17 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Assert.Empty(commandRunner.Commands);
}
[Fact]
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
{
var commandRunner = new CapturingCommandRunner(
"""
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
""");
var modelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
})
.Build();
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
var validator = CreateValidator(commandRunner, configuredOptions);
var valid = await validator.ValidateSampleAsync(
CreateWav(TimeSpan.FromSeconds(20)),
CancellationToken.None);
Assert.True(valid);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
bool enabled = true)
{
return CreateValidator(
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
new MeetingAssistantOptions
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
@@ -109,8 +76,9 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Enabled = enabled,
MinimumSingleSpeakerCoverage = 0.90,
MinimumMatchingKnownSnippetRatio = 1,
Diarization = new PyannoteRuntimeOptions
Diarization = new PyannoteDiarizationOptions
{
Enabled = true,
BuildImage = false,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote:local",
@@ -126,20 +94,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
}
}
}
});
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
MeetingAssistantOptions configuredOptions)
{
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(configuredOptions),
}),
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
}
@@ -156,8 +156,9 @@ public sealed class PyannoteTranscriptFinalizerTests
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var explicitDiarization = new PyannoteRuntimeOptions
var explicitDiarization = new PyannoteDiarizationOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-azure:local",
@@ -168,7 +169,7 @@ public sealed class PyannoteTranscriptFinalizerTests
CommandTimeout = TimeSpan.FromMinutes(1)
};
await finalizer.FinalizeEnabledAsync(
await finalizer.FinalizeAsync(
audioPath,
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
explicitDiarization,
@@ -185,8 +186,9 @@ public sealed class PyannoteTranscriptFinalizerTests
{
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
var diarization = new PyannoteRuntimeOptions
var diarization = new PyannoteDiarizationOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-warmup:local",
@@ -216,8 +218,9 @@ public sealed class PyannoteTranscriptFinalizerTests
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: null);
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
{
Enabled = true,
Token = "",
TokenEnv = "",
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
File diff suppressed because it is too large Load Diff
@@ -1,596 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentificationServiceTests
{
[Fact]
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(7, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Single(fixture.Encoder.Requests);
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
}
[Fact]
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Sabrina", saved.CanonicalName);
Assert.Equal(3, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Null(saved.CanonicalName);
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
}
[Fact]
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(8, saved.VoiceVectors.Count);
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
{
await using var fixture = await Fixture.CreateAsync(options =>
{
options.OutlierPruningMinimumVectors = 20;
options.OutlierPruningNeighborSimilarity = 0.90;
options.OutlierPruningMinimumNeighbors = 3;
options.OutlierPruningMinimumClusterRatio = 0.60;
});
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(16, saved.VoiceVectors.Count);
Assert.All(
saved.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
"Chris",
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
var currentRunVectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
var service = fixture.CreateService();
var liveResult = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
fixture.Encoder.Vectors = currentRunVectors;
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
{
KnownSpeakerMappings = liveResult.SpeakerMappings
};
if (transcriptAlreadyRelabeled)
{
finalRequest = finalRequest with { Segments = liveResult.Segments };
}
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(10, saved.VoiceVectors.Count);
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
}
[Fact]
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
{
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors =
[
UnitVector(0),
UnitVector(0, 2, 0.01f),
UnitVector(0, 3, 0.02f),
UnitVector(0, 4, 0.03f),
UnitVector(0, 5, 0.04f)
];
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(6, saved.VoiceVectors.Count);
Assert.Single(saved.References);
}
[Fact]
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyFinishedSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments[0].Speaker);
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
Assert.All(
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal("John", saved.CanonicalName);
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
Assert.Equal("John", result.Segments.Single().Speaker);
Assert.All(saved.References, reference =>
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
}
[Fact]
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Null(saved.CanonicalName);
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
}
[Fact]
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Manuel", saved.CanonicalName);
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
}
[Fact]
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
{
await using var fixture = await Fixture.CreateAsync(
configureSpeaker: options => options.MaxMatchCandidates = 1);
await fixture.AddIdentityAsync("Legacy WAV identity", []);
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
}
[Fact]
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
{
await using var fixture = await Fixture.CreateAsync();
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.02f))
.ToList();
var service = fixture.CreateService();
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(target.Id);
Assert.Single(saved.Snippets);
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
Assert.Single(await context.SpeakerIdentities.ToListAsync());
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private sealed class Fixture : IAsyncDisposable
{
private readonly string directory;
private readonly string databasePath;
private readonly SpeakerIdentificationOptions speakerOptions;
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
{
this.directory = directory;
this.databasePath = databasePath;
this.speakerOptions = speakerOptions;
}
public FakeEncoder Encoder { get; } = new();
public FakeSnippetExtractor SnippetExtractor { get; } = new();
public string DatabasePath => databasePath;
public static async Task<Fixture> CreateAsync(
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
{
var directory = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
var databasePath = Path.Combine(directory, "speaker-identities.db");
var speakerOptions = new SpeakerIdentificationOptions
{
DatabasePath = databasePath,
MatchBatchSize = 6,
MaxMatchCandidates = 100,
MatchIdentityActiveAge = TimeSpan.FromDays(365),
MinimumSampleSpeechDuration = TimeSpan.Zero,
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
}
};
configure?.Invoke(speakerOptions.Resemblyzer);
configureSpeaker?.Invoke(speakerOptions);
await using var context = new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
return new Fixture(directory, databasePath, speakerOptions);
}
public ResemblyzerSpeakerIdentificationService CreateService()
{
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
return new ResemblyzerSpeakerIdentificationService(
new TestDbContextFactory(databasePath),
SnippetExtractor,
Encoder,
new ResemblyzerVoiceClusterMatcher(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(appOptions),
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
}
public SpeakerIdentificationRequest CreateRequest(
string speaker,
int sampleCount,
IReadOnlyList<string>? attendees = null,
int segmentCount = 1)
{
var transcriptPath = Path.Combine(directory, "transcript.md");
File.WriteAllText(transcriptPath, "Transcript");
var segments = Enumerable.Range(0, segmentCount)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 30),
TimeSpan.FromSeconds((index + 1) * 30),
speaker,
"enough useful words for a speaker sample"))
.ToList();
var segment = segments[0];
return new SpeakerIdentificationRequest(
Path.Combine(directory, "meeting.wav"),
new MeetingNote(
Path.Combine(directory, "meeting.md"),
new MeetingNoteFrontmatter
{
Title = "Test",
Attendees = attendees?.ToList() ?? ["Chris"],
Transcript = transcriptPath,
AssistantContext = Path.Combine(directory, "context.md"),
Summary = Path.Combine(directory, "summary.md")
},
""),
segments,
Enumerable.Range(0, sampleCount)
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
.ToList());
}
public async Task<SpeakerIdentity> AddIdentityAsync(
string? name,
IReadOnlyList<float[]> vectors,
IReadOnlyList<string>? candidates = null,
IReadOnlyList<string>? aliases = null)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = speakerOptions.Resemblyzer.ModelId,
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"known-{index}",
CreatedAt = now.AddMinutes(index)
}).ToList()
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
return identity;
}
public async Task AddCurrentRunCandidateAsync(
SpeakerIdentificationRequest request,
string candidateName)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CreatedAt = now,
UpdatedAt = now,
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "older-model",
Dimensions = 256,
VectorBytes = ToBytes(UnitVector(1)),
Fingerprint = "older-model-vector",
CreatedAt = now
}
],
References =
[
SpeakerIdentityReferences.Create(
request.MeetingNote.Path,
request.MeetingNote.Frontmatter.Transcript,
now)
]
});
await context.SaveChangesAsync();
}
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Aliases)
.Include(identity => identity.References)
.SingleAsync(identity => identity.Id == id);
}
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.References)
.SingleAsync();
}
public ValueTask DisposeAsync()
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
return ValueTask.CompletedTask;
}
}
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
{
public IReadOnlyList<float[]> Vectors { get; set; } = [];
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
{
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
public Task<byte[]> ExtractSnippetAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> speakerSegments,
CancellationToken cancellationToken)
{
Requests.Add(speakerSegments.ToList());
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
}
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -1,174 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
{
[Fact]
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
var databasePath = Path.Combine(directory, "identities.db");
var factory = new TestDbContextFactory(databasePath);
await using (var context = factory.CreateDbContext())
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(CreateIdentity(
"Chris",
DateTimeOffset.UtcNow.AddMonths(-2),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 20, 0.02f))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Chris duplicate",
DateTimeOffset.UtcNow.AddDays(-1),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.02f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 40, 0.02f)))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Decoy",
DateTimeOffset.UtcNow.AddMonths(-1),
[UnitVector(0)],
directory));
await context.SaveChangesAsync();
}
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
};
var service = new ResemblyzerSpeakerIdentityMergeService(
factory,
new ResemblyzerVoiceClusterMatcher(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
MergeRecentIdentityAge = TimeSpan.FromDays(14),
MaxMatchCandidates = 1,
MaxSnippetsPerSpeaker = 3,
Resemblyzer = resemblyzerOptions
}
}),
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
Assert.Equal(2, result.MatchAttempts);
Assert.Equal(1, result.MergedPairs);
await using var verification = factory.CreateDbContext();
var saved = await verification.SpeakerIdentities
.Include(identity => identity.Aliases)
.Include(identity => identity.VoiceVectors)
.SingleAsync(identity => identity.CanonicalName == "Chris");
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
Assert.Equal("Chris", saved.CanonicalName);
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
Assert.Equal(20, saved.VoiceVectors.Count);
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
}
finally
{
Directory.Delete(directory, recursive: true);
}
}
private static SpeakerIdentity CreateIdentity(
string name,
DateTimeOffset createdAt,
IReadOnlyList<float[]> vectors,
string directory)
{
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
File.WriteAllText(transcriptPath, "Transcript");
return new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = createdAt,
UpdatedAt = createdAt,
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"{name}-{index}",
CreatedAt = createdAt.AddMinutes(index)
}).ToList(),
References =
[
new SpeakerIdentityReference
{
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
TranscriptPath = transcriptPath,
CreatedAt = createdAt
}
]
};
}
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
{
var vector = new float[256];
vector[primary] = 1;
if (secondary is { } index)
{
vector[index] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private static float[] ToVector(byte[] bytes)
{
var vector = new float[bytes.Length / sizeof(float)];
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
return vector;
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -1,122 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceClusterMatcherTests
{
[Fact]
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
{
var matcher = CreateMatcher();
var query = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
.ToList();
var candidates = new[]
{
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
};
var result = matcher.Match(query, candidates);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
Assert.True(result.Cohesion >= 0.99);
Assert.True(result.BestSimilarity >= 0.99);
Assert.True(result.RunnerUpSimilarity < 0.1);
}
[Fact]
public void FourVectorsDoNotTriggerAutomaticMatching()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 4).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("4/5", result.Reason);
}
[Fact]
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
{
var result = CreateMatcher().Match(
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("cohesion", result.Reason);
Assert.Null(result.BestSimilarity);
}
[Fact]
public void SimilarityBelowThresholdIsRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
Assert.False(result.Accepted);
Assert.Contains("best similarity", result.Reason);
Assert.Equal(0, result.BestSimilarity);
}
[Fact]
public void SimilarCandidatesWithinMarginAreRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
]);
Assert.False(result.Accepted);
Assert.Contains("margin", result.Reason);
Assert.NotNull(result.RunnerUpSimilarity);
}
[Fact]
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
]);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
}
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
{
return new ResemblyzerVoiceClusterMatcher(
new ResemblyzerSpeakerRecognitionOptions
{
RequiredVectorsPerSpeaker = 5,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05
},
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
}
@@ -1,141 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
{
[Fact]
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(16, identity.VoiceVectors.Count);
Assert.All(
identity.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Fact]
public void AmbiguousDenseClustersPreserveAllVectors()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 10)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(20, identity.VoiceVectors.Count);
}
[Fact]
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 15)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(19, identity.VoiceVectors.Count);
}
[Fact]
public void PruningPreservesOtherModelsAndMalformedStoredRows()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
SpeakerVoiceVectors.AddDistinct(
identity,
[UnitVector(2, 60, 0.04f)],
"older-model",
1000,
DateTimeOffset.UtcNow);
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = [1],
Fingerprint = "malformed",
CreatedAt = DateTimeOffset.UtcNow
});
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "zero-magnitude",
CreatedAt = DateTimeOffset.UtcNow
});
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(19, identity.VoiceVectors.Count);
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
}
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
{
return new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions
{
ModelId = "resemblyzer-0.1.4-pretrained",
OutlierPruningMinimumVectors = 20,
OutlierPruningNeighborSimilarity = 0.90,
OutlierPruningMinimumNeighbors = 3,
OutlierPruningMinimumClusterRatio = 0.60
},
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
}
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
{
var identity = new SpeakerIdentity();
SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
"resemblyzer-0.1.4-pretrained",
1000,
DateTimeOffset.UtcNow);
return identity;
}
private static float[] UnitVector(
int primaryDimension,
int secondaryDimension,
float secondaryValue)
{
var vector = new float[256];
vector[primaryDimension] = 1;
vector[secondaryDimension] = secondaryValue;
return vector;
}
}
@@ -1,78 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerWarmupHostedServiceTests
{
[Fact]
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
{
var encoder = new BlockingEncoder();
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(options),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await encoder.WaitForWarmupAsync();
await service.StopAsync(CancellationToken.None);
Assert.True(encoder.CancellationObserved);
}
[Fact]
public async Task DisabledWarmupDoesNotInvokeEncoder()
{
var encoder = new BlockingEncoder();
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(new MeetingAssistantOptions()),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await service.StopAsync(CancellationToken.None);
Assert.False(encoder.WarmupStarted);
}
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
{
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool WarmupStarted { get; private set; }
public bool CancellationObserved { get; private set; }
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
WarmupStarted = true;
started.TrySetResult();
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
CancellationObserved = true;
}
}
public Task WaitForWarmupAsync()
{
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
}
}
}
@@ -8,22 +8,13 @@ namespace MeetingAssistant.Tests;
public sealed class SpeakerAudioSampleCollectorTests
{
[Fact]
public void DefaultSampleDurationsAreTenAndSixtySeconds()
{
var options = new SpeakerIdentificationOptions();
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
}
[Fact]
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
@@ -46,8 +37,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
@@ -57,61 +48,6 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
}
[Fact]
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
var sample = Assert.Single(collector.Snapshot());
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
}
[Fact]
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
Assert.Empty(collector.Snapshot());
}
[Fact]
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
var sample = collector.TryAdd(Segment(
0,
70,
"Guest01",
"one two three four five six seven eight nine ten."));
Assert.NotNull(sample);
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
}
[Fact]
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
{
@@ -119,8 +55,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60),
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1),
logger: logger);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
@@ -133,47 +69,6 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.Contains("minimum duration", message);
}
[Fact]
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
Assert.Collection(
samples,
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
}
[Fact]
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
Assert.Single(collector.Snapshot());
}
private static TranscriptionSegment Segment(
double start,
double end,
@@ -107,9 +107,6 @@ public sealed class SpeakerIdentityMergeServiceTests
return new SpeakerIdentityMergeService(
new TestSpeakerIdentityDbContextFactory(dbPath),
Matcher,
new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions(),
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
@@ -1,47 +0,0 @@
using MeetingAssistant.Speakers;
namespace MeetingAssistant.Tests;
public sealed class SpeakerIdentityMergerTests
{
[Fact]
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
{
var now = DateTimeOffset.UtcNow;
var target = new SpeakerIdentity
{
CanonicalName = "Chris",
VoiceVectors =
[
Vector("oldest", 1, now.AddMinutes(-4)),
Vector("duplicate", 2, now.AddMinutes(-3))
]
};
var source = new SpeakerIdentity
{
VoiceVectors =
[
Vector("duplicate", 2, now.AddMinutes(-2)),
Vector("newer", 3, now.AddMinutes(-1)),
Vector("newest", 4, now)
]
};
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
}
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
{
return new SpeakerVoiceVector
{
ModelId = "model",
Dimensions = 256,
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
Fingerprint = fingerprint,
CreatedAt = createdAt
};
}
}
@@ -1,80 +0,0 @@
using MeetingAssistant.Speakers;
using Microsoft.Data.Sqlite;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace MeetingAssistant.Tests;
public sealed class SpeakerRecognitionFeatureSelectionTests
{
[Theory]
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
bool enabled,
Type expectedIdentificationType,
Type expectedMergeType,
int expectedMinimumSamples,
bool expectedNonOverlappingSamples)
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
await using var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, configuration) =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
});
});
builder.ConfigureTestServices(services =>
{
services.RemoveAll<IResemblyzerVoiceEncoder>();
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
});
});
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
Assert.IsType(expectedIdentificationType, service);
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
Assert.IsType(expectedMergeType, mergeService);
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
}
finally
{
SqliteConnection.ClearAllPools();
Directory.Delete(directory, recursive: true);
}
}
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
{
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}
@@ -1,142 +0,0 @@
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Tests;
public sealed class SpeakerSampleSpanSelectorTests
{
[Fact]
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(7, 13),
Segment(20, 70)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Equal(3, span.Count);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
[Fact]
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Empty(span);
}
[Fact]
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
{
var segments = Enumerable.Range(0, 6)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 10),
TimeSpan.FromSeconds((index + 1) * 10),
"Guest01",
$"segment {index} has enough words"))
.ToList();
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
segments,
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(5, spans.Count);
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
Assert.All(
spans.Zip(spans.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
}
[Fact]
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 10),
Segment(9, 19),
Segment(20, 30)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(2, spans.Count);
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
}
[Fact]
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(7, 13)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
}
[Fact]
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Empty(spans);
}
[Fact]
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[Segment(0, 70)],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
private static TranscriptionSegment Segment(double start, double end)
{
return new TranscriptionSegment(
TimeSpan.FromSeconds(start),
TimeSpan.FromSeconds(end),
"Guest01",
"enough words for this segment");
}
}
@@ -1,106 +0,0 @@
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
namespace MeetingAssistant.Tests;
public sealed class SpeakerVoiceVectorPersistenceTests
{
[Fact]
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using (var context = new SpeakerIdentityDbContext(options))
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-a",
CreatedAt = DateTimeOffset.UtcNow
}
]
});
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
var identity = await context.SpeakerIdentities
.Include(candidate => candidate.VoiceVectors)
.SingleAsync();
var vector = Assert.Single(identity.VoiceVectors);
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
Assert.Equal(256, vector.Dimensions);
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
context.SpeakerIdentities.Remove(identity);
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
}
}
[Fact]
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using var context = new SpeakerIdentityDbContext(options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
var identity = new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
context.SpeakerVoiceVectors.AddRange(
CreateVector(identity.Id, "same-vector"),
CreateVector(identity.Id, "same-vector"));
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
}
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
{
return new SpeakerVoiceVector
{
SpeakerIdentityId = identityId,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = fingerprint,
CreatedAt = DateTimeOffset.UtcNow
};
}
}
@@ -1,209 +0,0 @@
using System.Threading.Channels;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Summary;
using Microsoft.Extensions.Logging.Abstractions;
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Tests;
public sealed class SummaryAgentRequestTests
{
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("shutdown")]
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
{
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
finally { await requests.StopAsync(CancellationToken.None); }
Assert.Empty(editor.Prompts);
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
}
[Fact]
public async Task InvalidRequestsAreRefusedBeforeNotification()
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
Assert.False(prompts.HasMore);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
}
[Fact]
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
Assert.StartsWith("Requested", result);
var prompt = await prompts.NextAsync();
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
Assert.Empty(editor.Prompts);
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
Assert.True(tools.SummaryWasWritten);
prompt.Response.SetResult(true);
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.False(prompt.Response.TrySetResult(true));
Assert.Single(editor.Prompts);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
}
finally { await requests.StopAsync(CancellationToken.None); }
}
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("canceled")]
[InlineData("removed")]
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
using var cancellation = new CancellationTokenSource();
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "canceled") await cancellation.CancelAsync();
if (outcome == "removed")
{
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
prompt.Response.SetResult(true);
}
if (outcome == "canceled")
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
else
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
var prompt = await prompts.NextAsync();
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
Assert.False(request.IsCompleted);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
prompt.Response.SetResult(true);
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
Assert.StartsWith("Approved", result);
Assert.Contains("Alpha instructions", result);
Assert.Contains("Alpha Platform", result);
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
Assert.Contains("Edited by user", note);
Assert.Contains("custom: keep-me", note);
Assert.Contains("User's latest notes.", note);
Assert.Contains("Existing", note);
Assert.Contains("Alpha", note);
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
Assert.False(prompts.HasMore);
}
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var project = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(project);
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
}
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
{
public List<string> Prompts { get; } = [];
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Show() { }
public void ShowWithPrompt(string initialPrompt)
{
Prompts.Add(initialPrompt);
Started.TrySetResult(initialPrompt);
}
}
private sealed class ControlledPrompts : IAgentApprovalPromptService
{
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
var request = new PendingPrompt(prompt, cancellationToken);
requests.Writer.TryWrite(request);
try { return await request.Response.Task.WaitAsync(cancellationToken); }
finally { request.Removed.TrySetResult(); }
}
}
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
+4 -29
View File
@@ -74,8 +74,7 @@ public sealed class TaskbarIconTests
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
("Exit", MeetingTaskbarAction.Exit, true));
@@ -95,32 +94,10 @@ public sealed class TaskbarIconTests
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, true),
("Exit", MeetingTaskbarAction.Exit, true));
}
[Fact]
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
{
var runningMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
[Profile("default")]);
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
[Profile("default")]);
Assert.Contains(runningMenu.Items, item =>
item.Action == MeetingTaskbarAction.PauseTranscription &&
item.Text == "Pause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
item.Text == "Unpause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.StopRecording &&
item.Text == "Finish meeting");
}
[Fact]
public void ProcessingMenuShowsSummarizingButAllowsStartingNewRecordings()
{
@@ -223,8 +200,7 @@ public sealed class TaskbarIconTests
private static RecordingStatus Status(
bool isRecording = false,
RecordingProcessState state = RecordingProcessState.Idle,
string? profile = null,
bool isPaused = false)
string? profile = null)
{
return new RecordingStatus(
isRecording,
@@ -233,8 +209,7 @@ public sealed class TaskbarIconTests
state == RecordingProcessState.Idle ? null : "context.md",
state == RecordingProcessState.Idle ? null : "summary.md",
state,
profile,
isPaused);
profile);
}
[SupportedOSPlatform("windows")]
@@ -1,249 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class VenvResemblyzerVoiceEncoderTests
{
[Fact]
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
{
using var fixture = new EncoderFixture();
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
Assert.DoesNotContain(
fixture.Runner.Commands,
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
Assert.Contains(
fixture.Runner.Commands,
command => command.FileName == "python-test"
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("torch==2.14.0+cpu")
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("Resemblyzer==0.1.4")
&& command.Arguments.Contains("--no-deps"));
}
[Fact]
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
{
using var fixture = new EncoderFixture(
warmupExitCode: 21,
warmupError: "incompatible environment");
await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
Assert.Empty(Directory.EnumerateFiles(
fixture.RuntimeFolder,
".ready",
SearchOption.AllDirectories));
}
[Fact]
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
{
var first = new float[256];
first[0] = 2;
var second = new float[256];
second[1] = 3;
using var fixture = new EncoderFixture(VectorOutput([first, second]));
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
Assert.Equal(2, vectors.Count);
Assert.Equal(1f, vectors[0][0], 5);
Assert.Equal(1f, vectors[1][1], 5);
var encodingCommand = Assert.Single(
fixture.Runner.Commands,
command => command.Arguments.Any(
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
Assert.EndsWith(
Path.Combine("Scripts", "python.exe"),
encodingCommand.FileName,
StringComparison.OrdinalIgnoreCase);
Assert.Equal(2, encodingCommand.Arguments.Count);
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
}
[Fact]
public async Task EncodeAsyncRejectsWrongVectorDimension()
{
var vector = new float[255];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("255 dimensions", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsMalformedVectorJson()
{
using var fixture = new EncoderFixture(
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsNonFiniteVector()
{
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
using var fixture = new EncoderFixture(
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsWrongResultCount()
{
var vector = new float[256];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
{
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("zero magnitude", error.Message);
}
[Fact]
public async Task EncodeAsyncReportsFailedLocalCommand()
{
using var fixture = new EncoderFixture("", 17, "runtime failed");
var error = await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("exit code 17", error.Message);
Assert.Contains("runtime failed", error.Message);
}
private static string VectorOutput(IReadOnlyList<float[]> vectors)
{
return $"""
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
{JsonSerializer.Serialize(vectors)}
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
""";
}
private sealed class EncoderFixture : IDisposable
{
public EncoderFixture(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
RuntimeFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"resemblyzer");
Runner = new CapturingCommandRunner(
encodingOutput,
encodingExitCode,
encodingError,
warmupExitCode,
warmupError);
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
Encoder = new VenvResemblyzerVoiceEncoder(
Runner,
Options.Create(options),
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
}
public CapturingCommandRunner Runner { get; }
public VenvResemblyzerVoiceEncoder Encoder { get; }
public string RuntimeFolder { get; }
public void Dispose()
{
if (Directory.Exists(RuntimeFolder))
{
Directory.Delete(RuntimeFolder, recursive: true);
}
}
}
private sealed class CapturingCommandRunner : ICommandRunner
{
private readonly string encodingOutput;
private readonly int encodingExitCode;
private readonly string encodingError;
private readonly int warmupExitCode;
private readonly string warmupError;
public CapturingCommandRunner(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
this.encodingOutput = encodingOutput;
this.encodingExitCode = encodingExitCode;
this.encodingError = encodingError;
this.warmupExitCode = warmupExitCode;
this.warmupError = warmupError;
}
public List<CapturedCommand> Commands { get; } = [];
public Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
if (isEncoding)
{
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
}
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
return Task.FromResult(isWarmup
? new CommandResult(warmupExitCode, string.Empty, warmupError)
: new CommandResult(0, string.Empty, string.Empty));
}
}
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
}
@@ -809,32 +809,6 @@ public sealed class WorkflowRulesEditorTests
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
[Fact]
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
{
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
{
SpeakerIdentityId = identity.Id,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-1",
CreatedAt = DateTimeOffset.UtcNow
});
await fixture.Context.SaveChangesAsync();
var tools = fixture.CreateTools();
var searchResult = await tools.SearchIdentities("Sabrina");
var readResult = await tools.ReadIdentity(identity.Id);
Assert.Contains("\"sampleCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", readResult);
Assert.DoesNotContain("vector-1", readResult);
}
[Fact]
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
{
@@ -843,7 +817,7 @@ public sealed class WorkflowRulesEditorTests
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
@@ -940,30 +914,6 @@ public sealed class WorkflowRulesEditorTests
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
}
[Fact]
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
{
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
{
Draft = "My unfinished draft"
};
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
var task = requested.SendPromptAsync(prompt);
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(requested.IsThinking);
Assert.Empty(pipeline.LastConversation);
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
Assert.Equal("My unfinished draft", existing.Draft);
Assert.Empty(existing.Messages);
pipeline.Release.SetResult();
await task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(requested.IsThinking);
}
[Fact]
public async Task ViewModelShowsAgentErrorWhenRequestTimesOut()
{
@@ -32,14 +32,7 @@ Calendar-Feature is refined, Golem feature needs more work.
PROJECT.md should use this structure:
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
```markdown
---
name: Project display name
description: Concise project purpose and distinguishing context.
---
# Executive Summary
<Elevator Pitch>
@@ -1,5 +1,4 @@
using Microsoft.Extensions.Configuration;
using MeetingAssistant.Speakers;
namespace MeetingAssistant.LaunchProfiles;
@@ -49,7 +48,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
var options = BindDefaultOptions();
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
ValidateSpeakerSampleDurations(options, DefaultProfileName);
return new LaunchProfile(DefaultProfileName, options);
}
@@ -61,7 +59,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
profileSection.Bind(options);
ApplyArrayOverrides(profileSection, options);
ValidateSpeakerSampleDurations(options, profileName);
return new LaunchProfile(profileName, options);
}
@@ -152,18 +149,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
: name.Trim();
}
private static void ValidateSpeakerSampleDurations(
MeetingAssistantOptions options,
string profileName)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
options.SpeakerIdentification,
$"Launch profile '{profileName}' has invalid speaker sample durations");
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
options.SpeakerIdentification.Resemblyzer,
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
}
private static void ApplyArrayOverrides(
IConfigurationSection profileSection,
MeetingAssistantOptions options)
+6 -6
View File
@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PreserveCompilationContext>true</PreserveCompilationContext>
<MicrosoftSpeechVersion>1.52.0</MicrosoftSpeechVersion>
<MicrosoftSpeechVersion>1.51.1</MicrosoftSpeechVersion>
</PropertyGroup>
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
@@ -20,16 +20,16 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.17.0" />
<PackageReference Include="DiffPlex" Version="1.9.0" />
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
<PackageReference Include="NAudio" Version="3.1.0" />
<PackageReference Include="NCalcSync" Version="7.2.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="NCalcSync" Version="7.0.2" />
<PackageReference Include="RazorLight" Version="2.3.1" />
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
<PackageReference Include="System.Drawing.Common" Version="10.0.10" />
<PackageReference Include="Whisper.net" Version="1.9.1" />
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
+7 -52
View File
@@ -165,8 +165,6 @@ public sealed class RecordingInactivitySafeguardOptions
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
@@ -183,8 +181,10 @@ public sealed class WhisperLocalOptions
public PyannoteDiarizationOptions Diarization { get; set; } = new();
}
public class PyannoteRuntimeOptions
public sealed class PyannoteDiarizationOptions
{
public bool Enabled { get; set; } = true;
public string DockerCommand { get; set; } = "docker";
public string BaseImage { get; set; } = "python:3.11-slim";
@@ -208,11 +208,6 @@ public class PyannoteRuntimeOptions
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
}
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
{
public bool Enabled { get; set; } = true;
}
public sealed class AzureSpeechOptions
{
public string Endpoint { get; set; } = "";
@@ -345,9 +340,9 @@ public sealed class SpeakerIdentificationOptions
public int MaxSnippetsPerSpeaker { get; set; } = 3;
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
@@ -360,47 +355,6 @@ public sealed class SpeakerIdentificationOptions
public AzureSpeechOptions AzureSpeech { get; set; } = new();
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
}
public sealed class ResemblyzerSpeakerRecognitionOptions
{
public bool Enabled { get; set; }
public int RequiredVectorsPerSpeaker { get; set; } = 5;
public int MaxVectorsPerIdentity { get; set; } = 1000;
public int OutlierPruningMinimumVectors { get; set; } = 20;
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
public double MinimumClusterCohesion { get; set; } = 0.75;
public double MinimumIdentitySimilarity { get; set; } = 0.75;
public double MinimumSimilarityMargin { get; set; } = 0.05;
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
public string PackageVersion { get; set; } = "0.1.4";
public string PythonCommand { get; set; } = "python";
public string TorchVersion { get; set; } = "2.14.0+cpu";
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
public string WebRtcVadVersion { get; set; } = "2.0.14";
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
}
public sealed class SpeakerIdentityPyannoteValidationOptions
@@ -411,8 +365,9 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
public PyannoteRuntimeOptions Diarization { get; set; } = new()
public PyannoteDiarizationOptions Diarization { get; set; } = new()
{
Enabled = true,
CommandTimeout = TimeSpan.FromHours(1),
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
};
@@ -1,56 +0,0 @@
namespace MeetingAssistant.Notifications;
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
public interface IAgentApprovalPromptService
{
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
}
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
{
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
}
public sealed class AgentApprovalService
{
private readonly IAgentApprovalPromptService prompts;
private readonly ILogger<AgentApprovalService> logger;
private readonly TimeSpan timeout;
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
{
this.prompts = prompts;
this.logger = logger;
this.timeout = timeout;
}
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
deadline.CancelAfter(timeout);
try
{
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
.WaitAsync(deadline.Token);
cancellationToken.ThrowIfCancellationRequested();
return accepted && !deadline.IsCancellationRequested;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
return false;
}
finally
{
await deadline.CancelAsync();
}
}
}
@@ -1,85 +0,0 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Notifications;
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
: IAgentApprovalPromptService, IDisposable
{
private const string Source = "summary-agent-approval";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly object gate = new();
private bool registered;
private bool disposed;
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (disposed) return false;
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source).AddArgument("requestId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText(prompt.Title)
.AddText(prompt.Message)
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
});
}
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("requestId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
disposed = true;
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
+3 -47
View File
@@ -4,7 +4,6 @@ using MeetingAssistant.Hotkeys;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Logging;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Recording;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers;
@@ -17,10 +16,7 @@ using Microsoft.Extensions.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
builder.Services.AddOptions<MeetingAssistantOptions>()
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
.ValidateOnStart();
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
#if WINDOWS
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
@@ -29,9 +25,6 @@ builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
@@ -43,8 +36,6 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
#else
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
#endif
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
@@ -100,35 +91,8 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
builder.Services.AddSingleton<SpeakerIdentityService>();
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
builder.Services.AddSingleton(services =>
{
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer;
return resemblyzer.Enabled
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
: SpeakerSampleCollectionPolicy.ExistingBackend;
});
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
: services.GetRequiredService<SpeakerIdentityService>());
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
: services.GetRequiredService<SpeakerIdentityMergeService>());
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
@@ -138,11 +102,7 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
#if WINDOWS
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
#else
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
#endif
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
#if WINDOWS
@@ -152,9 +112,6 @@ builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRule
#else
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
#endif
builder.Services.AddSingleton<AgentApprovalService>();
builder.Services.AddSingleton<WorkflowChangeRequestService>();
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
builder.Services.AddSingleton<AsrDiagnosticService>();
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
@@ -177,7 +134,6 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
#if WINDOWS
builder.Services.AddHostedService<GlobalHotkeyService>();
builder.Services.AddHostedService<UnoTaskbarIconService>();
@@ -6,8 +6,6 @@ public interface IMeetingInactivityPromptService
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken);
Task DismissAllAsync(CancellationToken cancellationToken);
}
public sealed record MeetingInactivityPromptRequest(
@@ -20,32 +18,9 @@ public enum MeetingInactivityPromptResponse
{
Dismissed,
Continue,
Pause,
Stop
}
internal sealed record MeetingInactivityPromptAction(
string Content,
string ResponseArgument,
MeetingInactivityPromptResponse Response);
internal static class MeetingInactivityPromptActions
{
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
[
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
new("No", "continue", MeetingInactivityPromptResponse.Continue),
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
];
public static MeetingInactivityPromptResponse ParseResponse(string? value)
{
return All.FirstOrDefault(action =>
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
?.Response ?? MeetingInactivityPromptResponse.Continue;
}
}
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
{
public Task ShowStopPromptAsync(
@@ -55,9 +30,4 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
{
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
@@ -2,7 +2,5 @@ namespace MeetingAssistant.Recording;
public interface IMicrophoneCaptureSourceFactory
{
// The caller owns the returned source and disposes it when it implements IDisposable,
// including when capture is canceled before enumeration starts.
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
}
@@ -1,12 +0,0 @@
namespace MeetingAssistant.Recording;
public interface IMicrophoneSwitchPromptService
{
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
}
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
{
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
=> Task.FromResult(false);
}
@@ -33,17 +33,6 @@ public interface ITranscriptStore
string line,
CancellationToken cancellationToken);
async Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
foreach (var line in lines)
{
await AppendLineAsync(session, line, cancellationToken);
}
}
Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -34,7 +34,6 @@ public sealed class MeetingRecordingCoordinator
private readonly IMeetingInactivityClock inactivityClock;
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
private readonly MeetingAssistantOptions options;
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private RecordingRun? currentRun;
@@ -64,8 +63,7 @@ public sealed class MeetingRecordingCoordinator
IMeetingRunArtifactCleaner? artifactCleaner = null,
IMeetingInactivityPromptService? inactivityPromptService = null,
IMeetingInactivityClock? inactivityClock = null,
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -88,8 +86,6 @@ public sealed class MeetingRecordingCoordinator
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
this.options = options.Value;
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
?? SpeakerSampleCollectionPolicy.ExistingBackend;
this.logger = logger;
}
@@ -100,8 +96,7 @@ public sealed class MeetingRecordingCoordinator
currentArtifacts?.AssistantContextPath,
currentArtifacts?.SummaryPath,
GetProcessState(currentRun),
currentRun?.LaunchProfileName,
IsRecording && currentRun?.IsTranscriptionPaused == true);
currentRun?.LaunchProfileName);
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
@@ -294,12 +289,10 @@ public sealed class MeetingRecordingCoordinator
startedAt,
launchProfile.Name,
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
logger);
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
if (runOptions.Recording.InactivitySafeguard.Enabled)
{
_ = Task.Run(
() => RunInactivitySafeguardAsync(run),
@@ -341,14 +334,11 @@ public sealed class MeetingRecordingCoordinator
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return await StopAsync(null, null, null, null, cancellationToken);
return await StopAsync(null, cancellationToken);
}
private async Task<RecordingStatus> StopAsync(
DateTimeOffset? inferredEndTime,
RecordingRun? expectedRun,
long? expectedActivityVersion,
DateTimeOffset? expectedPauseStartedAt,
CancellationToken cancellationToken)
{
RecordingRun run;
@@ -357,30 +347,13 @@ public sealed class MeetingRecordingCoordinator
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is null ||
currentRun.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
if (currentRun is null || currentRun.IsCaptureStopping)
{
return CurrentStatus;
}
run = currentRun;
var inactivityStopClaimed = expectedActivityVersion is not null;
if (expectedActivityVersion is not null &&
(inferredEndTime is null ||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
{
return CurrentStatus;
}
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
if (expectedPauseStartedAt is not null &&
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
{
return CurrentStatus;
}
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
if (inferredEndTime is not null)
{
run.SetInferredEndTime(inferredEndTime.Value);
}
@@ -390,7 +363,7 @@ public sealed class MeetingRecordingCoordinator
{
run.Abort();
}
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
else
{
run.StopCapture();
}
@@ -416,12 +389,6 @@ public sealed class MeetingRecordingCoordinator
}
catch (TimeoutException)
{
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
{
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
return CurrentStatus;
}
if (IsAzureSpeechRun(run))
{
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
@@ -442,65 +409,6 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
CancellationToken cancellationToken)
{
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
}
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
RecordingRun? expectedRun,
long? expectedActivityVersion,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
var run = currentRun;
if (run is null ||
run.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
{
return CurrentStatus;
}
var changed = false;
await run.PipelineGate.WaitAsync(cancellationToken);
try
{
var changedAt = inactivityClock.Now;
changed = run.TrySetTranscriptionPaused(
isPaused,
changedAt,
expectedActivityVersion);
}
finally
{
run.PipelineGate.Release();
}
if (changed && isPaused)
{
await DismissInactivityPromptsAsync(cancellationToken);
}
if (changed)
{
logger.LogInformation(
"Meeting transcription {PauseState}",
isPaused ? "paused" : "unpaused");
}
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
{
RecordingRun run;
@@ -753,7 +661,8 @@ public sealed class MeetingRecordingCoordinator
chunk.Channels);
}
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
}
logger.LogInformation(
@@ -792,6 +701,7 @@ public sealed class MeetingRecordingCoordinator
run.ResetSpeakerIdentification();
}
run.RecordTranscriptActivity(segment, inactivityClock.Now);
var sample = run.TryAddSpeakerSample(segment);
if (sample is not null)
{
@@ -808,56 +718,6 @@ public sealed class MeetingRecordingCoordinator
var relabeledSegment = run.Relabel(segment);
run.AddLiveSegment(relabeledSegment);
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
run.RecordTranscriptActivity(segment, inactivityClock.Now);
if (!string.IsNullOrWhiteSpace(segment.Text))
{
// Switching profiles holds the coordinator gate while draining this reader.
// Notification cleanup must not make transcript draining wait for that gate.
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
}
}
}
private async Task DismissInactivityPromptsForRunAsync(
RecordingRun run,
CancellationToken cancellationToken)
{
try
{
await gate.WaitAsync(cancellationToken);
try
{
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
{
return;
}
await DismissInactivityPromptsAsync(cancellationToken);
}
finally
{
gate.Release();
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Pending cleanup is no longer needed after this run stops capturing.
}
}
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
{
try
{
await inactivityPromptService.DismissAllAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
}
}
@@ -985,10 +845,7 @@ public sealed class MeetingRecordingCoordinator
private async Task RunInactivitySafeguardAsync(RecordingRun run)
{
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
var promptThresholds = safeguardOptions.Enabled
? GetInactivityPromptThresholds(safeguardOptions)
: [];
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
var promptedThresholds = new HashSet<TimeSpan>();
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
@@ -996,7 +853,7 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
string.Join(", ", promptThresholds),
safeguardOptions.AutoStopAfter,
checkInterval);
safeguardOptions.CheckInterval);
try
{
@@ -1012,38 +869,9 @@ public sealed class MeetingRecordingCoordinator
snapshot.LastTranscriptActivityAt);
}
var pauseStartedAt = snapshot.TranscriptionPausedAt;
if (pauseStartedAt is not null)
{
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
if (maximumPauseDuration > TimeSpan.Zero &&
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
{
logger.LogWarning(
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
maximumPauseDuration);
var stopStatus = await StopAsync(
null,
run,
null,
pauseStartedAt,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
}
await inactivityClock.DelayAsync(
checkInterval,
run.CaptureCancellation);
continue;
}
var now = inactivityClock.Now;
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
if (safeguardOptions.Enabled &&
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
inactivityDuration >= safeguardOptions.AutoStopAfter)
{
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
@@ -1051,16 +879,8 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
inactivityDuration,
inferredEndTime);
var stopStatus = await StopAsync(
inferredEndTime,
run,
snapshot.ActivityVersion,
null,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
await StopAsync(inferredEndTime, CancellationToken.None);
return;
}
foreach (var threshold in promptThresholds)
@@ -1077,7 +897,6 @@ public sealed class MeetingRecordingCoordinator
inactivityDuration,
threshold,
inferredEndTime);
var promptActivityVersion = snapshot.ActivityVersion;
await inactivityPromptService.ShowStopPromptAsync(
new MeetingInactivityPromptRequest(
inactivityDuration,
@@ -1093,36 +912,16 @@ public sealed class MeetingRecordingCoordinator
if (response == MeetingInactivityPromptResponse.Stop)
{
await StopAsync(
inferredEndTime,
run,
promptActivityVersion,
null,
CancellationToken.None);
}
else if (response == MeetingInactivityPromptResponse.Pause)
{
await SetTranscriptionPausedAsync(
true,
run,
promptActivityVersion,
CancellationToken.None);
await StopAsync(inferredEndTime, CancellationToken.None);
}
},
run.CaptureCancellation);
var currentActivity = run.GetTranscriptActivitySnapshot();
if (currentActivity.IsTranscriptionPaused ||
currentActivity.ActivityVersion != promptActivityVersion)
{
await DismissInactivityPromptsAsync(run.CaptureCancellation);
}
break;
}
await inactivityClock.DelayAsync(
checkInterval,
GetInactivitySafeguardCheckInterval(safeguardOptions),
run.CaptureCancellation);
}
}
@@ -1190,11 +989,11 @@ public sealed class MeetingRecordingCoordinator
try
{
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
if (checkpoint is null)
{
logger.LogInformation(
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
return;
}
@@ -2022,11 +1821,6 @@ public sealed class MeetingRecordingCoordinator
.ToList();
}
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
{
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
}
private static TimeSpan GetInactivitySafeguardCheckInterval(
RecordingInactivitySafeguardOptions options)
{
@@ -2083,8 +1877,6 @@ public sealed class MeetingRecordingCoordinator
private TimeSpan? lastTranscriptSegmentEnd;
private long transcriptActivityVersion;
private DateTimeOffset? inferredEndTime;
private bool isTranscriptionPaused;
private DateTimeOffset? transcriptionPausedAt;
public RecordingRun(
CancellationTokenSource captureCancellation,
@@ -2100,7 +1892,6 @@ public sealed class MeetingRecordingCoordinator
string launchProfileName,
TimeSpan liveSampleBufferDuration,
int maxSpeakerSamples,
bool requireNonOverlappingSpeakerSamples,
ILogger logger)
{
CaptureCancellationSource = captureCancellation;
@@ -2121,9 +1912,8 @@ public sealed class MeetingRecordingCoordinator
liveSampleBufferDuration,
maxSpeakerSamples,
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleDuration,
logger,
requireNonOverlappingSpeakerSamples);
options.SpeakerIdentification.MaximumSampleSegmentGap,
logger);
}
public CancellationTokenSource CaptureCancellationSource { get; }
@@ -2170,17 +1960,6 @@ public sealed class MeetingRecordingCoordinator
public bool HasAttachedPromptMetadata { get; private set; }
public bool IsTranscriptionPaused
{
get
{
lock (transcriptActivityGate)
{
return isTranscriptionPaused;
}
}
}
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public DateTimeOffset? InferredEndTime
@@ -2204,37 +1983,6 @@ public sealed class MeetingRecordingCoordinator
HasAttachedPromptMetadata = true;
}
public bool TrySetTranscriptionPaused(
bool isPaused,
DateTimeOffset changedAt,
long? expectedActivityVersion)
{
lock (transcriptActivityGate)
{
if (expectedActivityVersion is not null &&
transcriptActivityVersion != expectedActivityVersion.Value)
{
return false;
}
if (isTranscriptionPaused == isPaused)
{
return false;
}
isTranscriptionPaused = isPaused;
transcriptionPausedAt = isPaused ? changedAt : null;
if (!isPaused)
{
lastTranscriptActivityAt = changedAt;
}
transcriptActivityVersion++;
return true;
}
}
public void Abort()
{
IsAborted = true;
@@ -2263,17 +2011,13 @@ public sealed class MeetingRecordingCoordinator
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
}
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
await PipelineGate.WaitAsync(cancellationToken);
try
{
var routedChunk = IsTranscriptionPaused
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
: chunk;
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
AppendAudio(routedChunk);
await Pipeline.WriteAsync(routedChunk, cancellationToken);
AppendAudio(chunk);
await Pipeline.WriteAsync(chunk, cancellationToken);
}
finally
{
@@ -2457,9 +2201,7 @@ public sealed class MeetingRecordingCoordinator
return new TranscriptActivitySnapshot(
lastTranscriptActivityAt,
lastTranscriptSegmentEnd,
transcriptActivityVersion,
isTranscriptionPaused,
transcriptionPausedAt);
transcriptActivityVersion);
}
}
@@ -2479,35 +2221,6 @@ public sealed class MeetingRecordingCoordinator
}
}
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
{
lock (transcriptActivityGate)
{
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
{
return false;
}
inferredEndTime = endTime;
StopCapture();
return true;
}
}
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
{
lock (transcriptActivityGate)
{
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
{
return false;
}
StopCapture();
return true;
}
}
public void AppendAudio(AudioChunk chunk)
{
speakerSampleCollector.AppendAudio(chunk);
@@ -2531,11 +2244,9 @@ public sealed class MeetingRecordingCoordinator
return speakerSampleCollector.Snapshot();
}
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
MeetingNote meetingNote,
IReadOnlyList<SpeakerAudioSample> samples)
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
{
var speakers = GetUnmappedSampleSpeakers(samples);
var speakers = GetUnmappedSampleSpeakers();
if (speakers.Count == 0)
{
return null;
@@ -2543,11 +2254,7 @@ public sealed class MeetingRecordingCoordinator
var checkpoint = new LiveIdentificationCheckpoint(
speakers,
BuildAttendeeSignature(meetingNote),
Options.SpeakerIdentification.Resemblyzer.Enabled
? speakers.Select(speaker => samples.Count(sample =>
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
: []);
BuildAttendeeSignature(meetingNote));
lock (liveIdentificationGate)
{
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
@@ -2605,9 +2312,9 @@ public sealed class MeetingRecordingCoordinator
StringComparer.OrdinalIgnoreCase);
}
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
{
return samples
return GetSpeakerSamplesSnapshot()
.Select(sample => sample.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
@@ -2643,9 +2350,7 @@ public sealed class MeetingRecordingCoordinator
public sealed record TranscriptActivitySnapshot(
DateTimeOffset LastTranscriptActivityAt,
TimeSpan? LastTranscriptSegmentEnd,
long ActivityVersion,
bool IsTranscriptionPaused,
DateTimeOffset? TranscriptionPausedAt);
long ActivityVersion);
private static int StateRank(AssistantContextState state)
{
@@ -2663,13 +2368,11 @@ public sealed class MeetingRecordingCoordinator
public sealed record LiveIdentificationCheckpoint(
IReadOnlyList<string> Speakers,
string AttendeeSignature,
IReadOnlyList<int> SampleCounts)
string AttendeeSignature)
{
public bool Matches(LiveIdentificationCheckpoint other)
{
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
SampleCounts.SequenceEqual(other.SampleCounts) &&
Speakers.Count == other.Speakers.Count &&
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
}
@@ -2702,8 +2405,7 @@ public sealed record RecordingStatus(
string? AssistantContextPath,
string? SummaryPath,
RecordingProcessState State = RecordingProcessState.Idle,
string? LaunchProfile = null,
bool IsPaused = false);
string? LaunchProfile = null);
public enum RecordingProcessState
{
@@ -8,30 +8,22 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
private readonly IMicrophoneCaptureSourceFactory captureSources;
private readonly ILogger<MicrophoneAudioSource> logger;
private readonly TimeSpan recoveryDelay;
private readonly MicrophoneSwitchMonitor? switchMonitor;
private readonly MicrophoneDeviceSelection? selection;
public MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
ILogger<MicrophoneAudioSource> logger)
: this(captureSources, logger, DefaultRecoveryDelay)
{
}
internal MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
TimeSpan recoveryDelay,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
TimeSpan recoveryDelay)
{
this.captureSources = captureSources;
this.logger = logger;
this.recoveryDelay = recoveryDelay;
this.switchMonitor = switchMonitor;
this.selection = selection;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
@@ -42,44 +34,21 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
MeetingAssistantOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
using var session = new CaptureSession(selection, cancellationToken);
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
try
{
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
{
yield return chunk;
}
}
finally
{
session.Stop();
await monitoring;
}
}
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
MeetingAssistantOptions options,
CaptureSession session,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var failedAttempts = 0;
while (!cancellationToken.IsCancellationRequested)
{
var captureToken = session.BeginCapture();
IMeetingAudioSource? captureSource = null;
IAsyncEnumerator<AudioChunk>? capture = null;
Exception? failure = null;
try
{
captureSource = captureSources.CreateCapture(options);
capture = captureSource
.CaptureAsync(options, captureToken)
.GetAsyncEnumerator(captureToken);
capture = captureSources
.CreateCapture(options)
.CaptureAsync(options, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -87,7 +56,6 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
failure = exception;
}
using var captureOwner = captureSource as IDisposable;
if (cancellationToken.IsCancellationRequested)
{
yield break;
@@ -97,14 +65,14 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
try
{
while (!captureToken.IsCancellationRequested)
while (!cancellationToken.IsCancellationRequested)
{
var hasNext = false;
try
{
hasNext = await capture.MoveNextAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -112,7 +80,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
failure = exception;
}
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
if (cancellationToken.IsCancellationRequested || failure is not null || !hasNext)
{
break;
}
@@ -134,7 +102,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
await capture.DisposeAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -144,19 +112,11 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
}
}
var switchRequested = captureToken.IsCancellationRequested;
session.EndCapture();
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
if (switchRequested)
{
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
continue;
}
failedAttempts++;
logger.LogWarning(
failure,
@@ -182,53 +142,4 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
return false;
}
}
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
{
private readonly object gate = new();
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
private CancellationTokenSource? capture;
public CancellationToken Token => lifetime.Token;
public CancellationToken BeginCapture()
{
lock (gate)
{
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
return capture.Token;
}
}
public void EndCapture()
{
lock (gate)
{
capture?.Dispose();
capture = null;
}
}
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
{
lock (gate)
{
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
selection.Select(device.Id);
capture?.Cancel();
}
}
public void Stop()
{
lock (gate) lifetime.Cancel();
}
public void Dispose()
{
Stop();
EndCapture();
lifetime.Dispose();
}
}
}
@@ -1,114 +0,0 @@
namespace MeetingAssistant.Recording;
public sealed class MicrophoneSwitchMonitor
{
private readonly IMicrophoneDeviceProvider devices;
private readonly IMicrophoneSwitchPromptService prompts;
private readonly ILogger<MicrophoneSwitchMonitor> logger;
private readonly TimeSpan pollInterval;
private readonly TimeSpan promptLifetime;
public MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger)
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
{
}
internal MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger,
TimeSpan pollInterval,
TimeSpan promptLifetime)
{
this.devices = devices;
this.prompts = prompts;
this.logger = logger;
this.pollInterval = pollInterval;
this.promptLifetime = promptLifetime;
}
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
{
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
try
{
while (true)
{
await Task.Delay(pollInterval, cancellationToken);
var available = ReadDevices();
if (available is null) continue;
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var (id, prompt) in pending.ToArray())
{
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
if (prompt.Task.IsCompleted)
{
prompt.Cancellation.Dispose();
pending.Remove(id);
}
}
foreach (var device in available)
{
if (known.Add(device.Id))
{
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
promptCancellation.CancelAfter(promptLifetime);
pending.Add(device.Id, new PendingPrompt(promptCancellation,
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
}
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
finally
{
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
}
}
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
CancellationToken cancellationToken)
{
try
{
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
&& !cancellationToken.IsCancellationRequested
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
&& !cancellationToken.IsCancellationRequested)
{
switchMicrophone(device, cancellationToken);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
}
}
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
{
try
{
return devices.GetAvailableMicrophones();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not check for newly available microphones");
return null;
}
}
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
}
@@ -4,12 +4,11 @@ using NAudio.Wave;
namespace MeetingAssistant.Recording;
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource
{
private readonly IWaveIn capture;
private readonly string sourceName;
private readonly ILogger logger;
private int disposed;
public NaudioCaptureAudioSource(
IWaveIn capture,
@@ -23,15 +22,13 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposabl
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureWith(cancellationToken);
return CaptureWith(capture, sourceName, logger, cancellationToken);
}
public void Dispose()
{
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
}
private async IAsyncEnumerable<AudioChunk> CaptureWith(
private static async IAsyncEnumerable<AudioChunk> CaptureWith(
IWaveIn capture,
string sourceName,
ILogger logger,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded<AudioChunk>();
@@ -86,7 +83,7 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposabl
}
finally
{
Dispose();
capture.Dispose();
}
}
}
@@ -117,25 +117,8 @@ public sealed class OfflineTranscriptionBacklogProcessor
pipelineOptions,
cancellationToken);
if (!finishedSegments.Any(segment =>
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
{
throw new InvalidOperationException(
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
}
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
await transcriptStore.AppendLinesAsync(session,
[
"",
"## Offline transcription recovery",
"",
"> The following transcript was recovered by replaying the entire recording. "
+ "It may duplicate earlier transcript passages. "
+ "Treat repeated passages as the same discussion when summarizing.",
"",
.. lines
], cancellationToken);
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
@@ -9,31 +9,37 @@ internal sealed class SpeakerAudioSampleCollector
private readonly object gate = new();
private readonly RollingAudioBuffer audioBuffer;
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly int maxSamplesPerSpeaker;
private readonly TimeSpan minimumSampleSpeechDuration;
private readonly TimeSpan maximumSampleDuration;
private readonly bool requireNonOverlappingSamples;
private readonly TimeSpan minimumUninterruptedSpeechDuration;
private readonly TimeSpan maximumSegmentGap;
private readonly ILogger? logger;
private SpeakerSampleSpan? pendingSpan;
private PendingSpeakerSpan? pendingSpan;
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
: this(
bufferDuration,
maxSamplesPerSpeaker,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(1),
logger: null)
{
}
public SpeakerAudioSampleCollector(
TimeSpan bufferDuration,
int maxSamplesPerSpeaker,
TimeSpan minimumSampleSpeechDuration,
TimeSpan maximumSampleDuration,
ILogger? logger = null,
bool requireNonOverlappingSamples = false)
TimeSpan minimumUninterruptedSpeechDuration,
TimeSpan maximumSegmentGap,
ILogger? logger = null)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
minimumSampleSpeechDuration,
maximumSampleDuration,
"Speaker sample collector configuration is invalid");
audioBuffer = new RollingAudioBuffer(bufferDuration);
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
this.maximumSampleDuration = maximumSampleDuration;
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
? minimumUninterruptedSpeechDuration
: TimeSpan.Zero;
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
? maximumSegmentGap
: TimeSpan.Zero;
this.logger = logger;
}
@@ -47,7 +53,6 @@ internal sealed class SpeakerAudioSampleCollector
lock (gate)
{
samplesBySpeaker.Clear();
lastAcceptedEndBySpeaker.Clear();
pendingSpan = null;
audioBuffer.Reset();
}
@@ -63,54 +68,34 @@ internal sealed class SpeakerAudioSampleCollector
return null;
}
SpeakerSampleSpan? sampleSpan;
SpeakerSampleSpan? previousSpan;
TranscriptionSegment sampleSegment;
PendingSpanReset? reset;
lock (gate)
{
if (requireNonOverlappingSamples &&
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
segment.Start < lastAcceptedEnd)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
segment.Speaker,
lastAcceptedEnd);
return null;
}
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
(sampleSegment, reset) = ExtendPendingSpan(segment);
}
if (sampleSpan is null)
if (reset is not null)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
segment.Speaker);
return null;
}
var sampleSegment = sampleSpan.ToSegment();
if (previousSpan is not null)
{
logger?.LogInformation(
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
previousSpan.Speaker,
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
reset.PreviousSpeaker,
segment.Speaker,
previousSpan.End,
segment.Start);
reset.PreviousEnd,
segment.Start,
reset.Gap,
maximumSegmentGap);
}
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
if (!score.Accepted)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
sampleSegment.Speaker,
score.Reason,
sampleSpan.SpeechDuration,
sampleSegment.End - sampleSegment.Start,
minimumSampleSpeechDuration,
minimumUninterruptedSpeechDuration,
score.WordCount);
return null;
}
@@ -130,13 +115,6 @@ internal sealed class SpeakerAudioSampleCollector
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
lock (gate)
{
if (requireNonOverlappingSamples &&
ReferenceEquals(pendingSpan, sampleSpan))
{
pendingSpan = null;
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
}
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
{
samples = [];
@@ -185,26 +163,35 @@ internal sealed class SpeakerAudioSampleCollector
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
}
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
{
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
if (pendingSpan is null ||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
{
pendingSpan = extended;
return (pendingSpan, null);
var reset = pendingSpan is null
? null
: new PendingSpanReset(
pendingSpan.Speaker,
pendingSpan.End,
segment.Start - pendingSpan.End);
pendingSpan = new PendingSpeakerSpan(
segment.Speaker,
segment.Start,
segment.End,
[segment.Text]);
return (pendingSpan.ToSegment(), reset);
}
var previousSpan = pendingSpan;
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
return (pendingSpan, previousSpan);
pendingSpan = pendingSpan.Extend(segment);
return (pendingSpan.ToSegment(), null);
}
private static SampleScore Score(
TranscriptionSegment segment,
TimeSpan speechDuration,
TimeSpan minimumSampleSpeechDuration)
TimeSpan minimumUninterruptedSpeechDuration)
{
var durationSeconds = speechDuration.TotalSeconds;
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
{
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
}
@@ -215,7 +202,7 @@ internal sealed class SpeakerAudioSampleCollector
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
}
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
var wordScore = Math.Min(words / 60.0, 1);
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
segment.Text.TrimEnd().EndsWith('?') ||
@@ -234,4 +221,30 @@ internal sealed class SpeakerAudioSampleCollector
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
private sealed record PendingSpeakerSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
IReadOnlyList<string> TextParts)
{
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
{
return this with
{
End = segment.End > End ? segment.End : End,
TextParts = TextParts.Append(segment.Text).ToList()
};
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
}
}
}
@@ -49,25 +49,6 @@ public sealed class VaultTranscriptStore : ITranscriptStore
cancellationToken);
}
public Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
if (lines.Count == 0)
{
return Task.CompletedTask;
}
return WithFileGateAsync(
session.TranscriptPath,
() => File.AppendAllTextAsync(
session.TranscriptPath,
string.Concat(lines.Select(line => line + Environment.NewLine)),
cancellationToken),
cancellationToken);
}
public async Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
private const string NotificationGroup = "meeting-inactivity";
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
private readonly object notificationGate = new();
private readonly object registrationGate = new();
private bool registered;
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
@@ -37,27 +37,24 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
try
{
lock (notificationGate)
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
catch (Exception exception)
{
@@ -79,36 +76,6 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
lock (notificationGate)
{
if (pendingPrompts.IsEmpty)
{
return Task.CompletedTask;
}
pendingPrompts.Clear();
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
{
return Task.CompletedTask;
}
try
{
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
logger.LogInformation("Dismissed all native Windows inactivity notifications");
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
}
return Task.CompletedTask;
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
@@ -116,28 +83,29 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
public void Dispose()
{
lock (notificationGate)
if (!registered)
{
if (!registered)
{
return;
}
return;
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
registered = false;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
}
}
private void EnsureRegistered()
{
lock (notificationGate)
if (registered)
{
return;
}
lock (registrationGate)
{
if (registered)
{
@@ -154,25 +122,29 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
string promptId,
MeetingInactivityPromptRequest request)
{
var builder = new ToastContentBuilder()
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "stop")
.SetBackgroundActivation();
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "continue")
.SetBackgroundActivation();
return new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("Stop meeting?")
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
foreach (var action in MeetingInactivityPromptActions.All)
{
builder.AddButton(new ToastButton()
.SetContent(action.Content)
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", action.ResponseArgument)
.SetBackgroundActivation());
}
return builder;
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
.AddButton(yesButton)
.AddButton(noButton);
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
@@ -205,9 +177,10 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue)
? MeetingInactivityPromptActions.ParseResponse(responseValue)
: MeetingInactivityPromptResponse.Continue;
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
? MeetingInactivityPromptResponse.Stop
: MeetingInactivityPromptResponse.Continue;
_ = Task.Run(async () =>
{
try
@@ -1,96 +0,0 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
namespace MeetingAssistant.Recording;
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
{
private const string Source = "meeting-microphone-switch";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
private readonly object gate = new();
private bool registered;
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
{
this.logger = logger;
}
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source)
.AddArgument("promptId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("New microphone available")
.AddText($"Switch this meeting to {microphone.Name}?")
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
});
}
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try
{
ToastNotificationManagerCompat.History.Remove(id, Source);
}
catch (Exception exception)
{
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
}
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("promptId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
return [];
}
return await finalizer.FinalizeEnabledAsync(
return await finalizer.FinalizeAsync(
wavPath,
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
options.Diarization,
@@ -1,631 +0,0 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerSnippetExtractor snippetExtractor;
private readonly IResemblyzerVoiceEncoder encoder;
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
public ResemblyzerSpeakerIdentificationService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerSnippetExtractor snippetExtractor,
IResemblyzerVoiceEncoder encoder,
ResemblyzerVoiceClusterMatcher clusterMatcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentificationService> logger)
{
this.dbContextFactory = dbContextFactory;
this.snippetExtractor = snippetExtractor;
this.encoder = encoder;
this.clusterMatcher = clusterMatcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
}
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
}
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
}
public async Task ApplySpeakerOverrideAsync(
SpeakerIdentificationRequest request,
string sourceSpeaker,
string targetSpeaker,
CancellationToken cancellationToken)
{
if (!options.Enabled ||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
string.IsNullOrWhiteSpace(targetSpeaker) ||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
{
return;
}
var sourceLabel = sourceSpeaker.Trim();
var targetName = targetSpeaker.Trim();
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var sourceCandidate = identities
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => identity.Id)
.FirstOrDefault();
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
if (target is null && sourceCandidate is null && vectors.Count == 0)
{
logger.LogWarning(
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
sourceLabel,
targetName);
return;
}
var now = DateTimeOffset.UtcNow;
if (target is null)
{
target = sourceCandidate ?? new SpeakerIdentity
{
CreatedAt = now
};
if (target.Id == 0)
{
context.SpeakerIdentities.Add(target);
}
}
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
{
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
sourceCandidate,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
context.SpeakerIdentities.Remove(sourceCandidate);
}
target.CanonicalName = targetName;
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
outlierPruner.AddAndPrune(
target,
vectors,
now);
target.UpdatedAt = now;
await context.SaveChangesAsync(cancellationToken);
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
target.References,
sourceLabel,
targetName,
cancellationToken);
}
public async Task DeleteSpeakerIdentityAsync(
string identity,
CancellationToken cancellationToken)
{
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
{
return;
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities.FirstOrDefault(candidate =>
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
if (target is null)
{
return;
}
context.SpeakerIdentities.Remove(target);
await context.SaveChangesAsync(cancellationToken);
}
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
SpeakerIdentificationRequest request,
bool final,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
if (!options.Enabled || request.Segments.Count == 0)
{
return EmptyResult(request.Segments);
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var knownMappings = request.KnownSpeakerMappings ??
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (final && knownMappings.Count > 0)
{
await PersistMappedSpeakerEvidenceAsync(
context,
request,
knownMappings,
cancellationToken);
}
var identifiedNames = knownMappings.Values
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var segmentSpeaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
{
identifiedNames.Add(segmentSpeaker.Trim());
}
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
var pendingAudits = new List<PendingIdentificationAudit>();
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
{
continue;
}
var vectors = await ResolveAutomaticVectorsAsync(
request,
speaker,
allowAudioFallback,
cancellationToken);
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
{
logger.LogInformation(
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
speaker,
vectors.Count,
resemblyzerOptions.RequiredVectorsPerSpeaker);
continue;
}
var (identity, decision) = await FindMatchAsync(
context,
attendees,
identifiedNames,
vectors,
cancellationToken);
if (identity is null)
{
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
{
unmatchedSpeakers.Add((speaker, vectors));
}
continue;
}
var now = DateTimeOffset.UtcNow;
var previousCanonicalName = identity.CanonicalName;
var previousReferenceCount = identity.References.Count;
outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
if (final)
{
UpdateMatchedIdentity(identity, attendees);
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
}
var displayName = identity.GetDisplayName();
if (!string.IsNullOrWhiteSpace(displayName))
{
mappings[speaker] = displayName;
identifiedNames.Add(displayName);
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
identifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
displayName,
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
}
}
if (final)
{
LearnUnmatchedSpeakers(
context,
request.MeetingNote,
attendees,
matchedAcceptedNames,
unmatchedSpeakers,
pendingAudits);
}
await context.SaveChangesAsync(cancellationToken);
await AppendAuditsAsync(pendingAudits, cancellationToken);
var relabeled = request.Segments
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
? segment with { Speaker = name }
: segment)
.ToList();
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
}
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
SpeakerIdentityDbContext context,
IReadOnlyList<string> attendees,
IReadOnlySet<string> identifiedNames,
IReadOnlyList<float[]> queryVectors,
CancellationToken cancellationToken)
{
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
var identities = await LoadIdentities(context)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var candidates = identities
.Select(identity => new
{
Identity = identity,
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.OrderByDescending(candidate => candidate.IsAttendee)
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
.ThenBy(candidate => candidate.Identity.Id)
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
candidate.Identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
candidate.Identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
var match = clusterMatcher.Match(queryVectors, candidates);
return (
match.IdentityId is { } identityId
? identities.Single(identity => identity.Id == identityId)
: null,
match);
}
private void LearnUnmatchedSpeakers(
SpeakerIdentityDbContext context,
MeetingNote meetingNote,
IReadOnlyList<string> attendees,
IReadOnlySet<string> matchedAcceptedNames,
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
ICollection<PendingIdentificationAudit> pendingAudits)
{
var candidates = attendees
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (candidates.Count == 0)
{
return;
}
foreach (var (speaker, vectors) in unmatchedSpeakers)
{
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates
.Select(name => new SpeakerCandidateName { Name = name })
.ToList(),
References = [CreateReference(meetingNote, now)]
};
outlierPruner.AddAndPrune(
identity,
vectors,
now);
context.SpeakerIdentities.Add(identity);
logger.LogInformation(
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
speaker,
identity.VoiceVectors.Count,
string.Join(", ", candidates));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
}
}
private async Task AppendAuditsAsync(
IEnumerable<PendingIdentificationAudit> pendingAudits,
CancellationToken cancellationToken)
{
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
audit.Identity.References,
audit.Speaker,
audit.Name,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
audit.Identity.Id);
}
}
}
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback,
cancellationToken);
if (samples.Count < requiredCount)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task PersistMappedSpeakerEvidenceAsync(
SpeakerIdentityDbContext context,
SpeakerIdentificationRequest request,
IReadOnlyDictionary<string, string> knownMappings,
CancellationToken cancellationToken)
{
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
foreach (var (speaker, mappedName) in knownMappings)
{
var identity = identities
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
.OrderBy(candidate =>
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(candidate => candidate.Id)
.FirstOrDefault();
if (identity is null)
{
logger.LogWarning(
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
speaker,
mappedName);
continue;
}
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
if (vectors.Count == 0)
{
continue;
}
var now = DateTimeOffset.UtcNow;
var previousReferenceCount = identity.References.Count;
var vectorUpdate = outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
logger.LogInformation(
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
vectorUpdate.AddedCount,
speaker,
identity.Id);
}
}
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
CancellationToken cancellationToken)
{
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback: true,
cancellationToken);
if (samples.Count == 0)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
SpeakerIdentificationRequest request,
string speaker,
int maxSamples,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var suppliedSamples = request.Samples?
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
.Where(sample => sample.WavBytes.Length > 0)
.OrderByDescending(sample => sample.Score)
.Take(maxSamples)
.ToList()
?? [];
var wavSamples = suppliedSamples
.Select(sample => sample.WavBytes)
.ToList();
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
{
return wavSamples;
}
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
request.Segments,
speaker,
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
request.Segments.Count);
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
{
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
span,
cancellationToken);
if (wavBytes.Length > 0)
{
wavSamples.Add(wavBytes);
}
if (wavSamples.Count >= maxSamples)
{
break;
}
}
logger.LogInformation(
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
wavSamples.Count,
maxSamples,
speaker,
suppliedSamples.Count,
wavSamples.Count - suppliedSamples.Count);
return wavSamples;
}
private static bool OverlapsSuppliedSample(
IReadOnlyList<TranscriptionSegment> span,
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
{
if (span.Count == 0)
{
return false;
}
var start = span[0].Start;
var end = span[^1].End;
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
}
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
{
return context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References);
}
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return;
}
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
}
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
{
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
}
private sealed record PendingIdentificationAudit(
SpeakerIdentity Identity,
string Speaker,
string Name);
}
@@ -1,172 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ResemblyzerVoiceClusterMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
public ResemblyzerSpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ResemblyzerVoiceClusterMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
TimeSpan? recentIdentityAge,
CancellationToken cancellationToken)
{
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var recentIds = identities
.Where(identity => identity.CreatedAt >= cutoff)
.Select(identity => identity.Id)
.ToList();
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
var attempts = 0;
var mergedPairs = 0;
var pendingAudits = new List<PendingMergeAudit>();
foreach (var sourceId in recentIds)
{
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
if (source is null)
{
continue;
}
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
source,
resemblyzerOptions.ModelId,
logger)
.Take(required * 2)
.ToList();
if (sourceVectors.Count < required * 2)
{
logger.LogInformation(
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
source.Id,
sourceVectors.Count,
required * 2);
continue;
}
var candidates = identities
.Where(identity => identity.Id != source.Id)
.Where(identity => identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(identity => new ResemblyzerVoiceVectorCandidate(
identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
if (candidates.Count == 0)
{
continue;
}
attempts++;
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
if (!first.Accepted || first.IdentityId is not { } targetId)
{
continue;
}
attempts++;
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
if (!second.Accepted || second.IdentityId != targetId)
{
logger.LogInformation(
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
source.Id,
first.IdentityId,
second.IdentityId);
continue;
}
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
if (target is null)
{
continue;
}
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
source,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
pendingAudits.Add(new PendingMergeAudit(
target,
targetName,
sourceName));
context.SpeakerIdentities.Remove(source);
identities.Remove(source);
mergedPairs++;
}
await context.SaveChangesAsync(cancellationToken);
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
audit.Identity.References,
audit.TargetName,
audit.SourceName,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
audit.Identity.Id);
}
}
return new SpeakerIdentityMergeResult(
recentIds.Count,
identities.Count,
attempts,
mergedPairs);
}
private sealed record PendingMergeAudit(
SpeakerIdentity Identity,
string TargetName,
string SourceName);
}
@@ -1,204 +0,0 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorCandidate(
int IdentityId,
IReadOnlyList<float[]> Vectors);
public sealed record ResemblyzerVoiceClusterMatchResult(
int? IdentityId,
string Reason,
double Cohesion,
double? BestSimilarity,
double? RunnerUpSimilarity)
{
public bool Accepted => IdentityId.HasValue;
}
public sealed class ResemblyzerVoiceClusterMatcher
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
public ResemblyzerVoiceClusterMatcher(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceClusterMatcher> logger)
{
this.options = options;
this.logger = logger;
}
public ResemblyzerVoiceClusterMatchResult Match(
IReadOnlyList<float[]> queryVectors,
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
{
var requiredCount = options.RequiredVectorsPerSpeaker;
if (queryVectors.Count < requiredCount)
{
return Reject(
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
cohesion: 0,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var normalizedQuery = queryVectors
.Take(requiredCount)
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var cohesion = MeanPairwiseCosine(normalizedQuery);
if (cohesion < options.MinimumClusterCohesion)
{
return Reject(
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
cohesion,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var scored = candidates
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
.Where(candidate => candidate is not null)
.Select(candidate => candidate!.Value)
.OrderByDescending(candidate => candidate.Similarity)
.ThenBy(candidate => candidate.IdentityId)
.ToList();
if (scored.Count == 0)
{
return Reject("no compatible identity vectors were available", cohesion, null, null);
}
var best = scored[0];
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
if (best.Similarity < options.MinimumIdentitySimilarity)
{
return Reject(
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
cohesion,
best.Similarity,
runnerUp);
}
if (runnerUp is { } runnerUpSimilarity &&
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
{
return Reject(
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
cohesion,
best.Similarity,
runnerUpSimilarity);
}
logger.LogInformation(
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
best.IdentityId,
cohesion,
best.Similarity,
runnerUp);
return new ResemblyzerVoiceClusterMatchResult(
best.IdentityId,
"accepted",
cohesion,
best.Similarity,
runnerUp);
}
private ResemblyzerVoiceClusterMatchResult Reject(
string reason,
double cohesion,
double? bestSimilarity,
double? runnerUpSimilarity)
{
logger.LogInformation(
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
return new ResemblyzerVoiceClusterMatchResult(
null,
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
}
private (int IdentityId, double Similarity)? ScoreCandidate(
IReadOnlyList<float[]> normalizedQuery,
ResemblyzerVoiceVectorCandidate candidate)
{
if (candidate.Vectors.Count == 0)
{
return null;
}
try
{
var normalizedCandidate = candidate.Vectors
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
var similarities = normalizedQuery
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
.Order()
.ToList();
return (candidate.IdentityId, Median(similarities));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
candidate.IdentityId);
return null;
}
}
private static float[] Centroid(IReadOnlyList<float[]> vectors)
{
var centroid = new float[ResemblyzerVectorContract.Dimensions];
foreach (var vector in vectors)
{
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] += vector[index];
}
}
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] /= vectors.Count;
}
return centroid;
}
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
{
if (vectors.Count < 2)
{
return 1;
}
double total = 0;
var pairs = 0;
for (var first = 0; first < vectors.Count - 1; first++)
{
for (var second = first + 1; second < vectors.Count; second++)
{
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
pairs++;
}
}
return total / pairs;
}
private static double Median(IReadOnlyList<double> sortedValues)
{
var middle = sortedValues.Count / 2;
return sortedValues.Count % 2 == 0
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
: sortedValues[middle];
}
}
@@ -1,208 +0,0 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorUpdateResult(
int AddedCount,
int RemovedCount)
{
public bool Changed => AddedCount > 0 || RemovedCount > 0;
}
public sealed class ResemblyzerVoiceVectorOutlierPruner
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
public ResemblyzerVoiceVectorOutlierPruner(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
{
this.options = options;
this.logger = logger;
}
public int Prune(SpeakerIdentity identity)
{
var compatible = new List<DecodedSpeakerVoiceVector>();
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
identity,
options.ModelId,
logger))
{
try
{
compatible.Add(new DecodedSpeakerVoiceVector(
entry.Stored,
SpeakerVoiceVectors.Normalize(
entry.Vector,
$"Stored voice vector {entry.Stored.Id}")));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
entry.Stored.Id,
identity.Id);
}
}
var minimumVectors = options.OutlierPruningMinimumVectors;
if (compatible.Count < minimumVectors)
{
return 0;
}
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
var ranked = clusters
.GroupBy(cluster => cluster)
.Where(group => group.Key > 0)
.Select(group => new { Id = group.Key, Count = group.Count() })
.OrderByDescending(group => group.Count)
.ThenBy(group => group.Id)
.ToList();
if (ranked.Count == 0)
{
return Skip(identity, "no dense cluster was found");
}
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
{
return Skip(identity, "no uniquely largest dense cluster was found");
}
var dominant = ranked[0];
var ratio = (double)dominant.Count / compatible.Count;
if (ratio < options.OutlierPruningMinimumClusterRatio)
{
return Skip(
identity,
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
}
var removed = compatible
.Where((_, index) => clusters[index] != dominant.Id)
.Select(item => item.Stored)
.ToList();
foreach (var vector in removed)
{
identity.VoiceVectors.Remove(vector);
}
if (removed.Count > 0)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
}
logger.LogInformation(
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
removed.Count,
identity.Id,
dominant.Count,
compatible.Count);
return removed.Count;
}
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
SpeakerIdentity identity,
IReadOnlyList<float[]> vectors,
DateTimeOffset createdAt)
{
var removedBeforeAdding = Prune(identity);
var added = SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
options.ModelId,
options.MaxVectorsPerIdentity,
createdAt);
var removedAfterAdding = Prune(identity);
var result = new ResemblyzerVoiceVectorUpdateResult(
added,
removedBeforeAdding + removedAfterAdding);
if (result.Changed)
{
identity.UpdatedAt = createdAt;
}
return result;
}
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
{
var neighbors = Enumerable.Range(0, vectors.Count)
.Select(index => Enumerable.Range(0, vectors.Count)
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
.ToList())
.ToList();
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
var labels = new int[vectors.Count];
var clusterId = 0;
for (var point = 0; point < vectors.Count; point++)
{
if (labels[point] != 0)
{
continue;
}
if (neighbors[point].Count < minimumNeighbors)
{
labels[point] = -1;
continue;
}
clusterId++;
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
}
return labels;
}
private static void ExpandCluster(
int seed,
int clusterId,
IReadOnlyList<List<int>> neighbors,
int[] labels,
int minimumNeighbors)
{
labels[seed] = clusterId;
var pending = new Queue<int>(neighbors[seed]);
var queued = neighbors[seed].ToHashSet();
while (pending.TryDequeue(out var point))
{
if (labels[point] == -1)
{
labels[point] = clusterId;
}
if (labels[point] != 0)
{
continue;
}
labels[point] = clusterId;
if (neighbors[point].Count < minimumNeighbors)
{
continue;
}
foreach (var neighbor in neighbors[point])
{
if (queued.Add(neighbor))
{
pending.Enqueue(neighbor);
}
}
}
}
private int Skip(
SpeakerIdentity identity,
string reason)
{
logger.LogInformation(
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
identity.Id,
reason);
return 0;
}
}
@@ -1,54 +0,0 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerWarmupHostedService : BackgroundService
{
private readonly IResemblyzerVoiceEncoder encoder;
private readonly bool speakerIdentificationEnabled;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
public ResemblyzerWarmupHostedService(
IResemblyzerVoiceEncoder encoder,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerWarmupHostedService> logger)
{
this.encoder = encoder;
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!speakerIdentificationEnabled || !options.Enabled)
{
return;
}
await Task.Yield();
try
{
logger.LogInformation(
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
await encoder.WarmUpAsync(stoppingToken);
logger.LogInformation(
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
}
catch (Exception exception)
{
logger.LogError(
exception,
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
VaultPath.Resolve(options.RuntimeFolder));
}
}
}
+1 -36
View File
@@ -21,8 +21,6 @@ public sealed class SpeakerIdentity
public List<SpeakerSnippet> Snippets { get; set; } = [];
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
public List<SpeakerIdentityReference> References { get; set; } = [];
[NotMapped]
@@ -95,7 +93,7 @@ public static class SpeakerIdentityReferences
string.IsNullOrWhiteSpace(reference.TranscriptPath);
}
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
@@ -126,25 +124,6 @@ public sealed class SpeakerSnippet
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerVoiceVector
{
public int Id { get; set; }
public int SpeakerIdentityId { get; set; }
public SpeakerIdentity? SpeakerIdentity { get; set; }
public string ModelId { get; set; } = "";
public int Dimensions { get; set; }
public byte[] VectorBytes { get; set; } = [];
public string Fingerprint { get; set; } = "";
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerAlias
{
public int Id { get; set; }
@@ -171,8 +150,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -198,11 +175,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.VoiceVectors)
.WithOne(vector => vector.SpeakerIdentity)
.HasForeignKey(vector => vector.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.References)
.WithOne(reference => reference.SpeakerIdentity)
.HasForeignKey(reference => reference.SpeakerIdentityId)
@@ -217,13 +189,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
modelBuilder.Entity<SpeakerIdentityReference>()
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
.IsUnique();
@@ -20,20 +20,17 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerIdentityMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ILogger<SpeakerIdentityMergeService> logger;
public SpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerIdentityMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<SpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
this.logger = logger;
}
@@ -47,11 +44,9 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
@@ -159,12 +154,10 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeIntoAndPrune(
SpeakerIdentityMerger.MergeInto(
target,
source,
options.MaxSnippetsPerSpeaker,
options.Resemblyzer.MaxVectorsPerIdentity,
outlierPruner);
options.MaxSnippetsPerSpeaker);
logger.LogInformation(
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
source.Id,
@@ -2,24 +2,10 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityMerger
{
public static void MergeIntoAndPrune(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets,
int maxVoiceVectors,
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
{
outlierPruner.Prune(target);
outlierPruner.Prune(source);
MergeInto(target, source, maxSnippets, maxVoiceVectors);
outlierPruner.Prune(target);
}
public static void MergeInto(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets,
int maxVoiceVectors = int.MaxValue)
int maxSnippets)
{
AddAlias(target, source.CanonicalName);
foreach (var alias in source.Aliases)
@@ -49,26 +35,6 @@ internal static class SpeakerIdentityMerger
.ToList();
target.Snippets.Clear();
target.Snippets.AddRange(retainedSnippets);
var retainedVectors = target.VoiceVectors
.Concat(source.VoiceVectors)
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
.OrderByDescending(vector => vector.CreatedAt)
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Take(Math.Max(1, maxVoiceVectors))
.Select(vector => new SpeakerVoiceVector
{
SpeakerIdentity = target,
ModelId = vector.ModelId,
Dimensions = vector.Dimensions,
VectorBytes = vector.VectorBytes.ToArray(),
Fingerprint = vector.Fingerprint,
CreatedAt = vector.CreatedAt
})
.ToList();
target.VoiceVectors.Clear();
target.VoiceVectors.AddRange(retainedVectors);
target.UpdatedAt = DateTimeOffset.UtcNow;
}
@@ -1,103 +0,0 @@
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityNaming
{
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[] { identity.CanonicalName }
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
public static bool MatchesAnyAcceptedName(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return GetAcceptedNames(identity).Any(names.Contains);
}
public static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(MeetingAttendeeNames.NormalizeDisplayName)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
public static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
public static bool UpdateCandidateNames(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return false;
}
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
var resetToAttendees = intersection.Count == 0;
SetCandidates(identity, resetToAttendees ? attendees : intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
}
return resetToAttendees;
}
public static void SetCandidates(
SpeakerIdentity identity,
IEnumerable<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
}
@@ -46,33 +46,6 @@ internal static class SpeakerIdentitySchema
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
"SpeakerIdentityId" INTEGER NOT NULL,
"ModelId" TEXT NOT NULL,
"Dimensions" INTEGER NOT NULL,
"VectorBytes" BLOB NOT NULL,
"Fingerprint" TEXT NOT NULL,
"CreatedAt" TEXT NOT NULL,
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
);
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
""",
cancellationToken);
}
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
target.CanonicalName = targetName;
target.UpdatedAt = now;
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
ResetCandidates(target, [targetName]);
AddMeetingReference(target, meetingReference);
var snippetAdded = AddSnippetIfNeeded(target, snippet);
await context.SaveChangesAsync(cancellationToken);
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
{
alreadyIdentifiedNames.Add(speaker.Trim());
}
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
cancellationToken);
}
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
foreach (var acceptedName in GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
{
speakerMappings[speaker] = speakerName;
alreadyIdentifiedNames.Add(speakerName);
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
foreach (var acceptedName in GetAcceptedNames(identity))
{
alreadyIdentifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
speakerName,
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
GetAcceptedNames(identity).ToList()));
}
}
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Select(identity => new
{
Identity = identity,
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsAttendee = MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
string operation,
CancellationToken cancellationToken)
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
request.Segments,
speaker,
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration);
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration);
logger.LogInformation(
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
operation,
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Include(identity => identity.References)
.ToListAsync(cancellationToken);
return identities
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
.Where(identity => GetAcceptedNames(identity).Contains(name))
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.ToListAsync(cancellationToken);
return candidates
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
.Where(identity => identity.CandidateNames.Any(candidate =>
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
identity.Aliases.Any(alias =>
@@ -646,11 +646,27 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
}
private static bool IsSameReference(
SpeakerIdentityReference first,
SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
}
private static bool MatchesAcceptedNames(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
return GetAcceptedNames(identity).Any(names.Contains);
}
private static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
}
private static Task<SpeakerIdentity?> LoadIdentityAsync(
@@ -678,25 +694,48 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
if (resetToAttendees)
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (intersection.Count == 0)
{
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
identity.Id,
FormatNames(attendees));
ResetCandidates(identity, attendees);
ReplaceOldestSnippet(identity, snippet);
return;
}
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
identity.Id,
FormatNames(currentCandidates),
FormatNames(attendees),
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
FormatNames(intersection));
ResetCandidates(identity, intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
logger.LogInformation(
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
identity.Id,
@@ -833,6 +872,51 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
}
}
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[]
{
identity.CanonicalName
}
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static string NormalizeAttendee(string attendee)
{
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(
@@ -1,154 +0,0 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed record SpeakerSampleCollectionPolicy(
int MinimumRetainedSamples,
bool RequireNonOverlappingSamples)
{
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
{
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
}
public int ResolveRetainedSampleLimit(int configuredLimit)
{
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
}
}
internal static class SpeakerSampleDurationConfiguration
{
public static void ValidateOrThrow(
SpeakerIdentificationOptions options,
string context)
{
ValidateOrThrow(
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
context);
}
public static void ValidateOrThrow(
TimeSpan minimumDuration,
TimeSpan maximumDuration,
string context)
{
var error = GetError(minimumDuration, maximumDuration);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(
TimeSpan minimumDuration,
TimeSpan maximumDuration)
{
if (minimumDuration < TimeSpan.Zero)
{
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
}
if (maximumDuration <= TimeSpan.Zero)
{
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
}
return minimumDuration > maximumDuration
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
: null;
}
}
internal static class ResemblyzerSpeakerRecognitionConfiguration
{
public static void ValidateOrThrow(
ResemblyzerSpeakerRecognitionOptions options,
string context)
{
var error = GetError(options);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
{
if (options.RequiredVectorsPerSpeaker < 1)
{
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
}
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
}
if (options.OutlierPruningMinimumNeighbors < 1 ||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
}
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
}
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
}
if (!IsCosine(options.MinimumClusterCohesion))
{
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
}
if (!IsCosine(options.MinimumIdentitySimilarity))
{
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
}
return !double.IsFinite(options.MinimumSimilarityMargin) ||
options.MinimumSimilarityMargin is < 0 or > 2
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
: null;
}
private static bool IsCosine(double value)
{
return double.IsFinite(value) && value is >= -1 and <= 1;
}
}
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
: IValidateOptions<MeetingAssistantOptions>
{
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
{
var error = SpeakerSampleDurationConfiguration.GetError(
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleDuration) ??
ResemblyzerSpeakerRecognitionConfiguration.GetError(
options.SpeakerIdentification.Resemblyzer);
return error is null
? ValidateOptionsResult.Success
: ValidateOptionsResult.Fail(error);
}
}
@@ -1,82 +0,0 @@
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Speakers;
internal sealed record SpeakerSampleSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
TimeSpan SpeechDuration,
TimeSpan MaximumDuration,
IReadOnlyList<TranscriptionSegment> Segments)
{
public static SpeakerSampleSpan? Create(
TranscriptionSegment segment,
TimeSpan maximumDuration)
{
if (maximumDuration <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(maximumDuration),
maximumDuration,
"Speaker sample maximum duration must be greater than zero.");
}
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
return boundedEnd > segment.Start
? new SpeakerSampleSpan(
segment.Speaker,
segment.Start,
boundedEnd,
boundedEnd - segment.Start,
maximumDuration,
[segment with { End = boundedEnd }])
: null;
}
public bool TryExtend(
TranscriptionSegment segment,
out SpeakerSampleSpan extended)
{
extended = this;
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
segment.Start >= Start + MaximumDuration)
{
return false;
}
var boundedEnd = Min(segment.End, Start + MaximumDuration);
if (boundedEnd <= segment.Start)
{
return false;
}
var uncoveredStart = segment.Start > End ? segment.Start : End;
var additionalSpeechDuration = boundedEnd > uncoveredStart
? boundedEnd - uncoveredStart
: TimeSpan.Zero;
extended = this with
{
End = boundedEnd > End ? boundedEnd : End,
SpeechDuration = SpeechDuration + additionalSpeechDuration,
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
};
return true;
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', Segments
.Select(segment => segment.Text)
.Where(text => !string.IsNullOrWhiteSpace(text))));
}
private static TimeSpan Min(TimeSpan left, TimeSpan right)
{
return left < right ? left : right;
}
}
@@ -4,82 +4,54 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerSampleSpanSelector
{
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration)
public static bool CanExtend(
string currentSpeaker,
TimeSpan currentEnd,
TranscriptionSegment nextSegment,
TimeSpan maximumSegmentGap)
{
SpeakerSampleSpan? best = null;
SpeakerSampleSpan? current = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (!IsSpeaker(segment, speaker))
{
current = null;
continue;
}
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is not null &&
(best is null || current.SpeechDuration > best.SpeechDuration))
{
best = current;
}
}
return best is not null && best.SpeechDuration >= minimumDuration
? best.Segments
: [];
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
nextSegment.Start - currentEnd <= maximumSegmentGap;
}
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration,
int maxSpans)
TimeSpan maximumSegmentGap,
TimeSpan minimumDuration)
{
if (maxSpans <= 0)
{
return [];
}
var best = new List<TranscriptionSegment>();
var current = new List<TranscriptionSegment>();
var completed = new List<SpeakerSampleSpan>();
SpeakerSampleSpan? current = null;
TimeSpan? lastCompletedEnd = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (!IsSpeaker(segment, speaker))
if (current.Count == 0)
{
current = null;
if (IsSpeaker(segment, speaker))
{
current.Add(segment);
best = LongerSpan(current, best);
}
continue;
}
if (lastCompletedEnd is { } end && segment.Start < end)
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
{
current = null;
continue;
current.Clear();
if (!IsSpeaker(segment, speaker))
{
continue;
}
}
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is null || current.SpeechDuration < minimumDuration)
{
continue;
}
completed.Add(current);
lastCompletedEnd = current.End;
current = null;
current.Add(segment);
best = LongerSpan(current, best);
}
return completed
.OrderByDescending(span => span.SpeechDuration)
.ThenBy(span => span.Start)
.Take(maxSpans)
.OrderBy(span => span.Start)
.Select(span => span.Segments)
.ToList();
return SpanDuration(best) >= minimumDuration
? best
: [];
}
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
@@ -89,20 +61,19 @@ internal static class SpeakerSampleSpanSelector
: segments[^1].End - segments[0].Start;
}
private static SpeakerSampleSpan? ExtendOrStart(
SpeakerSampleSpan? current,
TranscriptionSegment segment,
TimeSpan maximumSampleDuration)
{
return current is not null && current.TryExtend(segment, out var extended)
? extended
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
}
private static bool IsSpeaker(
TranscriptionSegment segment,
string speaker)
{
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
}
private static List<TranscriptionSegment> LongerSpan(
List<TranscriptionSegment> current,
List<TranscriptionSegment> best)
{
return SpanDuration(current) > SpanDuration(best)
? current.ToList()
: best;
}
}
@@ -1,191 +0,0 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
namespace MeetingAssistant.Speakers;
internal sealed record DecodedSpeakerVoiceVector(
SpeakerVoiceVector Stored,
float[] Vector);
internal static class ResemblyzerVectorContract
{
public const int Dimensions = 256;
}
internal static class SpeakerVoiceVectors
{
public static float[] Decode(SpeakerVoiceVector stored)
{
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
{
throw new InvalidDataException(
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
}
var vector = new float[stored.Dimensions];
for (var index = 0; index < vector.Length; index++)
{
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
}
}
return vector;
}
public static int AddDistinct(
SpeakerIdentity identity,
IEnumerable<float[]> vectors,
string modelId,
int maximumCount,
DateTimeOffset createdAt)
{
var limit = Math.Max(1, maximumCount);
var fingerprints = identity.VoiceVectors
.Select(vector => vector.Fingerprint)
.ToHashSet(StringComparer.Ordinal);
var added = 0;
foreach (var vector in vectors)
{
if (identity.VoiceVectors.Count >= limit)
{
break;
}
var bytes = Encode(vector);
var fingerprint = Fingerprint(modelId, bytes);
if (!fingerprints.Add(fingerprint))
{
continue;
}
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = modelId,
Dimensions = vector.Length,
VectorBytes = bytes,
Fingerprint = fingerprint,
CreatedAt = createdAt
});
added++;
}
return added;
}
public static IReadOnlyList<float[]> DecodeCompatible(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
return DecodeCompatibleEntries(identity, modelId, logger)
.Select(entry => entry.Vector)
.ToList();
}
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
var vectors = new List<DecodedSpeakerVoiceVector>();
foreach (var stored in identity.VoiceVectors
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
.OrderBy(vector => vector.CreatedAt)
.ThenBy(vector => vector.Id))
{
try
{
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
stored.Id,
identity.Id);
}
}
return vectors;
}
public static double Cosine(float[] first, float[] second)
{
double dot = 0;
for (var index = 0; index < first.Length; index++)
{
dot += first[index] * second[index];
}
return dot;
}
public static float[] Normalize(float[]? vector, string description)
{
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
double magnitudeSquared = 0;
foreach (var value in vector)
{
if (!float.IsFinite(value))
{
throw new InvalidDataException($"{description} contains a non-finite value.");
}
magnitudeSquared += value * value;
}
var magnitude = Math.Sqrt(magnitudeSquared);
if (magnitude <= double.Epsilon)
{
throw new InvalidDataException($"{description} has zero magnitude.");
}
return vector.Select(value => (float)(value / magnitude)).ToArray();
}
private static byte[] Encode(float[] vector)
{
if (vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
var bytes = new byte[vector.Length * sizeof(float)];
for (var index = 0; index < vector.Length; index++)
{
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException("Voice vector contains a non-finite value.");
}
BinaryPrimitives.WriteSingleLittleEndian(
bytes.AsSpan(index * sizeof(float), sizeof(float)),
vector[index]);
}
return bytes;
}
private static string Fingerprint(string modelId, byte[] vectorBytes)
{
var modelBytes = Encoding.UTF8.GetBytes(modelId);
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
modelBytes.CopyTo(input, 0);
input[modelBytes.Length] = 0;
vectorBytes.CopyTo(input, modelBytes.Length + 1);
return Convert.ToHexString(SHA256.HashData(input));
}
}
@@ -1,391 +0,0 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public interface IResemblyzerVoiceEncoder
{
Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken);
Task WarmUpAsync(CancellationToken cancellationToken);
}
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
{
private const string EnvironmentSchemaVersion = "venv-v2";
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
private const string LibrosaRequirement = "librosa>=0.9.1";
private const string ScipyRequirement = "scipy>=1.2.1";
private readonly SemaphoreSlim commandLock = new(1, 1);
private readonly ICommandRunner commandRunner;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
private string? verifiedEnvironmentPythonPath;
public VenvResemblyzerVoiceEncoder(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<VenvResemblyzerVoiceEncoder> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
public async Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
if (wavSamples.Count == 0)
{
return [];
}
if (wavSamples.Any(sample => sample.Length == 0))
{
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
}
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(inputFolder);
try
{
for (var index = 0; index < wavSamples.Count; index++)
{
await File.WriteAllBytesAsync(
Path.Combine(inputFolder, $"{index:D4}.wav"),
wavSamples[index],
cancellationToken);
}
await commandLock.WaitAsync(cancellationToken);
try
{
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
}
finally
{
commandLock.Release();
}
}
finally
{
try
{
Directory.Delete(inputFolder, recursive: true);
}
catch (DirectoryNotFoundException)
{
}
catch (IOException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
catch (UnauthorizedAccessException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
}
}
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
string inputFolder,
int expectedCount,
CancellationToken cancellationToken)
{
return await RunWithTimeoutAsync(async token =>
{
var pythonPath = await EnsureEnvironmentAsync(token);
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
var result = await RunRequiredAsync(
pythonPath,
[scriptPath, Path.GetFullPath(inputFolder)],
"encoding",
token);
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
}, cancellationToken);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
await commandLock.WaitAsync(cancellationToken);
try
{
await RunWithTimeoutAsync(async token =>
{
await EnsureEnvironmentAsync(token);
return true;
}, cancellationToken);
}
finally
{
commandLock.Release();
}
}
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
{
ValidateDependencySettings();
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
var readyPath = Path.Combine(environmentFolder, ".ready");
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
&& File.Exists(pythonPath)
&& File.Exists(readyPath))
{
return pythonPath;
}
if (File.Exists(pythonPath) && File.Exists(readyPath))
{
var verification = await commandRunner.RunAsync(
pythonPath,
WarmupArguments,
cancellationToken);
if (verification.ExitCode == 0)
{
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
logger.LogWarning(
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
verification.ExitCode,
verification.StandardError);
File.Delete(readyPath);
}
Directory.CreateDirectory(environmentFolder);
await RunRequiredAsync(
options.PythonCommand,
["-m", "venv", "--clear", environmentFolder],
"virtual environment creation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
["-m", "pip", "install", "--upgrade", "pip"],
"pip upgrade",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
"--index-url", options.TorchIndexUrl,
TorchRequirement
],
"CPU PyTorch installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
.. RuntimeDependencyRequirements
],
"Resemblyzer dependency installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install", "--no-deps",
ResemblyzerRequirement
],
"Resemblyzer installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
WarmupArguments,
"environment verification",
cancellationToken);
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
private async Task<CommandResult> RunRequiredAsync(
string fileName,
IReadOnlyList<string> arguments,
string operation,
CancellationToken cancellationToken)
{
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
ThrowIfFailed(result, operation);
return result;
}
private static string BuildEncodingScript()
{
return
"import json\n"
+ "import sys\n"
+ "import numpy as np\n"
+ "from pathlib import Path\n"
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
+ "vectors = []\n"
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
+ " wav = preprocess_wav(path)\n"
+ " vector = encoder.embed_utterance(wav)\n"
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(vectors, allow_nan=False))\n"
+ $"print('{JsonEnd}')\n";
}
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
{
var json = ExtractJson(output);
float[][]? vectors;
try
{
vectors = JsonSerializer.Deserialize<float[][]>(json);
}
catch (JsonException exception)
{
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
}
if (vectors is null || vectors.Length != expectedCount)
{
throw new InvalidDataException(
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
}
return vectors
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
.ToList();
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private string BuildEnvironmentKey()
{
var settings = string.Join('\n', EnvironmentManifest);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
}
private string TorchRequirement => $"torch=={options.TorchVersion}";
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
private static string[] WarmupArguments =>
[
"-c",
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
];
private string[] RuntimeDependencyRequirements =>
[
NumpyBeforePython313Requirement,
NumpyFromPython313Requirement,
LibrosaRequirement,
ScipyRequirement,
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
];
private IEnumerable<string> EnvironmentManifest =>
[
EnvironmentSchemaVersion,
options.PythonCommand,
options.TorchIndexUrl,
TorchRequirement,
.. RuntimeDependencyRequirements,
ResemblyzerRequirement
];
private static string GetEnvironmentPythonPath(string environmentFolder)
{
return OperatingSystem.IsWindows()
? Path.Combine(environmentFolder, "Scripts", "python.exe")
: Path.Combine(environmentFolder, "bin", "python");
}
private void ValidateDependencySettings()
{
if (string.IsNullOrWhiteSpace(options.PythonCommand))
{
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
}
ValidateVersion(options.PackageVersion, "Resemblyzer package");
ValidateVersion(options.TorchVersion, "PyTorch");
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|| indexUri.Scheme != Uri.UriSchemeHttps)
{
throw new InvalidOperationException(
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
}
}
private static void ValidateVersion(string version, string dependency)
{
if (!PackageVersionPattern().IsMatch(version))
{
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
}
}
private async Task<T> RunWithTimeoutAsync<T>(
Func<CancellationToken, Task<T>> operation,
CancellationToken cancellationToken)
{
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(options.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
return await operation(linkedSource?.Token ?? cancellationToken);
}
catch (OperationCanceledException) when (
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
}
}
private static void ThrowIfFailed(CommandResult result, string operation)
{
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
private static partial Regex PackageVersionPattern();
}
@@ -18,12 +18,6 @@ public sealed class BoundMeetingProjectResolver
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
public List<BoundMeetingProject> GetAllProjects() => !Directory.Exists(ProjectsRoot) ? [] :
Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken = default)
@@ -39,8 +33,10 @@ public sealed class BoundMeetingProjectResolver
return [];
}
return GetAllProjects()
return Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
@@ -7,6 +7,7 @@ using System.Text.Json.Nodes;
using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Logging;
using OpenAI;
using OpenAI.Responses;
namespace MeetingAssistant.Summary;
@@ -797,7 +798,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
string apiKey,
AsyncLocal<string?> requestInitiator)
{
var options = new ResponsesClientOptions
var options = new OpenAIClientOptions
{
Endpoint = httpClient.BaseAddress
?? throw new InvalidOperationException("LiteLLM HTTP client requires a base address."),
@@ -20,20 +20,12 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
Use override_speaker only when you are very certain that a transcript speaker label belongs to a named person, for example from user notes, OCR evidence with a matching timestamp, or very clear context cues. Provide the exact speaker label from the transcript and the replacement speaker name. If the replacement speaker already exists in the transcript, use merge=true only when you are certain both speaker labels are the same identity; otherwise do not merge them.
Use delete_identity only when you are certain that an existing speaker identity was wrongfully matched. Provide the exact identity name currently used in the transcript.
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
Use list_projects first to see which projects are bound to this meeting. Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. When embedding them, encode spaces in image-link targets as `%20`; never leave literal spaces in the link target. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
Keep the output grounded in the source material and explicitly say when a section has no known items.
""";
private const string ProjectAndRequestInstructions = """
Project discovery and approval:
list_projects returns every configured project's id, displayname, and description; only projects in the current meeting note's projects frontmatter are accessible for reads, writes, search, and historical summaries. Catalog visibility does not grant file access.
Before finalizing the summary, use the catalog when meeting evidence suggests a missing project association. Call request_project_association with its exact id and a short evidence-based reason (one line, at most 100 characters). It waits for user approval for up to ten minutes. A denial or timeout grants no access; continue with the existing scope. Approval adds the project to the meeting and returns its current project instructions; follow them immediately.
For each associated project, maintain a root PROJECT.md with YAML frontmatter name (display name) and description (at most 256 characters). Describe the project's purpose and distinguishing context so future summarizers can select it. Read existing content first and use write_projectfile to create missing metadata or update it from grounded knowledge, preserving other fields and body notes. Do not invent facts or write unassociated projects.
If a workflow improvement would help, use request_workflow_change with a short one-line intention (at most 100 characters) and a detailed self-contained change prompt, preferably a scenario/specification with expected behavior. This returns requested immediately and does not mean approved or implemented. Continue summarizing; only user approval starts the separate settings agent. You cannot change workflows directly.
""";
private readonly MeetingAssistantOptions options;
private readonly BoundMeetingProjectResolver projectResolver;
@@ -58,7 +50,6 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt)
? DefaultInitialPrompt
: options.Agent.InitialPrompt.Trim();
instructions = instructions.TrimEnd() + "\n\n" + ProjectAndRequestInstructions;
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, options, cancellationToken);
return string.IsNullOrWhiteSpace(projectInstructions)
? instructions
@@ -69,15 +60,44 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, options, cancellationToken);
if (projects.Count == 0)
{
return "";
}
var blocks = projects.Select(project =>
$"# {project.Name}\n\n{project.Instructions.Trim()}");
return "---\nprojects:\n\n" + string.Join("\n\n", blocks);
}
private async Task<List<ProjectInstructions>> GetBoundProjectsWithInstructionsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var resolver = ReferenceEquals(options, this.options)
? projectResolver
: new BoundMeetingProjectResolver(options);
var blocks = new List<string>();
var projects = new List<ProjectInstructions>();
foreach (var project in await resolver.GetBoundProjectsAsync(artifacts, cancellationToken))
{
blocks.Add(await ProjectMetadata.BuildContextAsync(project, cancellationToken));
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
if (!File.Exists(agentsPath))
{
continue;
}
var content = await File.ReadAllTextAsync(agentsPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(content))
{
projects.Add(new ProjectInstructions(project.Name, content));
}
}
return blocks.Count == 0 ? "" : "---\nprojects:\n\n" + string.Join("\n\n", blocks);
return projects;
}
private sealed record ProjectInstructions(string Name, string Instructions);
}
+17 -89
View File
@@ -2,9 +2,6 @@ using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using System.Text.Json;
using MeetingAssistant.Notifications;
using YamlDotNet.RepresentationModel;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
@@ -19,8 +16,6 @@ public sealed class MeetingSummaryTools
private readonly SummaryAgentWriteAudit? writeAudit;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly BoundMeetingProjectResolver projectResolver;
private readonly AgentApprovalService? approvals;
private readonly WorkflowChangeRequestService? workflowRequests;
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
: this(artifacts, new MeetingAssistantOptions(), null)
@@ -32,9 +27,7 @@ public sealed class MeetingSummaryTools
MeetingAssistantOptions options,
IDictationWordStore? dictationWordStore = null,
SummaryAgentWriteAudit? writeAudit = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
AgentApprovalService? approvals = null,
WorkflowChangeRequestService? workflowRequests = null)
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{
this.artifacts = artifacts;
this.options = options;
@@ -42,8 +35,6 @@ public sealed class MeetingSummaryTools
this.writeAudit = writeAudit;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
projectResolver = new BoundMeetingProjectResolver(options);
this.approvals = approvals;
this.workflowRequests = workflowRequests;
}
public Task<string> ReadTranscript(int? @from = null, int? to = null, int? tail = null)
@@ -300,71 +291,8 @@ public sealed class MeetingSummaryTools
public async Task<string> ListProjects()
{
var projects = await Task.WhenAll(projectResolver.GetAllProjects().Select(project => ProjectMetadata.ReadAsync(project)));
return JsonSerializer.Serialize(projects.Select(project => new
{
id = project.Id, displayname = project.DisplayName, description = project.Description
}));
}
public async Task<string> RequestProjectAssociation(string project_id, string reason, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(reason) || reason.Length > 100 || reason.Contains('\n') || reason.Contains('\r'))
return "Refused: reason must be a nonempty single line of at most 100 characters.";
var project = projectResolver.GetAllProjects()
.FirstOrDefault(candidate => string.Equals(candidate.Name, project_id, StringComparison.OrdinalIgnoreCase));
if (project is null) return "Refused: unknown project ID.";
var associated = await projectResolver.ReadMeetingProjectNamesAsync(artifacts, cancellationToken);
if (!associated.Contains(project.Name))
{
var metadata = await ProjectMetadata.ReadAsync(project, cancellationToken);
if (approvals is null || !await approvals.ConfirmAsync("Project association requested",
$"Associate {metadata.DisplayName} ({metadata.Id}) with {Path.GetFileName(artifacts.MeetingNotePath)}?\n{reason}",
cancellationToken))
return "Denied: project association was not approved.";
cancellationToken.ThrowIfCancellationRequested();
if (!Directory.Exists(project.Path) || !File.Exists(artifacts.MeetingNotePath))
return "Refused: the project or meeting note no longer exists.";
await AddProjectAssociationAsync(project.Name, cancellationToken);
}
return "Approved: project is associated with this meeting.\n\n"
+ await ProjectMetadata.BuildContextAsync(project, cancellationToken);
}
public string RequestWorkflowChange(string intention, string detailed_prompt)
=> workflowRequests?.Request(intention, detailed_prompt, artifacts.MeetingNotePath)
?? "Refused: workflow change requests are unavailable.";
private async Task AddProjectAssociationAsync(string projectId, CancellationToken cancellationToken)
{
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath, cancellationToken);
var document = MarkdownDocumentParser.SplitRequired(content,
"Meeting note requires frontmatter.", "Meeting note frontmatter is not closed.");
var yaml = new YamlStream();
yaml.Load(new StringReader(document.Frontmatter));
var mapping = (YamlMappingNode)yaml.Documents.Single().RootNode;
mapping.Children.TryGetValue(new YamlScalarNode("projects"), out var existing);
var projects = existing as YamlSequenceNode ?? new YamlSequenceNode();
if (existing is YamlScalarNode scalar && !string.IsNullOrWhiteSpace(scalar.Value))
projects.Add(new YamlScalarNode(scalar.Value));
if (!projects.Children.OfType<YamlScalarNode>().Any(value => string.Equals(value.Value, projectId, StringComparison.OrdinalIgnoreCase)))
projects.Add(new YamlScalarNode(projectId));
mapping.Children[new YamlScalarNode("projects")] = projects;
using var writer = new StringWriter();
yaml.Save(writer, assignAnchors: false);
var pendingPath = artifacts.MeetingNotePath + "." + Guid.NewGuid().ToString("N") + ".tmp";
try
{
await File.WriteAllTextAsync(pendingPath, "---\n" + writer + "---\n\n" + document.Body, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
File.Replace(pendingPath, artifacts.MeetingNotePath, destinationBackupFileName: null);
}
finally
{
File.Delete(pendingPath);
}
var projects = await GetBoundProjectsAsync();
return string.Join('\n', projects.Select(project => project.Name));
}
public async Task<string> ListProjectFiles(string project)
@@ -447,18 +375,10 @@ public sealed class MeetingSummaryTools
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for append; set replace_file=true only for whole-file replacement.";
}
var target = await ResolveBoundProjectFileTargetAsync(project, path);
var target = ResolveExistingProjectFilePath(project, path);
if (target is null)
{
return "Refused: project is not assigned to this meeting, does not exist, or the path escapes the project folder.";
}
var existingContent = File.Exists(target.Path) ? await File.ReadAllTextAsync(target.Path) : "";
var updatedContent = AgentFileToolContent.ApplyLineEdit(existingContent, content, editMode);
if (string.Equals(Path.GetRelativePath(target.Project.Path, target.Path), "PROJECT.md", StringComparison.OrdinalIgnoreCase)
&& !ProjectMetadata.IsValidContent(updatedContent))
{
return "Refused: PROJECT.md requires YAML frontmatter name and description; description must be at most 256 characters.";
return "Refused: project does not exist or the path escapes the project folder.";
}
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
@@ -466,11 +386,11 @@ public sealed class MeetingSummaryTools
{
await writeAudit.CaptureFileWriteAsync(
target.Path,
() => File.WriteAllTextAsync(target.Path, updatedContent));
() => AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode));
}
else
{
await File.WriteAllTextAsync(target.Path, updatedContent);
await AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode);
}
return $"{target.Project.Name}/{AgentFileToolContent.ToToolPath(path)}";
@@ -669,7 +589,7 @@ public sealed class MeetingSummaryTools
return AgentFileToolContent.IsWithinDirectory(projectRoot, fullPath) ? fullPath : null;
}
private async Task<ProjectFileTarget?> ResolveBoundProjectFileTargetAsync(string project, string path)
private ProjectFileTarget? ResolveExistingProjectFilePath(string project, string path)
{
if (string.IsNullOrWhiteSpace(project) ||
string.IsNullOrWhiteSpace(path) ||
@@ -678,7 +598,15 @@ public sealed class MeetingSummaryTools
return null;
}
var projectFolder = await ResolveBoundProjectAsync(project);
var projectsRoot = GetProjectsRoot();
if (!Directory.Exists(projectsRoot))
{
return null;
}
var projectFolder = Directory.EnumerateDirectories(projectsRoot)
.Select(candidate => new BoundMeetingProject(Path.GetFileName(candidate), candidate))
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
if (projectFolder is null)
{
return null;
@@ -1,5 +1,4 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.Agents.AI;
@@ -20,8 +19,6 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
private readonly IMeetingSummaryInstructionBuilder instructionBuilder;
private readonly IDictationWordStore dictationWordStore;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly AgentApprovalService approvals;
private readonly WorkflowChangeRequestService workflowRequests;
public OpenAiMeetingSummaryAgentPipeline(
IOptions<MeetingAssistantOptions> options,
@@ -31,9 +28,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
IMeetingSummaryFailureWriter failureWriter,
IMeetingSummaryInstructionBuilder instructionBuilder,
IDictationWordStore dictationWordStore,
IMeetingWorkflowEngine meetingWorkflowEngine,
AgentApprovalService approvals,
WorkflowChangeRequestService workflowRequests)
IMeetingWorkflowEngine meetingWorkflowEngine)
{
this.options = options.Value;
this.loggerFactory = loggerFactory;
@@ -43,8 +38,6 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
this.instructionBuilder = instructionBuilder;
this.dictationWordStore = dictationWordStore;
this.meetingWorkflowEngine = meetingWorkflowEngine;
this.approvals = approvals;
this.workflowRequests = workflowRequests;
}
public async Task<MeetingSummaryRunResult> RunAsync(
@@ -67,9 +60,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
options,
dictationWordStore,
writeAudit,
meetingWorkflowEngine,
approvals,
workflowRequests);
meetingWorkflowEngine);
var tools = CreateTools(meetingTools);
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
using var compactionSummaryClient = LiteLlmResponsesChatClient.Create(
@@ -89,11 +80,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
.AsBuilder()
.UseFunctionInvocation(
loggerFactory,
client =>
{
client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync;
client.AllowConcurrentInvocation = false;
})
client => client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync)
.Build()
.AsAIAgent(
instructions,
@@ -189,15 +176,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
AIFunctionFactory.Create(
tools.ListProjects,
"list_projects",
"List all configured projects with id, displayname and description. Catalog visibility does not grant access; project files require current meeting association."),
AIFunctionFactory.Create(
tools.RequestProjectAssociation,
"request_project_association",
"Request user approval to associate a project_id with this meeting. reason must be one line and at most 100 characters. Waits up to ten minutes; approval adds the project and returns its AGENTS.md if present."),
AIFunctionFactory.Create(
tools.RequestWorkflowChange,
"request_workflow_change",
"Request a workflow change with a one-line intention (maximum 100 characters) and a detailed_prompt describing the desired scenario/specification. Returns requested immediately; user approval separately opens and starts the settings agent."),
"List the configured project folders bound to the current meeting note frontmatter."),
AIFunctionFactory.Create(
tools.ListProjectFiles,
"list_projectfiles",
@@ -209,7 +188,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
AIFunctionFactory.Create(
tools.WriteProjectFile,
"write_projectfile",
"Write a file inside a project associated with the current meeting. Root PROJECT.md requires YAML name and description (maximum 256 characters). With no line arguments, append or create. Set replace_file=true only for intentional whole-file replacement. With from and to, replace that inclusive 1-based line range; with insert, insert at that line."),
"Write a file inside an existing project folder. With no line arguments, append or create the file. Set replace_file=true only when intentionally replacing the whole file. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."),
AIFunctionFactory.Create(
tools.ListPastProjectMeetings,
"list_past_project_meetings",
@@ -1,55 +0,0 @@
using MeetingAssistant.MeetingNotes;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
public sealed record ProjectMetadata(string Id, string DisplayName, string Description)
{
public static async Task<string> BuildContextAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
{
var metadata = await ReadAsync(project, cancellationToken);
var context = $"# {metadata.Id}\nDisplay name: {metadata.DisplayName}\nDescription: {metadata.Description}";
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
if (File.Exists(agentsPath))
{
var instructions = await File.ReadAllTextAsync(agentsPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(instructions)) context += "\n\n" + instructions.Trim();
}
return context;
}
public static bool IsValidContent(string content) => Parse(content) is not null;
public static async Task<ProjectMetadata> ReadAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
{
var path = Path.Combine(project.Path, "PROJECT.md");
var metadata = File.Exists(path) ? Parse(await File.ReadAllTextAsync(path, cancellationToken)) : null;
return new(project.Name,
string.IsNullOrWhiteSpace(metadata?.Name) ? project.Name : metadata.Name.Trim(),
metadata?.Description?.Trim() ?? "");
}
private static Frontmatter? Parse(string content)
{
try
{
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter) return null;
var metadata = FrontmatterYamlDeserializer.Create().Deserialize<Frontmatter>(document.Frontmatter);
return string.IsNullOrWhiteSpace(metadata?.Name) || metadata.Description is null || metadata.Description.Length > 256
? null
: metadata;
}
catch (YamlException) { return null; }
catch (InvalidDataException) { return null; }
}
private sealed class Frontmatter
{
[YamlMember(Alias = "name")]
public string? Name { get; set; }
[YamlMember(Alias = "description")]
public string? Description { get; set; }
}
}
@@ -1,70 +0,0 @@
using MeetingAssistant.Notifications;
using MeetingAssistant.Workflow;
using System.Threading.Channels;
namespace MeetingAssistant.Summary;
public sealed class WorkflowChangeRequestService(
AgentApprovalService approvals,
IWorkflowRulesEditorWindowService editor,
ILogger<WorkflowChangeRequestService> logger) : BackgroundService
{
private readonly Channel<RequestDetails> requests = Channel.CreateUnbounded<RequestDetails>(
new UnboundedChannelOptions { SingleReader = true });
public string Request(string intention, string detailedPrompt, string meetingNotePath)
{
if (string.IsNullOrWhiteSpace(intention) || intention.Length > 100 || intention.Contains('\r') || intention.Contains('\n'))
return "Refused: intention must be a nonempty single line of at most 100 characters.";
if (string.IsNullOrWhiteSpace(detailedPrompt))
return "Refused: supply a detailed change prompt, preferably a scenario or specification.";
var request = new RequestDetails(Guid.NewGuid().ToString("N"), intention, detailedPrompt, meetingNotePath);
return requests.Writer.TryWrite(request) ? $"Requested: {request.Id}. User approval is pending."
: "Refused: application is stopping.";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var pending = new List<Task>();
try
{
await foreach (var request in requests.Reader.ReadAllAsync(stoppingToken))
{
pending.RemoveAll(task => task.IsCompleted);
pending.Add(HandleAsync(request, stoppingToken));
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
finally
{
await Task.WhenAll(pending);
}
}
private async Task HandleAsync(RequestDetails request, CancellationToken cancellationToken)
{
try
{
if (await approvals.ConfirmAsync($"Workflow change requested: {request.Intention}",
$"The summarizer for {Path.GetFileName(request.MeetingNotePath)} requests a workflow change. Open the settings agent to apply it?",
cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
editor.ShowWithPrompt(request.DetailedPrompt);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { }
catch (Exception exception)
{
logger.LogWarning(exception, "Workflow request {RequestId} could not be handled", request.Id);
}
}
public override Task StopAsync(CancellationToken cancellationToken)
{
requests.Writer.TryComplete();
return base.StopAsync(cancellationToken);
}
private sealed record RequestDetails(string Id, string Intention, string DetailedPrompt, string MeetingNotePath);
}
@@ -9,8 +9,6 @@ public enum MeetingTaskbarAction
OpenSubmenu,
StartRecording,
StopRecording,
PauseTranscription,
UnpauseTranscription,
AbortRecording,
SwitchProfile,
SelectMicrophone,
@@ -53,15 +51,6 @@ public static class MeetingTaskbarMenuBuilder
}
var secondaryControls = new List<MeetingTaskbarMenuItem>();
if (status.IsRecording)
{
secondaryControls.Add(new MeetingTaskbarMenuItem(
status.IsPaused ? "Unpause transcription" : "Pause transcription",
status.IsPaused
? MeetingTaskbarAction.UnpauseTranscription
: MeetingTaskbarAction.PauseTranscription));
}
if (microphones is { Count: > 0 })
{
secondaryControls.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
@@ -249,12 +249,6 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
case MeetingTaskbarAction.StopRecording:
await coordinator.StopAsync(CancellationToken.None);
break;
case MeetingTaskbarAction.PauseTranscription:
await coordinator.SetTranscriptionPausedAsync(true, CancellationToken.None);
break;
case MeetingTaskbarAction.UnpauseTranscription:
await coordinator.SetTranscriptionPausedAsync(false, CancellationToken.None);
break;
case MeetingTaskbarAction.AbortRecording:
await coordinator.AbortAsync(CancellationToken.None);
break;
@@ -96,18 +96,14 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
}
}
private IEnumerable<PyannoteRuntimeOptions> GetEnabledDiarizationOptions()
private IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions()
{
var transcriptionRuntimes = launchProfiles.GetProfiles()
.SelectMany(profile => GetEnabledTranscriptionDiarizationOptions(profile.Options));
var speakerValidationRuntimes = GetEnabledSpeakerValidationDiarizationOptions(
launchProfiles.GetRequiredProfile(null).Options);
return transcriptionRuntimes
.Concat(speakerValidationRuntimes)
return launchProfiles.GetProfiles()
.SelectMany(profile => GetEnabledDiarizationOptions(profile.Options))
.DistinctBy(CreateWarmUpKey);
}
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledTranscriptionDiarizationOptions(
private static IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions(
MeetingAssistantOptions options)
{
if (options.Recording.TranscriptionProvider.Equals("whisper-local", StringComparison.OrdinalIgnoreCase) &&
@@ -115,19 +111,15 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
{
yield return options.WhisperLocal.Diarization;
}
}
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledSpeakerValidationDiarizationOptions(
MeetingAssistantOptions options)
{
if (!options.SpeakerIdentification.Resemblyzer.Enabled &&
options.SpeakerIdentification.PyannoteValidation.Enabled)
if (options.SpeakerIdentification.PyannoteValidation.Enabled &&
options.SpeakerIdentification.PyannoteValidation.Diarization.Enabled)
{
yield return options.SpeakerIdentification.PyannoteValidation.Diarization;
}
}
private static string CreateWarmUpKey(PyannoteRuntimeOptions diarization)
private static string CreateWarmUpKey(PyannoteDiarizationOptions diarization)
{
return string.Join(
'\u001f',
@@ -28,12 +28,7 @@ public sealed class PyannoteTranscriptFinalizer
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
if (!options.WhisperLocal.Diarization.Enabled)
{
return [];
}
return await FinalizeEnabledAsync(
return await FinalizeAsync(
audioPath,
liveSegments,
options.WhisperLocal.Diarization,
@@ -41,14 +36,14 @@ public sealed class PyannoteTranscriptFinalizer
cancellationToken);
}
internal async Task<IReadOnlyList<TranscriptionSegment>> FinalizeEnabledAsync(
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> liveSegments,
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
if (liveSegments.Count == 0)
if (!diarization.Enabled || liveSegments.Count == 0)
{
return [];
}
@@ -88,9 +83,14 @@ public sealed class PyannoteTranscriptFinalizer
}
public async Task WarmUpAsync(
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
CancellationToken cancellationToken)
{
if (!diarization.Enabled)
{
return;
}
var token = ResolveToken(diarization);
if (string.IsNullOrWhiteSpace(token))
{
@@ -137,7 +137,7 @@ public sealed class PyannoteTranscriptFinalizer
private async Task<CommandResult> RunDiarizationAsync(
string fullAudioPath,
string token,
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions,
CancellationToken cancellationToken)
{
@@ -171,7 +171,7 @@ public sealed class PyannoteTranscriptFinalizer
}
private async Task EnsureDockerImageAsync(
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
string modelsFolder,
CancellationToken cancellationToken)
{
@@ -198,7 +198,7 @@ public sealed class PyannoteTranscriptFinalizer
}
}
private static string BuildDockerfile(PyannoteRuntimeOptions diarization)
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
{
return
$"FROM {diarization.BaseImage}\n"
@@ -211,7 +211,7 @@ public sealed class PyannoteTranscriptFinalizer
}
private string[] BuildDockerArguments(
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
string fullAudioPath,
string modelsFolder,
SpeechRecognitionPipelineOptions pipelineOptions)
@@ -242,7 +242,7 @@ public sealed class PyannoteTranscriptFinalizer
}
private string[] BuildWarmUpDockerArguments(
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
string modelsFolder)
{
return
@@ -269,7 +269,7 @@ public sealed class PyannoteTranscriptFinalizer
}
private static string BuildPythonCommand(
PyannoteRuntimeOptions diarization,
PyannoteDiarizationOptions diarization,
SpeechRecognitionPipelineOptions pipelineOptions)
{
var model = diarization.Model;
@@ -296,7 +296,7 @@ public sealed class PyannoteTranscriptFinalizer
+ "python /tmp/meeting_assistant_pyannote.py";
}
private static string BuildWarmUpPythonCommand(PyannoteRuntimeOptions diarization)
private static string BuildWarmUpPythonCommand(PyannoteDiarizationOptions diarization)
{
var model = diarization.Model;
return
@@ -475,7 +475,7 @@ public sealed class PyannoteTranscriptFinalizer
return turns;
}
private static string ResolveToken(PyannoteRuntimeOptions options)
private static string ResolveToken(PyannoteDiarizationOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Token))
{
@@ -3,7 +3,6 @@ namespace MeetingAssistant.Workflow;
public interface IWorkflowRulesEditorWindowService
{
void Show();
void ShowWithPrompt(string initialPrompt);
}
public sealed class NoopWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
@@ -19,6 +18,4 @@ public sealed class NoopWorkflowRulesEditorWindowService : IWorkflowRulesEditorW
{
logger.LogInformation("Workflow rules editor UI is only available on Windows");
}
public void ShowWithPrompt(string initialPrompt) => Show();
}
@@ -24,7 +24,6 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
private readonly IMeetingMetadataProvider meetingMetadataProvider;
private readonly ILaunchProfileOptionsProvider launchProfiles;
private readonly ISpeakerIdentityMergeService identityMergeService;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly AsrDiagnosticService asrDiagnosticService;
private readonly IConfiguration configuration;
@@ -39,7 +38,6 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
IMeetingMetadataProvider meetingMetadataProvider,
ILaunchProfileOptionsProvider launchProfiles,
ISpeakerIdentityMergeService identityMergeService,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
AsrDiagnosticService asrDiagnosticService,
IConfiguration configuration)
{
@@ -53,7 +51,6 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
this.meetingMetadataProvider = meetingMetadataProvider;
this.launchProfiles = launchProfiles;
this.identityMergeService = identityMergeService;
this.outlierPruner = outlierPruner;
this.asrDiagnosticService = asrDiagnosticService;
this.configuration = configuration;
}
@@ -81,8 +78,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
identityMergeService,
asrDiagnosticService,
configuration,
logger: logger,
outlierPruner: outlierPruner);
logger: logger);
var messages = conversation
.Select(ToChatMessage)
.Append(new ChatMessage(ChatRole.User, userMessage.Trim()))
@@ -25,11 +25,9 @@ public sealed class WorkflowRulesEditorChatViewModel
public event EventHandler? Changed;
public Task SendAsync(CancellationToken cancellationToken = default) => SendPromptAsync(Draft, cancellationToken);
public async Task SendPromptAsync(string prompt, CancellationToken cancellationToken = default)
public async Task SendAsync(CancellationToken cancellationToken = default)
{
prompt = prompt.Trim();
var prompt = Draft.Trim();
if (string.IsNullOrWhiteSpace(prompt) || IsThinking)
{
return;
@@ -8,7 +8,6 @@ public interface IWorkflowRulesEditorSamplePlaybackQueue
Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default);
}
#if WINDOWS
public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue, IDisposable
{
private readonly Channel<QueuedSample> channel = Channel.CreateUnbounded<QueuedSample>();
@@ -99,13 +98,3 @@ public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEdito
private sealed record QueuedSample(int SampleId, byte[] WavBytes);
}
#else
public sealed class UnavailableWorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue
{
public Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult("Speaker sample playback requires the Windows build of Meeting Assistant.");
}
}
#endif
@@ -10,7 +10,6 @@ using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
@@ -43,7 +42,6 @@ public sealed class WorkflowRulesEditorTools
private readonly IMeetingMetadataProvider? meetingMetadataProvider;
private readonly ILaunchProfileOptionsProvider? launchProfiles;
private readonly ISpeakerIdentityMergeService? identityMergeService;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly AsrDiagnosticService? asrDiagnosticService;
private readonly IConfiguration? configuration;
private readonly ILogger? logger;
@@ -63,8 +61,7 @@ public sealed class WorkflowRulesEditorTools
string? logDirectory = null,
string? specRootPath = null,
string? projectAgentsTemplatePath = null,
ILogger? logger = null,
ResemblyzerVoiceVectorOutlierPruner? outlierPruner = null)
ILogger? logger = null)
{
this.options = options;
rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath);
@@ -85,9 +82,6 @@ public sealed class WorkflowRulesEditorTools
this.meetingMetadataProvider = meetingMetadataProvider;
this.launchProfiles = launchProfiles;
this.identityMergeService = identityMergeService;
this.outlierPruner = outlierPruner ?? new ResemblyzerVoiceVectorOutlierPruner(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
this.asrDiagnosticService = asrDiagnosticService;
this.configuration = configuration;
this.logger = logger;
@@ -749,8 +743,8 @@ public sealed class WorkflowRulesEditorTools
string[]? candidateNames = null)
{
return Task.FromResult(
"Refused: speaker identities require audio evidence (a WAV sample or voice vector). " +
"Use a speaker override from an existing transcript speaker, or update/merge an existing identity.");
"Refused: speaker identities require at least one audio sample. " +
"Use a speaker override from an existing transcript speaker sample, or update/merge an existing sampled identity.");
}
public async Task<string> UpdateIdentity(
@@ -832,12 +826,10 @@ public sealed class WorkflowRulesEditorTools
return $"Could not find target {targetIdentityId} or source {sourceIdentityId}.";
}
SpeakerIdentityMerger.MergeIntoAndPrune(
SpeakerIdentityMerger.MergeInto(
target,
source,
speakerOptions.MaxSnippetsPerSpeaker,
speakerOptions.Resemblyzer.MaxVectorsPerIdentity,
outlierPruner);
speakerOptions.MaxSnippetsPerSpeaker);
context.SpeakerIdentities.Remove(source);
await context.SaveChangesAsync();
return ToJson(ToIdentityDetail(target));
@@ -1575,11 +1567,9 @@ public sealed class WorkflowRulesEditorTools
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
{
return context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References);
}
@@ -1601,7 +1591,6 @@ public sealed class WorkflowRulesEditorTools
identity.Aliases.Select(alias => alias.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
identity.CandidateNames.Select(candidate => candidate.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
identity.Snippets.Count,
identity.VoiceVectors.Count,
identity.References.Count,
identity.UpdatedAt);
}
@@ -1661,7 +1650,6 @@ public sealed class WorkflowRulesEditorTools
IReadOnlyList<string> Aliases,
IReadOnlyList<string> CandidateNames,
int SampleCount,
int VoiceVectorCount,
int ReferenceCount,
DateTimeOffset UpdatedAt);
@@ -37,26 +37,6 @@ internal sealed class WpfWorkflowRulesEditorWindowService : IWorkflowRulesEditor
dispatcher?.BeginInvoke(ShowOnUiThread);
}
public void ShowWithPrompt(string initialPrompt)
{
EnsureUiThread();
dispatcher?.BeginInvoke(new Action(async () =>
{
try
{
var viewModel = services.GetRequiredService<WorkflowRulesEditorChatViewModel>();
var requestedWindow = new WorkflowRulesEditorWpfWindow(viewModel, linkResolver, () => { });
requestedWindow.Show();
requestedWindow.Activate();
await viewModel.SendPromptAsync(initialPrompt);
}
catch (Exception exception)
{
logger.LogError(exception, "Could not start settings agent for an approved workflow request");
}
}));
}
private void EnsureUiThread()
{
lock (sync)
+4 -24
View File
@@ -36,7 +36,6 @@
"00:10:00"
],
"AutoStopAfter": "00:30:00",
"MaximumPauseDuration": "04:00:00",
"InferredEndPadding": "00:01:00",
"CheckInterval": "00:00:15"
}
@@ -120,17 +119,18 @@
"MaxMatchCandidates": 100,
"MatchIdentityActiveAge": "365.00:00:00",
"MaxSnippetsPerSpeaker": 3,
"MinimumSampleSpeechDuration": "00:00:10",
"MaximumSampleDuration": "00:01:00",
"MinimumSampleSpeechDuration": "00:00:30",
"MaximumSampleSegmentGap": "00:00:01",
"SilenceBetweenSnippetsSeconds": 1,
"LiveSampleBufferDuration": "00:10:00",
"MergeRecentIdentityAge": "14.00:00:00",
"MatchTimeout": "00:03:00",
"PyannoteValidation": {
"Enabled": false,
"Enabled": true,
"MinimumSingleSpeakerCoverage": 0.9,
"MinimumMatchingKnownSnippetRatio": 0.5,
"Diarization": {
"Enabled": false,
"DockerCommand": "docker",
"BaseImage": "python:3.11-slim",
"Image": "meeting-assistant-pyannote:local",
@@ -142,26 +142,6 @@
"TokenEnv": "HF_TOKEN",
"CommandTimeout": "01:00:00"
}
},
"Resemblyzer": {
"Enabled": false,
"RequiredVectorsPerSpeaker": 5,
"MaxVectorsPerIdentity": 1000,
"OutlierPruningMinimumVectors": 20,
"OutlierPruningNeighborSimilarity": 0.75,
"OutlierPruningMinimumNeighbors": 3,
"OutlierPruningMinimumClusterRatio": 0.60,
"MinimumClusterCohesion": 0.75,
"MinimumIdentitySimilarity": 0.75,
"MinimumSimilarityMargin": 0.05,
"ModelId": "resemblyzer-0.1.4-pretrained",
"PackageVersion": "0.1.4",
"PythonCommand": "python",
"TorchVersion": "2.14.0+cpu",
"TorchIndexUrl": "https://download.pytorch.org/whl/cpu",
"WebRtcVadVersion": "2.0.14",
"RuntimeFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Resemblyzer",
"CommandTimeout": "00:15:00"
}
},
"Automation": {
+3 -11
View File
@@ -50,33 +50,26 @@ The default global controls are:
- `Ctrl+Alt+Z`: abort the active run and delete its artifacts.
- `Ctrl+Alt+S`: capture the active window into the meeting context.
The tray presents `Finish meeting` as the primary action during capture; pause/unpause transcription, microphone selection, cancel/discard, and profile switching remain separate controls. A paused meeting stays active and can still be finished or canceled normally. `Exit` is always available and requires confirmation while any meeting is recording or processing.
The tray presents `Finish meeting` as the primary action during capture; microphone selection, cancel/discard, and profile switching remain separate controls. `Exit` is always available and requires confirmation while any meeting is recording or processing.
The loopback HTTP surface exposes health and recording status, recording start/stop/toggle/abort operations, profile-specific equivalents, and diagnostics/retry operations. It has no application authentication. Some generated retry links use `GET` while starting work, so port `5090` must remain a trusted loopback-only control surface.
Stopping capture lets buffered transcription, speaker work, meeting-note image OCR, screenshot OCR, and summary generation finish. Another meeting can start while an older stopped run finalizes; each run retains isolated options and artifact paths.
Pausing transcription keeps the active recognition pipeline and meeting session alive, including Azure conversation transcription and the run's speaker context. Captured audio is replaced with equal-length silence before it reaches the temporary WAV or transcription backend, so real audio from the paused interval is discarded while provider continuity and meeting-relative timing are preserved. Normal transcript-inactivity notifications and auto-stop are suppressed during pause, while a separate four-hour maximum continuous pause prevents a forgotten paused meeting from running indefinitely. The recording status response exposes pause state as `isPaused`.
During an active run, microphone creation failures and disconnects are retried every second with fresh endpoint selection. The meeting and system-loopback capture stay active, with microphone silence mixed in until capture resumes. This recovery does not cover a failed system-loopback source.
A newly available microphone during a meeting triggers a Windows notification with its name and Yes/No actions, including while transcription is paused. Yes switches the current microphone within the same meeting; No or no response keeps capture unchanged. Each device is offered once per meeting, and unanswered prompts expire after one minute.
Outlook enrichment selects an unambiguous current or imminent appointment. A scheduled prompt shown during an active recording can apply that exact appointment's title, eligible attendees, agenda, and scheduled end without interrupting capture; explicit prompt metadata wins over a slower background lookup.
## Data And Side Effects
Meeting Assistant writes meeting notes, transcripts, assistant context, summaries, and project knowledge into the configured Obsidian vault. Assistant context is persistent meeting-specific memory: the summarizer records problems and assumptions, and the interactive agent can read it and append later repairs and conclusions.
Project discovery uses the `name` and short `description` in each project's `PROJECT.md`. The summarizer can request missing associations through a Yes/No notification that waits up to ten minutes; only approval adds the project and enables its file access. Workflow-change requests return immediately and, when approved, open a separate settings-agent conversation with the proposed change.
Agents are intentionally stateful. Depending on the invoked tools, they can change workflow rules and appsettings, create or update project and meeting files, change frontmatter, merge or delete speaker identities and samples, run diagnostics, and trigger transcription or summary work. Screenshot OCR can add recognized attendee names to the meeting note after workflow transformation; OCR of images already embedded in the meeting note does not add attendees or modify that note.
Local runtime state outside the vault includes:
- `%LOCALAPPDATA%\MeetingAssistant\Recordings`: mixed WAV files are normally deleted after completion, and unqueued stale files are deleted at startup. If an Azure stop cannot drain within `Recording:StopProcessingTimeout`, the WAV plus a JSON item under `offline-transcription-backlog` are retained and retried every minute. Both are removed only after successful replay, transcript finalization, and summary processing.
- `%LOCALAPPDATA%\MeetingAssistant\SpeakerIdentity\speaker-identities.db`: SQLite identities, aliases, meeting references, bounded WAV snippets for the existing matcher, and separate versioned voice vectors for the optional Resemblyzer matcher.
- `%LOCALAPPDATA%\MeetingAssistant\Resemblyzer`: content-versioned managed Python environments, the local encoder script, and temporary encoder inputs. Per-meeting WAV inputs are deleted after each encoding command.
- `%LOCALAPPDATA%\MeetingAssistant\SpeakerIdentity\speaker-identities.db`: SQLite identities, aliases, meeting references, and bounded voice snippets.
- `%LOCALAPPDATA%\MeetingAssistant\FunASR\models` and `%LOCALAPPDATA%\MeetingAssistant\Pyannote\models`: persistent model, hotword, Hugging Face, and torch caches for optional local backends.
- `%TEMP%\MeetingAssistant\Logs\meeting-assistant.log`: application log with four rotated predecessors. Paths, transcript text, agent diagnostics, and provider errors can make these logs sensitive.
@@ -89,7 +82,7 @@ Abort is destructive: it removes the active run's note, transcript, context, sum
- The default `azure-speech` provider sends mixed meeting audio and dictation phrase hints to Azure AI Speech. Azure-backed speaker matching also sends selected voice audio.
- Summary, screenshot OCR, and interactive-agent requests go to the configured OpenAI-compatible Responses endpoint. They can include meeting/transcript/project text, screenshots, configuration, logs, and speaker samples when corresponding tools are used. The checked-in endpoint is a loopback proxy; its ultimate provider, data path, and retention policy are outside this repository.
- Outlook Classic access is local COM and reads appointment metadata; it does not provide the primary capture path.
- A managed FunASR run pulls its configured image, removes any same-named container, starts a disposable container privileged by default, publishes the configured host port, and mounts the model/hotword cache. Pyannote may build a local image and start disposable containers with audio inputs mounted read-only; those paths require Docker Desktop or a compatible Docker CLI. The opt-in Resemblyzer speaker matcher instead provisions an isolated local Python venv with CPU-only dependencies. First use may download images, Python packages, or models from external registries.
- A managed FunASR run pulls its configured image, removes any same-named container, starts a disposable container privileged by default, publishes the configured host port, and mounts the model/hotword cache. Pyannote may build a local image and starts disposable containers with the input WAV mounted read-only and its model cache read/write. These paths require Docker Desktop or a compatible Docker CLI and may download images/models from external registries.
## Configuration
@@ -102,7 +95,6 @@ The settings with the largest operational effect are:
- `Recording:MicrophoneDeviceId`, mix gains, stop timeout, minimum duration, and temporary folder: control capture selection, audio, cleanup, and Azure backlog behavior.
- `Recording:InactivitySafeguard`: prompts and can auto-finish a run after no new transcript text; it is not an audio-silence detector.
- `LaunchProfiles`: overlay named recording/ASR/agent settings and require distinct hotkeys.
- `SpeakerIdentification:Resemblyzer:Enabled`: selects the local, managed-Python-venv vector matcher for the whole application; when disabled, the existing WAV/Azure path stays active. Five vectors unlock matching by default without capping retained evidence, and mature profiles use configurable fail-safe density clustering to remove likely mixed-speaker outliers.
- `Automation:RulesPath`: points to the local YAML rules file, normally ignored `meeting-rules.local.yaml`.
- `CalendarRecordingPrompts` and `Screenshots`: control Outlook prompts, capture, attachments, and configured OCR.
- `Agent` and `WorkflowRulesEditor`: select the Responses endpoint/model, streaming or non-streaming transport, reasoning, retry, output, and compaction behavior; the available tools are defined by the application.
+13 -72
View File
@@ -38,7 +38,6 @@ This example is abbreviated so the most common shape is readable. The checked-in
"FirstPromptAfter": "00:02:00",
"ReminderPromptAfter": [ "00:05:00", "00:10:00" ],
"AutoStopAfter": "00:30:00",
"MaximumPauseDuration": "04:00:00",
"InferredEndPadding": "00:01:00",
"CheckInterval": "00:00:15"
}
@@ -132,25 +131,20 @@ The default profile is always named `default`. Non-default profile hotkeys are r
During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription.
The tray's fine-grained controls expose `Pause transcription` while a meeting is active and `Unpause transcription` while it is paused. Pause does not stop audio devices, finish the meeting, replace the speech-recognition pipeline, or reset run-local speaker mappings and collected samples. Instead, each captured mixed chunk is replaced with equal-length PCM silence before it reaches the temporary WAV, live speaker buffer, or configured transcription provider. This discards real audio from the paused interval while preserving the provider session and meeting-relative timing. In particular, Azure keeps the same active `ConversationTranscriber` and push stream; pause therefore does not suspend Azure connection time or billing. `Finish meeting` and cancel/discard remain available while paused, and `/recording/status` reports the state in `isPaused`.
On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits.
During active capture, including paused transcription, active microphone endpoints are checked once per second. A newly available endpoint produces a Windows notification naming the microphone with Yes/No actions; endpoints already present when capture starts are not offered. Yes changes the shared runtime microphone selection and replaces the current microphone capture, preserving the meeting, system audio, transcription session, and pause state. The selection also applies to later recordings until changed or the process exits. No or an unanswered prompt leaves capture unchanged. Each endpoint is offered at most once per meeting. Prompts become invalid after one minute, when the endpoint disappears, or when that recording stops; a stale notification cannot change a later recording.
`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings and replays each WAV through a fresh speech pipeline. A replay without non-blank speech text leaves the artifacts unchanged and retains the backlog item and WAV for another retry. A usable replay preserves the existing transcript body and appends a recovery section explaining that the full recording was replayed and may duplicate earlier passages; the summary agent receives both the original and recovered text with that explanation. The worker then completes meeting metadata and summary generation and removes the backlog item and WAV.
`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV.
`Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings.
`Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text is written. Writing new transcript text also dismisses every outstanding inactivity notification and invalidates its actions. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with Yes, No, and `Pause transcription` actions, requests reminder-style toast behavior, keeps each stop reminder actionable for 1 minute, and automatically stops normally after 30 minutes. While transcription is intentionally paused, those prompts and the ordinary transcript-inactivity stop are fully suspended. A separate `MaximumPauseDuration`, defaulting to 4 hours, normally stops a meeting that remains continuously paused without showing an inactivity notification; it remains active when the ordinary inactivity safeguard is disabled, while a non-positive value disables the paused-session cutoff. Unpausing clears the continuous-pause timer and restarts transcript-inactivity timing from that moment. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When transcript inactivity stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding.
`Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text arrives. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with action buttons, requests reminder-style toast behavior, keeps each stop reminder actionable for 1 minute, and automatically stops normally after 30 minutes. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When the safeguard stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding.
| Setting | Purpose |
| --- | --- |
| `Enabled` | Enables transcript-inactivity prompts and `AutoStopAfter`; `MaximumPauseDuration` remains independent. |
| `Enabled` | Enables the inactivity safeguard for active recordings. |
| `FirstPromptAfter` | First transcript-inactivity duration before showing the stop prompt. |
| `ReminderPromptAfter` | Additional transcript-inactivity durations before showing another stop prompt. |
| `AutoStopAfter` | Transcript-inactivity duration after which Meeting Assistant stops the recording normally without prompting again. |
| `MaximumPauseDuration` | Maximum continuous transcription pause before Meeting Assistant stops the meeting normally without an inactivity notification; defaults to 4 hours, and a non-positive value disables it. |
| `InferredEndPadding` | Padding added to the last transcript timestamp, or meeting start when no transcript arrived, for safeguard-triggered end times. |
| `CheckInterval` | Polling interval for checking the active recording inactivity state. |
@@ -225,11 +219,11 @@ When `WhisperLocal:Diarization:Enabled` is true, the final post-processing pass
Active pyannote runtimes are warmed up on application start so image setup and model download do not wait for the first diarization request.
Pyannote runtime settings are shared by local Whisper finalization and speaker-identification validation. `WhisperLocal:Diarization:Enabled` controls the optional Whisper finalization pass; speaker-identification validation instead uses its single outer `SpeakerIdentification:PyannoteValidation:Enabled` switch.
Pyannote diarization settings are shared by local Whisper finalization and speaker-identification validation:
| Setting | Purpose |
| --- | --- |
| `Enabled` | Available under `WhisperLocal:Diarization` to enable the pyannote-backed Whisper finalization pass. It is not part of the nested speaker-validation runtime settings. |
| `Enabled` | Enables the pyannote-backed pass. |
| `DockerCommand` | Docker executable name or path. |
| `BaseImage` | Python base image used when building the local pyannote image. |
| `Image` | Local pyannote Docker image tag. |
@@ -270,9 +264,7 @@ Azure returns generic speaker IDs such as `Guest-1` and `Guest-2`, which Meeting
## Speaker Identification
Speaker identity matching keeps candidate samples only after a diarized speaker has at least `SpeakerIdentification:MinimumSampleSpeechDuration` of same-speaker audio, defaulting to 10 seconds. Consecutive lines with the same diarized speaker are combined even when any configured STT backend splits them around pauses. This applies to both live and final-only diarization. A different speaker ends the pending span, and `MaximumSampleDuration` caps every recognition WAV at 60 seconds by default.
Configuration is rejected when the minimum duration is negative, the maximum is not positive, or the minimum exceeds the maximum. The same validation applies to launch-profile overrides.
Speaker identity matching keeps candidate samples only after a diarized speaker has at least `SpeakerIdentification:MinimumSampleSpeechDuration` of continuous speech, defaulting to 30 seconds. Adjacent same-speaker segments may be combined when the gap is no larger than `MaximumSampleSegmentGap`, but a different speaker resets the pending span.
| Setting | Purpose |
| --- | --- |
@@ -280,12 +272,12 @@ Configuration is rejected when the minimum duration is negative, the maximum is
| `DatabasePath` | SQLite database path for identities, aliases, references, and samples. |
| `InitialDelay` | Delay after recording starts before the first live identity pass. |
| `Interval` | Interval between live identity passes. |
| `MatchBatchSize` | Number of identities processed per Azure model matching batch. Resemblyzer scores the full capped candidate set together so its ambiguity margin includes the global runner-up. |
| `MatchBatchSize` | Number of identities processed per model matching batch. |
| `MaxMatchCandidates` | Maximum known identities considered during one match pass. |
| `MatchIdentityActiveAge` | Age window for identities considered active enough for automatic matching. |
| `MaxSnippetsPerSpeaker` | Maximum stored WAV snippets retained per speaker identity by the existing backend. |
| `MinimumSampleSpeechDuration` | Minimum total diarized speaker audio needed in a same-speaker sample; provider-created pauses do not count toward it. |
| `MaximumSampleDuration` | Maximum duration of an extracted speaker-recognition WAV. Defaults to 60 seconds. |
| `MaxSnippetsPerSpeaker` | Maximum stored voice snippets retained per speaker identity. |
| `MinimumSampleSpeechDuration` | Minimum uninterrupted same-speaker speech span needed before storing a sample. |
| `MaximumSampleSegmentGap` | Maximum gap allowed when combining adjacent same-speaker segments into one sample. |
| `SilenceBetweenSnippetsSeconds` | Silence padding inserted between snippets during Azure identity matching. |
| `LiveSampleBufferDuration` | How long live transcript/audio material is retained for extracting samples. |
| `MergeRecentIdentityAge` | Age window used by diagnostics that merge recent duplicate identities. |
@@ -293,46 +285,14 @@ Configuration is rejected when the minimum duration is negative, the maximum is
`SpeakerIdentification:AzureSpeech` is an advanced nested override for speaker identity matching. It uses the same shape as `AzureSpeech` and lets identity matching use different Azure language, endpoint, or key settings than live transcription when needed. If it is left unset, the normal Azure Speech settings remain the practical default.
`SpeakerIdentification:PyannoteValidation` is an optional application-level secondary confidence layer and defaults to disabled. Its outer `Enabled` setting is the only validation toggle; the nested `Diarization` block contains runtime settings but no second enable switch. Launch profiles do not override speaker validation or its runtime settings. When enabled, pyannote rejects multi-speaker samples and must confirm an Azure-confirmed identity match before Meeting Assistant accepts it. It uses the same Docker-based pyannote runtime shape as local Whisper finalization and defaults the validation command timeout to 1 hour because local model setup can take substantial time.
`SpeakerIdentification:PyannoteValidation` is an optional secondary confidence layer. When enabled, pyannote rejects multi-speaker samples and must confirm an Azure-confirmed identity match before Meeting Assistant accepts it. It uses the same Docker-based pyannote runtime shape as local Whisper finalization and defaults the validation command timeout to 1 hour because local model setup can take substantial time.
| Setting | Purpose |
| --- | --- |
| `Enabled` | Sole switch for speaker-identity pyannote validation and its startup warm-up. |
| `MinimumSingleSpeakerCoverage` | Required fraction of the tested sample that pyannote must attribute to a single speaker. |
| `MinimumMatchingKnownSnippetRatio` | Required pyannote agreement ratio between the new sample and known snippets for an accepted identity. |
| `Diarization` | Nested pyannote settings used for this validation pass. |
`SpeakerIdentification:Resemblyzer` is a separate, application-level speaker-recognition backend and defaults to disabled. Setting its single `Enabled` flag to `true` selects the Resemblyzer identification and merge services for the process lifetime; Azure Speech and pyannote are then not used for identity matching or identity-match validation. Launch profiles do not override this choice. The normal `SpeakerIdentification:Enabled` setting remains the master switch for speaker identification as a whole.
The selected service reuses temporary WAV samples from diarized same-speaker runs, but starts a fresh span after every accepted sample so one speaker's vector inputs do not overlap. It waits for five samples by default before automatic matching; five is an eligibility threshold, not a storage cap. Additional samples for an unresolved speaker trigger another attempt at the configured interval, even when attendees and speaker labels have not changed. All distinct qualifying vectors from the run are retained up to the per-identity limit, including later evidence collected after a live match has already renamed the transcript. Providers that only diarize during finalization can extract missing non-overlapping samples from the completed mixed WAV. Resemblyzer runs locally in an application-managed Python virtual environment with CPU-only PyTorch, produces 256-value embeddings, and stores only versioned float32 vectors in the identity database; it does not persist the temporary WAV inputs as identity evidence. Explicit speaker assignments from the summary agent may save fewer than five available vectors.
For a query, the matcher first requires the mean pairwise cosine similarity of its vectors to meet `MinimumClusterCohesion`. It represents each compatible known identity by a normalized centroid, scores it with the median query-to-centroid cosine similarity, and accepts only if the best score meets `MinimumIdentitySimilarity` and beats the runner-up by `MinimumSimilarityMargin`. The runner-up margin is waived when only one identity is scoreable. Logs include measured scores and rejection reasons so these initial thresholds can be calibrated.
Once an identity has at least `OutlierPruningMinimumVectors` compatible vectors, a separate DBSCAN-style cosine-density pass protects the profile from mixed-speaker diarization errors. A vector is a neighbor when its cosine similarity meets `OutlierPruningNeighborSimilarity`, and a dense point requires `OutlierPruningMinimumNeighbors` neighbors including itself. Vectors outside the uniquely largest dense cluster are removed only when that cluster contains at least `OutlierPruningMinimumClusterRatio` of all compatible vectors. If there is no dense cluster, the largest clusters tie, or the ratio is too low, pruning fails safe and retains every vector. Invalid vectors and vectors produced by other model versions are preserved.
| Resemblyzer setting | Purpose |
| --- | --- |
| `Enabled` | Selects the complete local Resemblyzer identity backend. Defaults to `false`. |
| `RequiredVectorsPerSpeaker` | Independent vectors required for automatic matching and per-cluster merge validation. Defaults to `5`; this does not cap collection or persistence. |
| `MaxVectorsPerIdentity` | Maximum distinct vectors retained per identity. Defaults to `1000`; merges prefer the newest vectors. |
| `OutlierPruningMinimumVectors` | Compatible vectors required before density-cluster pruning runs. Defaults to `20`. |
| `OutlierPruningNeighborSimilarity` | Minimum cosine similarity for two vectors to count as density neighbors. Defaults to `0.75`. |
| `OutlierPruningMinimumNeighbors` | Neighbors required for a dense point, including the point itself. Defaults to `3`. |
| `OutlierPruningMinimumClusterRatio` | Minimum fraction of compatible vectors that the uniquely largest cluster must contain before other vectors are removed. Defaults to `0.60`. |
| `MinimumClusterCohesion` | Minimum mean pairwise cosine similarity within a query cluster. |
| `MinimumIdentitySimilarity` | Minimum median similarity from query vectors to a known identity centroid. |
| `MinimumSimilarityMargin` | Required difference between the best and runner-up identity scores. |
| `ModelId` | Version identifier persisted with vectors and required for compatible comparisons. |
| `PackageVersion` | Resemblyzer package version installed in the managed virtual environment. |
| `PythonCommand` | Python executable used to create the managed virtual environment. |
| `TorchVersion` | CPU-only PyTorch wheel version installed in the managed environment. |
| `TorchIndexUrl` | HTTPS package index used for the CPU-only PyTorch wheel. |
| `WebRtcVadVersion` | Version of the prebuilt Windows-compatible `webrtcvad-wheels` package. |
| `RuntimeFolder` | Local folder for versioned virtual environments, the encoder script, and short-lived WAV input batches. |
| `CommandTimeout` | Bound for first-time environment provisioning, warm-up, and encoder commands. |
The first enabled startup creates a content-versioned virtual environment under `RuntimeFolder`. Dependency-version changes select a new environment automatically. The workstation's global Python packages are not modified, and Docker is not required for Resemblyzer speaker recognition.
## Automation
`Automation:RulesPath` points to an optional local YAML rules file. The default `meeting-rules.local.yaml` is ignored by git. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, or transcript line writes; conditions are evaluated with NCalc-style expressions and step values can use Razor syntax against `Model.Meeting`, `Model.Event`, `Model.Speaker`, and `Model.Transcript`.
@@ -389,30 +349,11 @@ When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Cl
| `EnableCompaction` | Enables conversation compaction before requests exceed the configured context budget. |
| `CompactionRemainingRatio` | Remaining-context ratio that triggers compaction. |
| `ResponsesCompactPath` | Relative Responses compaction path, usually `responses/compact`. |
| `InitialPrompt` | Optional replacement for the default base summary instructions. Project metadata and approval guidance is appended to both default and custom prompts. |
| `InitialPrompt` | Optional complete replacement for the default summary-agent system instructions. |
After transcription has fully finished, Meeting Assistant automatically runs the summary pipeline for the meeting. The summary agent writes the full markdown summary through `write_summary` and must provide a required `oneliner` value, which is stored in summary frontmatter and must not contain line breaks.
The built-in summary-agent instructions treat assistant context as persistent meeting-specific memory. When the agent encounters an unexpected problem, missing information, or an assumption while summarizing, it appends a concise note through `write_context` so later work on the same meeting can use that history. A configured `Agent:InitialPrompt` replaces the built-in base instructions, so custom prompts must include equivalent memory guidance when this behavior is desired. Project metadata and approval guidance is always appended.
Every direct subfolder of `Vault:ProjectsFolder` is a project whose ID is its folder name. The summarizer maintains a root `PROJECT.md` for associated projects with YAML frontmatter such as:
```yaml
---
name: Alpha Platform
description: Meeting automation and shared project knowledge.
---
```
The display name comes from `name`; `description` is limited to 256 characters. Missing or malformed metadata falls back to the folder ID and an empty description. Bound project metadata is included in the initial prompt beside any root `AGENTS.md`, including when no instructions file exists. Both default and custom prompts instruct the agent to keep this metadata grounded and current while preserving other frontmatter and body notes.
`list_projects` returns the complete catalog as JSON entries with `id`, `displayname`, and `description`. This grants no file access. Reading, writing, searching, and retrieving historical summaries remain scoped to the meeting note's `projects` field. The summarizer uses its existing `write_projectfile` tool to maintain `PROJECT.md`; invalid resulting metadata is refused before writing.
`request_project_association(project_id, reason)` asks the user through a native Yes/No notification. The reason must be one nonempty line of at most 100 characters. The tool blocks the summarizer until the decision or a ten-minute timeout. Approval adds the canonical ID to the latest meeting note, preserves other metadata and notes, immediately enables project access, and returns the project's metadata and optional `AGENTS.md`. No, timeout, cancellation, or unavailable notifications grant no access. Expired notifications are removed and late actions have no effect. Already associated projects return context without another prompt.
`request_workflow_change(intention, detailed_prompt)` returns requested immediately. The intention is a nonempty single line of at most 100 characters; the detailed prompt should be a self-contained scenario or specification. Its native notification remains valid for ten minutes and can be accepted after the summary completes. Approval opens a separate settings-agent conversation and submits the complete prompt automatically. Declining or expiration opens nothing. Requests are held in memory and canceled on application shutdown. The summarizer itself receives no workflow-editing tools.
Once a stopped meeting reaches summarization, expiration of `Recording:StopProcessingTimeout` returns control while that run continues in the background. It does not cancel a summary waiting for project approval or queue that already-transcribed meeting for offline replay. The approval's own ten-minute deadline remains in effect.
The built-in summary-agent instructions treat assistant context as persistent meeting-specific memory. When the agent encounters an unexpected problem, missing information, or an assumption while summarizing, it appends a concise note through `write_context` so later work on the same meeting can use that history. A configured `Agent:InitialPrompt` completely replaces the built-in instructions, so custom prompts must include equivalent guidance when this behavior is desired.
The summary agent can add and remove meeting-note attendees when transcript or OCR evidence is clear. It can override transcript speaker labels only when the evidence is very certain, and it can delete wrongfully matched identities. Final speaker identity learning and candidate updates run after the summary pipeline finishes so they use the summary-refined attendee list and any recorded speaker identity changes.
-8
View File
@@ -1,7 +1,5 @@
# Meeting Workflow Engine
The interactive rules and identities editor can queue speaker samples for playback in the Windows build. The neutral build reports that playback requires the Windows build; it does not enqueue audio or initialize Windows audio devices.
Meeting Assistant has a small local workflow engine for meeting-specific automation. It is intended for rules that are too personal or environment-specific to hard-code, such as adding default attendees, cleaning meeting titles, binding projects, or adding context notes when known speakers are detected.
The workflow engine is deliberately narrow. It runs from a local YAML file, evaluates rules against the current meeting note read from disk, and applies a small set of meeting-safe mutations.
@@ -26,10 +24,6 @@ If the configured path is empty, missing, or points to a blank file, the workflo
The tray menu includes `Open agent`, which opens the `Meeting Summary Agent` chat window for this configured rules file, the local speaker identity database, appsettings configuration, and application logs. The assistant uses the summarizer agent configuration by default and can be overridden through `MeetingAssistant:WorkflowRulesEditor`.
The automatic summarizer can propose changes through `request_workflow_change(intention, detailed_prompt)` but has no direct workflow write tool. `intention` is one nonempty line of at most 100 characters; `detailed_prompt` contains the complete desired change, preferably as a scenario/specification. A Windows notification displays `Workflow change requested: ...` with Yes/No actions. The tool returns requested immediately so summarization continues. Yes opens a separate settings-agent window and automatically submits the complete detailed prompt; existing conversations and drafts are left intact. No or ten minutes without approval discards the request and removes the notification. Workflow requests can be approved after their summary finishes but are not persisted across application restarts. The settings agent applies approved requests through its normal validated tools.
Project association requests follow a separate blocking path. `request_project_association` waits up to ten minutes before the summarizer continues; approval appends the project ID to meeting frontmatter and returns its instructions. This does not emit a new workflow trigger or change the YAML rule schema. See the configuration guide for project catalog metadata and access rules.
```json
{
"MeetingAssistant": {
@@ -99,8 +93,6 @@ For every event, the engine:
For `transcript_line` events, the engine returns the possibly updated transcript line to the caller. Live transcript appends first write the original formatted line and keep a reference to that exact body line, then run transcript-line rules out of band. Later transcript segments can continue to be appended while the workflow runs. If rules return a changed line, Meeting Assistant rewrites the referenced line in the transcript file. If a transcript-line rule fails during live recording, Meeting Assistant logs the failure and keeps the original line so later transcript segments can continue to be written.
Offline transcription recovery applies `transcript_line` transformations to the recovered lines before appending them to the existing transcript. Earlier transcript content is preserved and is not transformed again. Each appended recovery section starts with a visible marker explaining that replaying the entire recording may produce duplicate passages, which the summary agent should treat as the same discussion. The marker bypasses transcript-line rules so the recovery explanation remains intact. A replay without non-blank speech text skips workflow events, transcript writes, and summarization, and retains its backlog item and WAV for another retry.
Rules are best-effort automation for live transcript persistence. The settings/logs write tool rejects invalid YAML, unknown steps, unsupported set-property fields, trigger or condition entries without a supported key, and step value Razor templates that fail against the workflow template model before writing the rules file. Invalid NCalc expressions can still fail at workflow runtime. Workflow execution logs every triggered rule with its rule name and event type, logs completion with whether the note or transcript line changed, and logs rule failures with the rule name and event type.
## YAML Shape
@@ -1,2 +0,0 @@
schema: spec-driven
created: 2026-09-02
@@ -1,102 +0,0 @@
## Context
The existing speaker-identification implementation stores bounded WAV snippets and sends composite audio to a dedicated Azure Speech diarization verifier, optionally followed by pyannote validation. Recording already maintains timestamped mixed audio and creates candidate WAV samples for diarized speaker labels. Identity names, aliases, candidate names, meeting references, transcript relabeling, summarizer overrides, deletion, and merges are stored locally in SQLite.
Resemblyzer 0.1.4 exposes a local `VoiceEncoder` that produces L2-normalized 256-value embeddings. Its upstream examples compare embeddings with dot products, which are cosine similarities for normalized vectors. The package includes its pretrained model but has Python, PyTorch, audio, and native VAD dependencies, so the application isolates them in a managed virtual environment rather than mutating the workstation Python installation.
The feature must remain opt-in and must not silently mix Resemblyzer vectors with evidence from the WAV/Azure backend. Existing identities remain shared because their names, aliases, references, and downstream behavior are backend-independent, but each backend reads and writes only its own voice evidence.
## Goals / Non-Goals
**Goals:**
- Select a separate local Resemblyzer recognition path with one application-level feature flag that defaults off.
- Convert temporary per-speaker WAV samples to versioned 256-float embeddings and persist only the embeddings for this path.
- Require five independent, coherent query embeddings before automatic matching.
- Make cohesion, acceptance similarity, ambiguity margin, required-vector count, runtime, and per-identity limit configurable.
- Preserve existing naming, attendee, relabeling, override, deletion, reference, and merge outcomes.
- Bound persisted embeddings to 1,000 per identity by default.
**Non-Goals:**
- Convert existing WAV snippets to Resemblyzer vectors automatically.
- Use Resemblyzer for ASR speaker diarization; diarized labels still come from the configured transcription backend.
- Run the Azure or pyannote speaker verifier as a second opinion when the Resemblyzer path is selected.
- Guarantee calibrated production thresholds before real meeting data has been observed.
## Decisions
### Select a complete backend at the application boundary
Add `SpeakerIdentification:Resemblyzer:Enabled`, defaulting to `false`. Dependency injection selects either the existing `SpeakerIdentityService`/`SpeakerIdentityMergeService` pair or a separate Resemblyzer identification/merge pair for the process lifetime. Resemblyzer configuration is application-level and launch profiles do not override it because the identity database and selected singleton backend are application-wide.
The alternative of adding conditional vector branches throughout the existing WAV service was rejected because it would make it easy to mix evidence types or accidentally invoke Azure/pyannote while the local backend is enabled.
### Reuse temporary sample capture but make vector samples independent
The recording run retains the configured number of best WAV samples in memory. Consecutive transcript lines with the same diarized speaker are treated as one same-speaker run regardless of which STT backend emitted them, even when that backend splits them around a pause; only an intervening different speaker or the configured maximum sample duration ends the run. Provider-created pauses remain inside the extracted time range but do not count toward its minimum speaker-audio duration. Samples are capped at 60 seconds by default. With Resemblyzer enabled, the collector starts a fresh span after each accepted sample so the five embeddings are based on non-overlapping speech. The WAV bytes are temporary inputs only and are not written to the identity database by the Resemblyzer service.
For providers that only yield speaker labels during finalization, the service extracts disjoint qualifying spans from the completed mixed WAV. Explicit summarizer assignments may learn from fewer than five valid vectors, but automatic matching and automatic unnamed-candidate learning wait for the configured required count.
`RequiredVectorsPerSpeaker` is an eligibility threshold, not a collection or persistence cap. The collector retains qualifying non-overlapping samples up to `MaxVectorsPerIdentity`, matching scores the configured minimum high-quality vectors, and an accepted live or final assignment persists every distinct compatible vector available for that meeting. A speaker matched during live transcription receives a final evidence-accumulation pass so samples collected after the initial match are not lost.
### Persist versioned float32 embeddings in a separate table
Add a `SpeakerVoiceVectors` table related to `SpeakerIdentities` with cascade deletion. Each row stores a little-endian float32 blob, dimension count, model identifier, SHA-256 fingerprint, and creation timestamp. The model identifier prevents comparisons across incompatible encoder versions. A unique identity/fingerprint index makes retrying the same evidence idempotent.
Vector rows are capped by `MaxVectorsPerIdentity`, default 1,000. Normal additions stop at the cap. Merges combine distinct rows and keep the most recently created vectors when the combined set exceeds the cap. WAV snippets and voice vectors remain independent collections.
### Run Resemblyzer in an application-managed Python virtual environment
`VenvResemblyzerVoiceEncoder` batches WAV files into one invocation of the managed virtual environment's Python executable, preprocesses each file with `preprocess_wav`, calls `VoiceEncoder("cpu").embed_utterance`, and returns marked JSON. The environment is content-versioned from its dependency settings, pins Resemblyzer and a CPU-only PyTorch wheel, and uses `webrtcvad-wheels` on Windows to avoid requiring Visual C++ build tooling. NumPy stays below 2 on Python versions where a compatible NumPy 1.x wheel exists and uses NumPy 2 on Python 3.13 or later. A non-blocking startup warm-up provisions and verifies the environment only when the feature is enabled.
The encoder validates result count, dimension, finite values, and nonzero magnitude before returning normalized vectors. Temporary input directories are deleted after each bounded invocation.
Installing packages into the workstation Python environment was rejected because it creates dependency conflicts and upstream `webrtcvad` requires native build tooling on clean Windows systems. The managed venv avoids both issues, while the compatible VAD wheel removes the compiler requirement. Docker was rejected because it adds an unnecessary VM/runtime dependency and caused CPU inference to pull multi-gigabyte CUDA packages from the default Linux PyTorch distribution. A long-lived inference service was deferred until measured process-start overhead warrants the extra lifecycle complexity.
### Use a coherent-query centroid heuristic with ambiguity rejection
All input vectors are normalized before scoring.
1. Query cohesion is the mean pairwise cosine similarity among the required query vectors. A query below `MinimumClusterCohesion` is rejected before identity comparison.
2. Each identity is represented by the normalized centroid of all stored vectors having the configured model identifier.
3. Candidate similarity is the median cosine similarity from the query vectors to that identity centroid. The median limits the effect of one noisy query sample.
4. The best candidate must meet `MinimumIdentitySimilarity` and exceed the runner-up by `MinimumSimilarityMargin`. The margin is waived when there is no runner-up.
Defaults are five vectors, `0.75` minimum cohesion, `0.75` minimum identity similarity, and `0.05` minimum margin. These are initial conservative values between the same-speaker and different-speaker similarities shown in Resemblyzer's upstream demonstrations; every threshold is configurable for calibration from local logs.
### Prune mature identity evidence with fail-safe density clustering
Once an identity has at least `OutlierPruningMinimumVectors` valid vectors for the configured model, defaulting to 20, run a separate DBSCAN-style clustering pass using cosine similarity. Two vectors are neighbors when their similarity meets `OutlierPruningNeighborSimilarity`; a dense point requires `OutlierPruningMinimumNeighbors`, including itself. This distinguishes isolated or small foreign-speaker groups without forcing every vector toward the matching centroid.
Pruning keeps a uniquely largest dense cluster only when it contains at least `OutlierPruningMinimumClusterRatio` of the compatible evidence, defaulting to 60%. Vectors outside that dominant cluster are removed, while incompatible-model and malformed rows are left untouched. If no dense cluster dominates, retain all evidence and log the ambiguity rather than arbitrarily selecting one voice. Run pruning after vector additions and identity merges; also prune before additions so a full identity can recover capacity previously occupied by outliers.
### Persist accepted evidence while keeping downstream identity behavior
When live or finished matching accepts a known identity, the query vectors and meeting reference are added immediately, bounded and deduplicated, and the existing canonical name is used for transcript relabeling and attendee updates. Final processing still performs candidate-name intersection/promotion and creates unmatched candidates using summary-refined attendees.
Summarizer overrides attach all available valid current-run vectors to the named identity, merge a current-run unnamed candidate when present, and create a named identity only when evidence or such a candidate exists. Identity deletion cascades to both evidence types.
Diagnostic automatic merge uses two disjoint query clusters and requires both to select the same target, preserving the existing two-pass confirmation rule. Manual merges always move bounded vector evidence along with aliases, candidates, references, and WAV snippets.
## Risks / Trade-offs
- [Initial thresholds may be too strict or permissive for mixed microphone/system audio] → Log cohesion, best similarity, runner-up similarity, margin, sample count, and rejection reason; expose every decision threshold in configuration.
- [Five independent 10-second samples can delay recognition] → Keep required count and minimum speech duration configurable; explicit summarizer assignments can seed an identity with fewer vectors.
- [Provider-created pauses can add silence to a same-speaker sample] → Bound every sample to 60 seconds and rely on Resemblyzer preprocessing to remove non-speech before embedding.
- [Existing identities have no vector evidence] → Do not cross-use WAV evidence automatically; identities become matchable after an explicit assignment or new vector-backed learning.
- [First-time virtual-environment provisioning and model startup add latency] → Pin CPU-only dependencies, content-version and reuse the environment, batch samples, warm non-blockingly, bound commands, and serialize encoder invocations to avoid concurrent model memory spikes.
- [A false positive can contaminate an identity with five vectors] → Require query cohesion, an absolute similarity threshold, an ambiguity margin, and two independent clusters for automatic merges.
- [Density clustering could discard a legitimate secondary acoustic mode] → Do not prune below 20 vectors or without a uniquely dominant 60% cluster; expose the neighborhood and dominance settings and log every decision.
- [Changing the encoder model invalidates comparisons] → Store and filter by model identifier; require an explicit configuration/migration decision for future model upgrades.
## Migration Plan
1. Apply the additive SQLite table/index migration while the flag remains disabled.
2. Provision and warm the configured local virtual environment, then enable Resemblyzer explicitly.
3. Calibrate thresholds from decision logs and corrected summarizer assignments.
4. Roll back by disabling the feature flag; the existing WAV/Azure backend and its stored snippets remain intact, while vector rows stay dormant.
## Open Questions
None.
@@ -1,31 +0,0 @@
## Why
The current speaker-recognition path persists WAV snippets and depends on Azure Speech plus optional pyannote validation. Meeting Assistant needs an opt-in, fully local alternative that persists compact voice embeddings and can accumulate stronger identity evidence over time without replacing the existing backend.
## What Changes
- Add an application-level feature flag that selects a separate Resemblyzer speaker-recognition backend while leaving the current WAV/Azure backend unchanged when disabled.
- Create temporary WAV samples during recording, encode each retained sample locally into a 256-value Resemblyzer voice vector, and persist vectors rather than WAV data for this backend.
- Require a configurable minimum of five coherent vectors for automatic recognition, compare their cluster with known identity vector clusters using configurable cosine-similarity, cohesion, and ambiguity thresholds, and learn the accepted vectors.
- Store at most a configurable 1,000 vectors per identity and retain vectors through identity naming, summarizer overrides, deletion, and merge operations.
- Keep transcript relabeling, attendee updates, candidate-name learning, meeting references, and identity-management behavior consistent with the existing speaker-identification flow.
- Add a managed local Python virtual environment for Resemblyzer with CPU-only PyTorch and document its configuration and tuning parameters.
- Merge consecutive transcript lines from any STT backend for the same diarized speaker into recognition samples despite provider-created pauses, while capping every sample at 60 seconds by default.
- Treat five vectors only as the default automatic-decision threshold, retain all qualifying current-run vectors up to the identity limit, and prune accumulated outliers with a configurable density-clustering pass once an identity has at least 20 compatible vectors.
## Capabilities
### New Capabilities
None.
### Modified Capabilities
- `meeting-transcription`: Add an opt-in local voice-vector speaker-recognition backend and define its collection, matching, persistence, and lifecycle behavior.
## Impact
- Speaker identity options, dependency registration, recording sample retention, and live/final identification orchestration.
- SQLite schema and identity merge/management tools gain a separate voice-vector collection.
- A local Python installation is required only when the feature is enabled; Resemblyzer and CPU-only PyTorch are isolated in an application-managed virtual environment.
- Canonical configuration and speaker-identification documentation gain the feature flag and tunable matching thresholds.
@@ -1,361 +0,0 @@
## ADDED Requirements
### Requirement: Speaker recognition can use local Resemblyzer voice vectors
Meeting Assistant SHALL expose `SpeakerIdentification:Resemblyzer:Enabled` as an application-level feature flag that defaults to disabled.
When the feature is disabled, Meeting Assistant SHALL use the existing WAV-snippet, Azure Speech, and optional pyannote speaker-identification backend without reading or writing Resemblyzer voice vectors.
When the feature is enabled, Meeting Assistant SHALL use a separate local Resemblyzer speaker-identification backend and SHALL NOT invoke the Azure Speech or pyannote speaker-identity matchers.
The Resemblyzer backend SHALL create temporary WAV samples from diarized same-speaker runs during recording, SHALL encode each retained sample locally as a versioned 256-value voice vector, and SHALL NOT persist those temporary WAV samples as identity evidence.
Resemblyzer sample spans retained for one speaker SHALL not overlap. For transcription providers that only produce diarized speakers during finalization, Meeting Assistant SHALL extract qualifying non-overlapping samples from the completed mixed recording.
Automatic matching SHALL wait until the configured required number of valid vectors is available for a diarized speaker. The default required count SHALL be five.
The required vector count SHALL be an automatic-decision threshold and SHALL NOT cap collection, encoding, or persistence. After the threshold is met, Meeting Assistant SHALL retain every distinct qualifying current-run vector up to the configured per-identity limit. When a speaker was assigned during live transcription, final processing SHALL attach qualifying vectors collected after that assignment to the same identity.
The matcher SHALL reject a query cluster whose mean pairwise cosine similarity is below the configured minimum cluster cohesion. For a coherent query, it SHALL represent each known identity by the normalized centroid of compatible stored vectors, SHALL score that identity using the median cosine similarity from query vectors to the centroid, and SHALL select an identity only when the best score meets the configured minimum identity similarity and exceeds the runner-up by the configured minimum similarity margin. The runner-up margin SHALL be waived when only one candidate can be scored.
The required vector count, minimum cluster cohesion, minimum identity similarity, minimum runner-up margin, encoder model identifier, local runtime settings, and command timeout SHALL be configurable.
When an identity has at least the configured outlier-pruning minimum number of valid vectors for the active model, defaulting to 20, Meeting Assistant SHALL run a separate cosine-density clustering pass. Neighbor similarity, minimum neighbors, and minimum dominant-cluster ratio SHALL be configurable.
Meeting Assistant SHALL remove vectors outside the uniquely largest dense cluster only when that cluster meets the configured minimum ratio of compatible evidence, defaulting to 60%. When no cluster qualifies or the largest cluster is tied, Meeting Assistant SHALL retain the evidence and log that pruning was skipped. Vectors for other model identifiers SHALL NOT be removed by this pass.
When enabled, the local encoder SHALL provision and reuse an application-managed Python virtual environment under the configured runtime folder. It SHALL install a pinned CPU-only PyTorch distribution and Windows-compatible VAD wheel without requiring Docker or a system-wide Python package installation.
The local encoder SHALL reject missing, malformed, non-finite, zero-magnitude, wrong-count, and wrong-dimension results without persisting them or falling back to the existing remote matcher.
#### Scenario: Disabled feature preserves existing backend
- **GIVEN** Resemblyzer speaker recognition is disabled
- **WHEN** Meeting Assistant tries to identify a diarized speaker
- **THEN** it uses the existing WAV-snippet speaker-identification backend
- **AND** does not create or compare Resemblyzer voice vectors
#### Scenario: Automatic matching waits for five vectors
- **GIVEN** Resemblyzer speaker recognition requires five vectors
- **AND** an unresolved diarized speaker has four valid samples
- **WHEN** live speaker identification runs
- **THEN** Meeting Assistant does not compare that speaker with known identities
- **WHEN** a fifth valid sample becomes available
- **THEN** Meeting Assistant can encode and compare the coherent five-vector cluster
#### Scenario: Five vectors do not cap retained evidence
- **GIVEN** Resemblyzer automatic matching requires five vectors
- **AND** a meeting yields eight distinct qualifying vectors for one speaker
- **WHEN** Meeting Assistant accepts or creates that speaker identity
- **THEN** it stores all eight vectors within the configured identity limit
#### Scenario: Final processing retains evidence collected after a live match
- **GIVEN** a diarized speaker was matched after five vectors during live transcription
- **AND** three more qualifying vectors were collected later in the meeting
- **AND** the finished transcript already uses the matched speaker's name while retained samples use the original diarized label
- **WHEN** final speaker processing runs with the existing mapping
- **THEN** the three later vectors are attached to the matched identity
#### Scenario: Mature identity outliers are pruned
- **GIVEN** an identity has at least 20 compatible vectors
- **AND** a uniquely largest cosine-density cluster contains at least 60% of them
- **WHEN** vector evidence is added or identities are merged
- **THEN** vectors outside the dominant cluster are removed
- **AND** the pruning decision and removed count are logged
#### Scenario: Ambiguous clusters are retained
- **GIVEN** an identity has at least 20 compatible vectors split between equally large or non-dominant dense clusters
- **WHEN** outlier pruning runs
- **THEN** Meeting Assistant removes no vectors
- **AND** logs that no uniquely dominant cluster qualified
#### Scenario: Incoherent query cluster is rejected
- **GIVEN** five query vectors have mean pairwise cosine similarity below the configured cohesion threshold
- **WHEN** Resemblyzer speaker identification runs
- **THEN** Meeting Assistant does not assign the speaker to a known identity
- **AND** logs the measured cohesion and rejection reason
#### Scenario: Similar and unambiguous cluster is accepted
- **GIVEN** a coherent five-vector query cluster
- **AND** its median similarity to Chris's vector centroid meets the configured identity threshold
- **AND** its score exceeds every other scored identity by the configured margin
- **WHEN** Resemblyzer speaker identification runs
- **THEN** Meeting Assistant identifies the diarized speaker as Chris
- **AND** adds the five query vectors to Chris's identity within the configured limit
#### Scenario: Ambiguous best cluster is rejected
- **GIVEN** a coherent five-vector query cluster meets the identity similarity threshold for Chris
- **AND** another identity's score is within the configured runner-up margin
- **WHEN** Resemblyzer speaker identification runs
- **THEN** Meeting Assistant leaves the diarized speaker unresolved
- **AND** logs both candidate scores and the insufficient margin
#### Scenario: Encoder failure preserves diarized labels
- **GIVEN** Resemblyzer speaker recognition is enabled
- **WHEN** the local encoder fails or returns invalid vectors
- **THEN** Meeting Assistant does not invoke the existing Azure or pyannote identity matcher as a fallback
- **AND** keeps the available diarized speaker labels
#### Scenario: Encoder provisions an isolated CPU environment
- **GIVEN** Resemblyzer speaker recognition is enabled
- **AND** its versioned virtual environment is not ready
- **WHEN** encoder warm-up runs
- **THEN** Meeting Assistant creates the virtual environment with the configured Python command
- **AND** installs the configured CPU-only PyTorch, Windows-compatible VAD, and Resemblyzer versions inside that environment
- **AND** does not invoke Docker
## MODIFIED Requirements
### Requirement: Speaker identity samples require uninterrupted speech
Meeting Assistant SHALL only retain speaker identity samples after a diarized speaker has produced a same-speaker sample span meeting the configured minimum duration.
The default minimum sample duration SHALL be 10 seconds.
Meeting Assistant SHALL combine consecutive transcript segments for the same diarized speaker into one sample span even when the transcription provider splits those segments around pauses. Provider-created pauses SHALL remain in the bounded extracted WAV but SHALL NOT count toward the configured minimum speaker-audio duration.
This aggregation behavior SHALL apply uniformly to diarized segments from every configured STT backend, whether segments arrive during live transcription or become available during finalization.
Meeting Assistant SHALL end the pending span when a different diarized speaker interrupts it or when the configured maximum sample duration is reached. The default maximum sample duration SHALL be 60 seconds, and no extracted recognition WAV SHALL exceed it.
When Resemblyzer recognition is enabled, Meeting Assistant SHALL start a new non-overlapping sample after accepting the previous sample from the same speaker.
#### Scenario: Short speaker span is not retained
- **GIVEN** the configured minimum sample duration is 10 seconds
- **WHEN** a diarized speaker produces only 8 seconds of uninterrupted speech
- **THEN** Meeting Assistant does not retain a speaker identity sample for that span
#### Scenario: Adjacent same-speaker segments form a sample
- **GIVEN** the configured minimum sample duration is 10 seconds
- **WHEN** a diarized speaker produces consecutive provider segments containing at least 10 seconds of speaker audio without another speaker interrupting
- **THEN** Meeting Assistant retains one speaker identity sample covering the continuous span
#### Scenario: Provider pause does not split a same-speaker sample
- **GIVEN** any configured STT backend emits consecutive lines for `Guest01` with a pause longer than the former segment-gap threshold
- **WHEN** no differently labeled speaker appears between those lines
- **THEN** Meeting Assistant combines the lines into one speaker-recognition sample span
- **AND** counts only their diarized speaker-audio durations toward the minimum
#### Scenario: Speaker sample is capped at 60 seconds
- **GIVEN** the maximum sample duration is 60 seconds
- **WHEN** consecutive transcript lines for one speaker span more than 60 seconds
- **THEN** every extracted speaker-recognition WAV is at most 60 seconds long
#### Scenario: Different speaker interrupts pending span
- **GIVEN** the configured minimum sample duration is 10 seconds
- **WHEN** `Guest01` speaks for 8 seconds and then `Guest02` speaks
- **THEN** Meeting Assistant discards the pending `Guest01` span instead of retaining or later extending it
### Requirement: Meeting Assistant learns speaker identities locally
Meeting Assistant SHALL maintain a local SQLite speaker identity database in the user's application data folder.
The speaker identity database SHALL store speaker identities, optional canonical names, aliases, candidate names, meeting file references, a bounded set of WAV snippets per identity for the existing backend, and a separate bounded set of versioned voice vectors per identity for the Resemblyzer backend.
Each persisted voice vector SHALL store its model identifier, dimension, creation time, and a fingerprint that makes adding the same vector to the same identity idempotent.
Meeting file references SHALL include the meeting note file address and the transcript file address.
Meeting Assistant SHALL calculate speaker identity participation counts from meeting file references when needed instead of persisting a denormalized transcript count.
Each speaker identity SHALL store a last-modified timestamp used by active-age filtering, and Meeting Assistant SHALL update it whenever the identity is created or modified by identification, candidate updates, snippet changes, voice-vector changes, reference changes, or merge operations.
The configured maximum snippet count and maximum voice-vector count per identity SHALL prevent unbounded growth. The default maximum voice-vector count SHALL be 1,000.
Except for adding newly accepted Resemblyzer match evidence and its meeting reference, final candidate elimination, canonical promotion, and new unmatched identity creation SHALL happen only after transcription is finished and after automatic summary generation has completed, using the latest meeting note frontmatter.
When the summary agent records a speaker override from a diarized transcript label to a named speaker, final speaker identity processing SHALL attach the current run's evidence to an existing identity with that name when one exists, or create a new canonical speaker identity with that name when none exists. For the existing backend that evidence SHALL be the resolved WAV snippet; for the Resemblyzer backend it SHALL be all available valid current-run voice vectors up to the configured per-run count. Meeting Assistant SHALL NOT create a new speaker identity for an override when no current run evidence or current run candidate can be resolved for the source speaker label.
When a speaker override maps a current-run unnamed candidate to an existing named identity, Meeting Assistant SHALL merge the candidate's meeting reference and useful backend-specific evidence into the named identity instead of leaving a duplicate candidate.
When the summary agent records that a speaker identity was wrongfully matched, final speaker identity processing SHALL delete the matching identity and all of its WAV and voice-vector evidence from the local speaker identity database so it cannot be matched again unless it is newly created in the future.
#### Scenario: Unknown speaker is learned from meeting attendees
- **WHEN** a finished transcript contains an unmatched diarized speaker and the meeting note has attendees
- **THEN** Meeting Assistant stores a new unnamed speaker identity with candidate names from the attendees that were not already matched in that meeting
- **AND** stores a meeting file reference for that identity
#### Scenario: Speaker snippets are bounded
- **WHEN** Meeting Assistant adds a snippet for an identity that already has the configured maximum number of snippets
- **THEN** Meeting Assistant does not store more snippets for that identity
#### Scenario: Speaker voice vectors are bounded
- **GIVEN** the Resemblyzer vector limit is 1,000
- **WHEN** Meeting Assistant adds vectors to an identity that already has 1,000 stored vectors
- **THEN** Meeting Assistant does not store more than 1,000 vectors for that identity
#### Scenario: Retried vector evidence is idempotent
- **GIVEN** an identity already contains a voice vector
- **WHEN** Meeting Assistant retries adding the same vector to that identity
- **THEN** it stores only one copy of that vector
#### Scenario: Identity modification updates active-age timestamp
- **WHEN** Meeting Assistant creates, identifies, updates candidates for, stores snippets or voice vectors for, stores references for, or merges a speaker identity
- **THEN** Meeting Assistant updates that identity's last-modified timestamp
#### Scenario: Final speaker identity learning uses summary-refined attendees
- **GIVEN** the summary agent changes meeting note attendees during automatic summary generation
- **WHEN** Meeting Assistant performs final speaker identity learning and candidate creation
- **THEN** it uses the attendee list from the meeting note after the summary agent changes
#### Scenario: Speaker override attaches to existing identity
- **GIVEN** the speaker identity database contains canonical speaker `Sabrina`
- **AND** the summary agent records that transcript speaker `Guest-01` is `Sabrina`
- **WHEN** final speaker identity processing runs
- **THEN** Meeting Assistant stores the meeting reference and current backend-specific speaker evidence on Sabrina's identity
- **AND** does not create a separate unnamed candidate for `Guest-01`
#### Scenario: Resemblyzer override stores available vectors
- **GIVEN** Resemblyzer speaker recognition is enabled
- **AND** the current run has three valid vectors for `Guest-01`
- **WHEN** the summary agent assigns `Guest-01` to `Sabrina`
- **THEN** Meeting Assistant attaches those three vectors to Sabrina's identity
- **AND** does not require five vectors for the explicit assignment
#### Scenario: Speaker override creates named identity
- **GIVEN** the speaker identity database has no accepted name `Sabrina`
- **AND** the summary agent records that transcript speaker `Guest-01` is `Sabrina`
- **WHEN** final speaker identity processing runs
- **THEN** Meeting Assistant creates a canonical speaker identity named `Sabrina`
- **AND** stores the meeting reference and current backend-specific speaker evidence on that identity
#### Scenario: Speaker override with missing source sample is skipped
- **GIVEN** the speaker identity database has no accepted name `Sabrina`
- **AND** the summary agent records that transcript speaker `Guest-5` is `Sabrina`
- **AND** final speaker identity processing has no sample, vector, or segment for `Guest-5`
- **WHEN** final speaker identity processing runs
- **THEN** Meeting Assistant does not create a speaker identity for `Sabrina`
#### Scenario: Speaker identity deletion removes a wrong match
- **GIVEN** the speaker identity database contains canonical speaker `Sabrina`
- **AND** the summary agent records that `Sabrina` was wrongfully matched
- **WHEN** final speaker identity processing runs
- **THEN** Meeting Assistant removes Sabrina's identity and backend-specific evidence from the speaker identity database
- **AND** the relabeled transcript uses `Removed-1` instead of `Sabrina`
### Requirement: Speaker identities can be merged diagnostically
Meeting Assistant SHALL expose a diagnostic endpoint that merges duplicate speaker identities.
The merge process SHALL compare recently-created identities, using a configurable recent age that defaults to two weeks, against all other identities using the selected backend's candidate-scoring strategy.
For the existing WAV backend, the merge process SHALL require a match and a second validation match using a different source sample. For the Resemblyzer backend, it SHALL require two disjoint coherent source-vector clusters to select the same target identity.
When identities are merged, Meeting Assistant SHALL retain one identity, move useful names from the merged identity into aliases, combine meeting file references, retain bounded sets of snippets and voice vectors from both identities, and append an audit line to each referenced transcript in the form `<date> <name 1> and <name 2> were merged`.
When combined Resemblyzer evidence exceeds the configured vector limit, Meeting Assistant SHALL keep no more than that limit, preferring the most recently created distinct vectors.
After combining Resemblyzer evidence, Meeting Assistant SHALL apply the configured mature-identity outlier-pruning policy.
#### Scenario: Recently-created duplicate identity is merged
- **GIVEN** a recently-created identity and an older identity have matching backend-specific speaker evidence
- **WHEN** the diagnostic merge endpoint is triggered
- **THEN** Meeting Assistant validates the match twice with different source evidence
- **AND** merges the recent identity into the older identity
- **AND** stores the recent identity name as an alias on the retained identity
- **AND** keeps meeting file references and bounded backend-specific evidence from both identities
- **AND** appends the merge audit line to the referenced transcripts
#### Scenario: Resemblyzer merge needs two clusters
- **GIVEN** Resemblyzer speaker recognition requires five vectors per cluster
- **AND** a recent identity has ten vectors split into two coherent clusters
- **WHEN** both clusters independently match the same target identity
- **THEN** Meeting Assistant merges the recent identity into that target
#### Scenario: Old identities are not used as merge sources
- **GIVEN** two identities older than the configured recent age
- **WHEN** the diagnostic merge endpoint is triggered
- **THEN** Meeting Assistant does not compare them as source identities
### Requirement: Speaker identity matches relabel transcripts
Meeting Assistant SHALL attempt to match unknown diarized speaker evidence against known speaker identities ordered by calculated meeting reference count.
When Resemblyzer speaker recognition is disabled, matching SHALL use the existing dedicated Azure Speech diarization verifier and optional pyannote validator with WAV snippets. When Resemblyzer speaker recognition is enabled, matching SHALL instead use only compatible locally calculated Resemblyzer voice-vector clusters.
For the existing WAV backend, the matcher SHALL test at most the configured batch size of known people per matching round and continue with later batches until a match is found or no candidates remain. The Resemblyzer backend SHALL score the capped candidate set together so ambiguity is measured against the global runner-up.
The matcher SHALL prioritize identities whose canonical name or aliases match current meeting attendees.
After attendee-matched identities, the matcher SHALL order identities by calculated meeting reference count, filter out non-attendee identities whose last update is older than the configured active age, and cap the candidate set at the configured maximum match candidate count.
When a match is confirmed, Meeting Assistant SHALL store a meeting file reference and the accepted backend-specific evidence for that identity within its configured limit.
When a match is confirmed and the identity has a canonical name, Meeting Assistant SHALL rewrite finished transcript segments for that diarized speaker with the canonical name.
When a match is confirmed and the matched speaker is not already listed in meeting note attendees by display name or alias, Meeting Assistant SHALL add the speaker display name to the attendee list.
When a match is confirmed and the meeting note attendees contain both the speaker display name and one or more accepted aliases for that same speaker, Meeting Assistant SHALL remove the alias attendee entries and keep the display name entry.
When Meeting Assistant writes attendees from calendar metadata, it SHALL match attendee display names exactly against known identity canonical names and aliases, replace matches with the identity display name, and deduplicate attendees that map to the same identity.
#### Scenario: Finished transcript is relabeled after a confirmed match
- **GIVEN** the speaker identity database contains canonical speaker `Chris`
- **WHEN** a finished transcript has diarized speaker `Guest03` and the selected matching backend confirms it is `Chris`
- **THEN** Meeting Assistant rewrites `Guest03` segments in the transcript as `Chris`
#### Scenario: Confirmed match stores meeting reference
- **GIVEN** the speaker identity database contains canonical speaker `Chris`
- **WHEN** a finished transcript has diarized speaker `Guest03` and the selected matching backend confirms it is `Chris`
- **THEN** Meeting Assistant stores the meeting note and transcript file addresses as a reference for `Chris`
- **AND** stores the accepted backend-specific evidence within its configured limit
#### Scenario: Confirmed match removes duplicate aliases
- **GIVEN** the speaker identity database contains canonical speaker `Christopher` with alias `Chris`
- **AND** the meeting note attendees contain both `Christopher` and `Chris <chris@example.com>`
- **WHEN** live or final speaker matching confirms a diarized speaker is `Christopher`
- **THEN** Meeting Assistant keeps `Christopher` in the meeting note attendees
- **AND** removes `Chris <chris@example.com>` from the meeting note attendees
### Requirement: Speaker matching runs during active transcription
Meeting Assistant SHALL start speaker identity matching only after the configured initial transcription duration has elapsed.
For backends that emit live diarized transcript segments, Meeting Assistant SHALL keep a bounded in-memory sliding audio buffer with chunk timestamps and extract candidate WAV samples from that buffer when live diarized segments arrive.
Meeting Assistant SHALL keep only the configured best candidate samples per diarized speaker in memory. Better samples SHALL be preferred when the segment looks like a continuous medium-length sentence. When Resemblyzer is enabled, accepted samples for one speaker SHALL be non-overlapping and the retained count SHALL be at least the configured required vector count.
Meeting Assistant SHALL periodically match unresolved diarized speaker evidence while transcription is active and attempt to match it against the local identity database.
Meeting Assistant SHALL run live matching incrementally at the configured interval only when at least one new unmapped diarized speaker sample appears or the meeting note attendee frontmatter changes while unmapped speaker samples still exist. For Resemblyzer, additional samples for an existing unresolved speaker SHALL also trigger another attempt so an earlier insufficient-vector result does not suppress matching when the required count becomes available.
When a speaker is matched during transcription, Meeting Assistant SHALL rewrite already-written live transcript segments for that diarized speaker and write future transcript segments using the canonical name.
For the existing WAV backend, live speaker matching SHALL be read-only with respect to the speaker identity database. For the Resemblyzer backend, a confirmed live match SHALL persist the accepted deduplicated voice vectors and meeting reference immediately so an assignment made during transcription is learned. Candidate elimination, canonical promotion, and new unmatched identity creation SHALL still happen only after transcription is finished and after automatic summary generation has completed, using the latest meeting note frontmatter.
For backends that only provide diarization after finalization, Meeting Assistant SHALL defer speaker identity matching until finished diarization is available, extract candidate samples from the completed temporary recording, complete identity matching, and only then allow summary generation to start.
#### Scenario: Matching waits for useful speech duration
- **WHEN** transcription has been active for less than the configured speaker identification initial delay
- **THEN** Meeting Assistant does not run speaker identity matching yet
#### Scenario: Live matching uses in-memory speaker samples
- **WHEN** a live diarized transcript segment identifies an unresolved speaker
- **THEN** Meeting Assistant extracts a temporary WAV sample for that segment from the in-memory sliding audio buffer
- **AND** uses retained speaker evidence for live identity matching without reading the temporary recording file
#### Scenario: Live match rewrites current and future transcript writes
- **WHEN** periodic matching confirms that diarized speaker `Guest03` is canonical speaker `Chris`
- **THEN** already-written live transcript segments for `Guest03` are rewritten as `Chris`
- **AND** later live transcript segments for `Guest03` are written as `Chris`
#### Scenario: Resemblyzer live match persists vectors
- **GIVEN** Resemblyzer speaker recognition is enabled
- **WHEN** periodic matching confirms a coherent five-vector cluster for `Guest03` as canonical speaker `Chris`
- **THEN** Meeting Assistant stores those vectors and the current meeting reference on Chris's identity
- **AND** does not persist the temporary WAV samples
#### Scenario: New live speaker triggers another identification round
- **GIVEN** live matching already checked the current unresolved speaker samples
- **WHEN** a new unmapped diarized speaker sample appears
- **THEN** Meeting Assistant runs another live matching round at the next configured interval
#### Scenario: Additional samples unlock live Resemblyzer matching
- **GIVEN** an earlier live attempt had fewer than five samples for an unresolved speaker
- **AND** no new speaker or attendee change occurs
- **WHEN** that speaker accumulates five qualifying samples
- **THEN** Meeting Assistant attempts matching again at the next configured interval
- **AND** does not repeatedly match unchanged evidence
#### Scenario: Attendee changes trigger another identification round
- **GIVEN** live matching already checked unresolved speaker samples
- **WHEN** the meeting note attendee frontmatter changes
- **THEN** Meeting Assistant runs another live matching round at the next configured interval using the latest attendees
#### Scenario: Final decisions use summary-refined attendees
- **WHEN** live matching finds or does not find a possible speaker identity during transcription
- **THEN** Meeting Assistant does not eliminate candidate names, promote canonical names, or create unmatched identities during that live pass
- **AND** the final speaker identity pass uses the latest meeting note attendees after transcription finishes
@@ -1,73 +0,0 @@
## 1. Voice-vector persistence
- [x] 1.1 Add a failing database behavior test for versioned, deduplicated voice vectors and cascade deletion
- [x] 1.2 Add the voice-vector entity, EF mapping, and additive SQLite schema migration
## 2. Local Resemblyzer encoding
- [x] 2.1 Add a failing encoder behavior test for batching WAV samples into validated 256-value vectors
- [x] 2.2 Implement the bounded local Resemblyzer encoder and non-blocking feature-gated warm-up
- [x] 2.3 Add behavior coverage for malformed, wrong-dimension, non-finite, and failed encoder results
## 3. Tunable cluster matching
- [x] 3.1 Add a failing behavior test for accepting a coherent, similar, unambiguous five-vector cluster
- [x] 3.2 Implement normalized-centroid, median-cosine, cohesion, threshold, and runner-up-margin scoring
- [x] 3.3 Add behavior coverage for insufficient, incoherent, below-threshold, and ambiguous clusters
## 4. Resemblyzer identity lifecycle
- [x] 4.1 Add a failing service behavior test proving a live vector match relabels the speaker and persists five vectors without WAV snippets
- [x] 4.2 Implement the separate Resemblyzer identification service with existing candidate ordering, naming, attendee, reference, and transcript outcomes
- [x] 4.3 Add and pass behavior tests for summary overrides with fewer than five vectors, unmatched learning, deduplication, and the 1,000-vector cap
## 5. Recording and merge integration
- [x] 5.1 Add behavior tests and implement non-overlapping Resemblyzer sample collection with at least the configured required count
- [x] 5.2 Add behavior tests and implement application-level feature selection without Azure/pyannote identity fallback
- [x] 5.3 Add behavior tests and implement two-cluster Resemblyzer diagnostic merging plus bounded vector retention in manual merges
- [x] 5.4 Expose vector counts in identity-management tools while keeping WAV playback operations separate
## 6. Configuration and verification
- [x] 6.1 Add the disabled-by-default canonical configuration and document runtime, persistence, and tuning behavior
- [x] 6.2 Run focused speaker, schema, encoder, matching, merge, recording, and workflow-tool tests
- [x] 6.3 Run the full solution test suite and validate the OpenSpec change strictly
## 7. Local virtual-environment correction
- [x] 7.1 Add a failing behavior test for provisioning a versioned venv with CPU-only PyTorch and no Docker command
- [x] 7.2 Replace the Docker encoder with managed-venv provisioning and direct venv Python batch encoding
- [x] 7.3 Replace Docker-specific Resemblyzer configuration and documentation with Python/venv settings
- [x] 7.4 Run focused and full tests, validate OpenSpec strictly, and verify enabled application warm-up through logs
## 8. Sample-duration tuning
- [x] 8.1 Add a failing configuration-default test for a 10-second minimum speaker sample
- [x] 8.2 Change the sample-duration default and canonical configuration to 10 seconds and update documentation
- [x] 8.3 Run focused tests, validate OpenSpec strictly, restart the enabled application, and verify health
## 9. STT segment aggregation and sample cap
- [x] 9.1 Add a failing live-collector behavior test proving consecutive same-speaker STT lines survive provider-created pauses
- [x] 9.2 Implement same-speaker aggregation for live and finalized samples while excluding provider pauses from the minimum speech duration
- [x] 9.3 Add a failing behavior test proving recognition WAVs never exceed the configurable 60-second default
- [x] 9.4 Implement and document the maximum sample duration across live and finalized collection
- [x] 9.5 Run focused and full tests, refactor, validate OpenSpec strictly, and verify operational readiness without interrupting active work
## 10. Complete evidence retention and outlier pruning
- [x] 10.1 Add a failing service behavior test proving five vectors unlock a decision without capping all qualifying current-run evidence
- [x] 10.2 Retain, encode, and persist all qualifying current-run vectors up to the configured identity limit, including evidence collected after a live match
- [x] 10.3 Add failing behavior tests for dominant density-cluster pruning and fail-safe ambiguous-cluster retention at the 20-vector floor
- [x] 10.4 Implement configurable cosine-density outlier pruning after vector additions and identity merges
- [x] 10.5 Document tuning settings, run focused/full tests and sequential refactor passes, validate OpenSpec strictly, and verify the enabled application without interrupting active work
## 11. Release verification corrections
- [x] 11.1 Reproduce and fix final evidence retention after live transcript relabeling
- [x] 11.2 Reproduce and fix live matching when an existing speaker reaches the required sample count
- [x] 11.3 Run focused/full tests, strictly validate OpenSpec, and verify operational readiness
- [x] 11.4 Split evidence-loading queries to avoid multiplying stored WAV blobs across vector, reference, and name rows
Release verification (2026-09-11): 128 initial focused tests passed. Both lifecycle regressions were reproduced and fixed; the full suite passed all 519 tests with compilation complete after two timing-sensitive audio tests failed during the concurrent Windows build and passed in isolation. The Windows target built successfully and strict OpenSpec validation passed. Local `/health` returned `ok`, recording status was idle, and application logs showed successful Resemblyzer warm-up, local encoding, and vector pruning. The release corrections were verified through behavior tests; the running workstation process was not restarted.
@@ -1,2 +0,0 @@
schema: spec-driven
created: 2026-09-25
@@ -1,35 +0,0 @@
## Context
Project IDs are direct subfolder names. BoundMeetingProjectResolver reads the meeting note on each access. The summarizer already has project read/write tools and an initial section for bound AGENTS.md instructions, but writes currently accept any existing project. Native actionable toasts and a WPF settings-agent window already exist.
## Goals / Non-Goals
**Goals:** Discover projects using compact metadata, extend project access only after approval, maintain metadata through the agent, and forward approved workflow requests to the interactive settings agent.
**Non-Goals:** Bulk-generate metadata for unassociated projects, let the summarizer edit workflows directly, change project IDs, migrate UI frameworks, or deploy/restart the workstation service.
## Decisions
- Store `name` and `description` in the root `PROJECT.md` frontmatter; `name` is the display name, and the directory name remains the stable ID. Missing/invalid metadata falls back to the ID and an empty description. Catalog results contain metadata, never AGENTS.md or arbitrary project content.
- Reuse write_projectfile for metadata maintenance. Validate resulting PROJECT.md frontmatter (name and description, maximum 256 description characters) before writing. Preserve ordinary file edit modes. Restrict writes to associated projects, matching existing read/search permissions.
- Append metadata and AGENTS.md at the existing initial project section. Add mandatory capability guidance even when a custom base prompt is configured. Instruct the agent to maintain metadata using grounded project knowledge and propose missing associations before finalizing the summary.
- Project approval is an awaited tool call, with cancellation and a ten-minute deadline. Return approval plus project metadata and AGENTS.md only after updating the latest meeting note. Preserve other frontmatter and body content; recheck target and cancellation before writing. Write to a temporary file in the same directory and replace the note only after that write completes, so cancellation or a partial write cannot truncate the note. Existing associations return current context without another prompt.
- Use an application-owned background request queue for workflow approvals. The tool returns requested immediately; the queue outlives a summary run. Requests also expire after ten minutes. Only acceptance opens a new settings-agent window and automatically submits the complete detailed prompt. A new conversation avoids overwriting drafts or interrupting an existing agent run.
- Reuse the installed native toast toolkit behind a testable approval-prompt interface. Timeouts and cancellation remove pending toasts and invalidate late actions. Shutdown cancels pending approvals; requests are not persisted across process restarts.
- Keep tool execution sequential so a blocking project approval cannot be bypassed by concurrent tool execution in the same model response.
## Risks / Trade-offs
- [User ignores project approval] → continue after ten minutes as denied; no project scope or note change.
- [User edits the note while a request is pending] → read again after acceptance and append only the project field, retaining all other current fields and notes.
- [Project disappears or becomes invalid while waiting] → refuse the association without expanding access.
- [Native UI cannot run in automated tests] → test approval, timeout, scope and handoff through controlled prompt/window services and build the Windows target; record hardware/UI verification limits.
- [Summarizer completes before workflow approval] → the application queue retains the request until decision, timeout, or shutdown.
## Migration Plan
No bulk migration. Existing folders remain discoverable without PROJECT.md, and associated projects gain metadata as the summarizer maintains them. Rollback removes the tools; PROJECT.md and normal meeting associations remain ordinary vault content.
## Open Questions
None. Workflow notifications use the same ten-minute validity as project notifications; workflow notification intentions are limited to 100 characters for readability.
@@ -1,25 +0,0 @@
## Why
The summarizer cannot currently discover missing project associations or propose workflow improvements with explicit user approval. Project folder names alone do not provide enough context for selecting the right project.
## What Changes
- Maintain `PROJECT.md` frontmatter with `name` and a description of at most 256 characters through the summarizer's existing project-file tools.
- Include available project metadata alongside bound project instructions and expose the full project metadata catalog through `list_projects`.
- Add `request_project_association` with a project ID and a reason of at most 100 characters. Wait up to ten minutes for a native Yes/No notification; approval adds the association and returns project instructions.
- Enforce meeting association for summarizer project-file writes as well as reads. Catalog visibility alone does not grant project access.
- Add nonblocking `request_workflow_change` with a short intention and detailed scenario/specification. Approval opens a settings-agent conversation and immediately submits the detailed request; the summarizer cannot edit workflows directly.
## Capabilities
### New Capabilities
- `summary-agent-requests`: User approval for blocking project association and asynchronous workflow-change handoff.
### Modified Capabilities
- `project-knowledge`: Project identity and discovery metadata in `PROJECT.md`.
- `agent-project-tools`: Full metadata catalog and association-scoped writes.
- `meeting-summary`: Project metadata in initial instructions and guidance for metadata maintenance and requests.
## Impact
Summary tools and agent instructions, project lookup and meeting frontmatter, native Windows notifications, the settings-agent window/prompt submission, DI registration, tests, and workflow/configuration documentation. Existing project IDs remain directory names; no bulk vault migration or live service restart is required.
@@ -1,91 +0,0 @@
## MODIFIED Requirements
### Requirement: Agents can use project context tools
Meeting Assistant SHALL expose tools that allow agents to look up project information, retrieve keyword-relevant context, and inspect meeting-derived knowledge.
Project tools SHALL treat each direct subfolder of the configured projects folder as one project. A meeting note binds projects by listing those subfolder names in the `projects` frontmatter field.
The summary agent SHALL expose these project tools:
- `list_projects`
- `request_project_association`
- `request_workflow_change`
- `list_projectfiles`
- `read_projectfile`
- `write_projectfile`
- `list_past_project_meetings`
- `read_past_project_meeting_summary`
- `search`
The `search` tool SHALL search the requested current-meeting project scope, or all projects assigned to the current meeting when no project scope is supplied.
The `search` tool SHALL search both configured project files for the scoped projects and past meeting summary notes in the configured summaries folder whose frontmatter projects intersect the scoped projects.
The `search` tool SHALL refuse requested project scopes that are not assigned to the current meeting.
#### Scenario: Agent looks up meeting project context
- **WHEN** a meeting note identifies a project
- **THEN** the agent can retrieve relevant project context through Meeting Assistant tools
#### Scenario: Agent discovers the full project catalog
- **WHEN** the agent calls `list_projects`
- **THEN** it receives every configured project with ID, display name, and description
- **AND** this catalog does not grant access to unassociated project files or AGENTS.md
#### Scenario: Agent reads a clamped project file range
- **WHEN** the agent reads a project file with optional line bounds outside the file length
- **THEN** `read_projectfile` clamps the requested range to the available file lines without failing
#### Scenario: Agent searches project knowledge
- **WHEN** the agent calls `search` with .NET regular expression keywords
- **THEN** Meeting Assistant runs the search against the requested projects or the meeting-bound projects and returns matches as `filename:line text`
#### Scenario: Agent searches past project meeting summaries
- **GIVEN** the current meeting note is assigned to `Project X`
- **AND** a prior summary note in the configured summaries folder is assigned to `Project X`
- **WHEN** the summary agent searches for a keyword
- **THEN** Meeting Assistant returns matches from both `Project X` project files and matching past summary notes
#### Scenario: Agent search rejects out-of-scope project
- **GIVEN** the current meeting note is assigned to `Project X`
- **WHEN** the summary agent searches with project scope `Project Z`
- **THEN** Meeting Assistant refuses the request because `Project Z` is not assigned to the current meeting
### Requirement: Agents can write files in existing projects
Meeting Assistant SHALL expose a `write_projectfile` tool that can create or update files inside an existing project folder.
The target project SHALL be an existing direct subfolder of the configured projects folder and SHALL be associated with the current meeting. The target file path SHALL stay inside that project folder. Newly approved associations SHALL take effect immediately for all project tools.
When no line edit arguments are supplied and `replace_file` is not true, `write_projectfile` SHALL append the supplied content to the file and SHALL create the file when it does not exist.
When `replace_file` is true, `write_projectfile` SHALL replace the whole file with the supplied content.
When both `from` and `to` are supplied, `write_projectfile` SHALL replace the inclusive 1-based line range with the supplied content.
When `insert` is supplied, `write_projectfile` SHALL insert the supplied content at that 1-based line position.
Meeting Assistant SHALL refuse ambiguous writes that combine `replace_file` with line edit arguments.
#### Scenario: Project file is appended or created
- **WHEN** the agent writes a project file without line edit arguments
- **THEN** Meeting Assistant appends the supplied content to an existing file
- **AND** creates the file with the supplied content when it does not exist
#### Scenario: Project file is explicitly replaced
- **WHEN** the agent writes a project file with `replace_file` set to true
- **THEN** Meeting Assistant writes the supplied content as the complete file content
#### Scenario: Project file line range is replaced
- **WHEN** the agent writes a project file with `from` and `to` line numbers
- **THEN** Meeting Assistant replaces the inclusive line range with the supplied content
#### Scenario: Project file content is inserted
- **WHEN** the agent writes a project file with an `insert` line number
- **THEN** Meeting Assistant inserts the supplied content at that line position
#### Scenario: Project file write target is invalid
- **WHEN** the target project is missing, is not associated with the meeting, or the target path escapes the project folder
- **THEN** Meeting Assistant refuses the project file write
#### Scenario: Project file write mode is ambiguous
- **WHEN** the agent combines `replace_file` with `from`, `to`, or `insert`
- **THEN** Meeting Assistant refuses the project file write
@@ -1,23 +0,0 @@
## MODIFIED Requirements
### Requirement: Summary agent receives bound project instructions
Meeting Assistant SHALL append a project section for projects bound to the meeting note frontmatter. Each entry SHALL identify the project ID and include its display name and description from PROJECT.md when available. For each bound project with a root AGENTS.md, the entry SHALL also include that file's content. Metadata SHALL appear even when AGENTS.md is absent. Unassociated project instructions SHALL NOT be appended.
#### Scenario: Project AGENTS files are appended
- **GIVEN** the meeting note frontmatter lists projects Alpha and Beta with root AGENTS.md files
- **WHEN** the summary agent is created
- **THEN** its project section includes both project IDs and their instructions
#### Scenario: Project metadata without instructions is appended
- **GIVEN** a bound project has PROJECT.md metadata but no AGENTS.md
- **WHEN** the summary agent is created
- **THEN** its entry contains ID, display name and description
## ADDED Requirements
### Requirement: Summarizer maintains project metadata and proposes missing associations
Both default and custom summary-agent instructions SHALL explain the full metadata catalog, the distinction between catalog visibility and project access, and the approval process. They SHALL instruct the agent to propose plausible missing associations with concise evidence before finalizing the summary and to maintain associated PROJECT.md name and description (maximum 256 characters). They SHALL explain that workflow improvements require a short intention and a detailed request and are executed only by the settings agent after user approval.
#### Scenario: Custom base prompt still receives capability guidance
- **WHEN** a custom summary base prompt is configured
- **THEN** the resulting instructions still explain metadata maintenance, association approvals and workflow-change requests
@@ -1,18 +0,0 @@
## ADDED Requirements
### Requirement: Projects have compact discovery metadata
Each project SHALL use a root `PROJECT.md` with YAML frontmatter `name` and `description` for agent-maintained discovery metadata. `name` SHALL be the display name; the project ID SHALL remain its direct subfolder name. The summarizer SHALL be instructed to create and maintain this metadata for associated projects from grounded project knowledge, preserving other frontmatter and body content. The description SHALL contain at most 256 characters.
Catalog lookup SHALL tolerate missing or malformed metadata by falling back to the ID as display name and an empty description. Metadata lookup SHALL NOT grant access to project files or instructions.
#### Scenario: Project has metadata
- **WHEN** a project has valid name and description frontmatter
- **THEN** catalog lookup exposes its ID, display name, and description
#### Scenario: Legacy project has no metadata
- **WHEN** a project has no PROJECT.md or no usable frontmatter
- **THEN** it remains discoverable with its ID as display name and an empty description
#### Scenario: Agent writes invalid metadata
- **WHEN** the summarizer attempts a PROJECT.md update without valid name/description or with a description longer than 256 characters
- **THEN** the write is refused without changing the existing file
@@ -1,56 +0,0 @@
## ADDED Requirements
### Requirement: Summarizer can request project association
The summarizer SHALL expose `request_project_association` accepting an existing project ID and a nonempty single-line reason of at most 100 characters. Invalid requests SHALL be refused without a notification or file changes.
The tool SHALL show a native notification identifying the meeting and project, explaining the association request and reason, with Yes/No actions. Tool completion and further summarizer work SHALL wait for the decision for up to ten minutes. No, expiration, notification unavailability, or canceled summarization SHALL NOT add the project. Expiration SHALL remove the notification, invalidate late activation, and return denial.
Once a stopped run has reached summarization, the recording stop-processing deadline SHALL return control while summary processing continues in the background. It SHALL NOT cancel a summary awaiting approval or queue an already-transcribed recording for offline replay.
On Yes, the tool SHALL append the canonical project ID to the latest meeting-note projects frontmatter without duplicates, preserving other fields and body. Project tools SHALL immediately honor the updated scope, and the result SHALL report approval with project metadata and the root AGENTS.md content when present. No unapproved AGENTS.md content SHALL be returned. Already associated projects SHALL return their current context without prompting.
#### Scenario: Approval extends scope during the same summary
- **GIVEN** a catalog project is not associated with the meeting
- **WHEN** the summarizer requests association and the user chooses Yes
- **THEN** the meeting note gains that project and the same agent can read/write it
- **AND** the result includes its AGENTS.md if present
#### Scenario: Denial or timeout preserves scope
- **WHEN** the user chooses No or ten minutes elapse
- **THEN** the waiting tool reports denial, the notification disappears, and scope and meeting note remain unchanged
#### Scenario: User edits note during approval
- **WHEN** the user changes other meeting frontmatter or notes while an association request waits
- **AND** then approves the project
- **THEN** the project is added without losing those edits
#### Scenario: Invalid or stale request cannot grant access
- **WHEN** the ID is unknown, reason exceeds 100 characters, the target disappears, or summarization is canceled
- **THEN** the request cannot expand project access
#### Scenario: Stop wait expires while summary approval is pending
- **GIVEN** transcription has completed and summarization is waiting for a user decision
- **WHEN** the recording stop-processing timeout expires
- **THEN** stopping returns with summarization still active
- **AND** the summary continues with its own approval deadline without an offline transcription backlog item
### Requirement: Summarizer can request workflow changes without editing workflows
The summarizer SHALL expose `request_workflow_change` accepting a nonempty single-line intention of at most 100 characters and a nonempty detailed change prompt, preferably a scenario or specification. The tool SHALL return requested without waiting for approval, and the summarizer SHALL continue normally without direct workflow-editing capabilities.
A native notification SHALL show `Workflow change requested: ...` with the intention and Yes/No actions. The detailed prompt SHALL be retained by the application independently of summary completion. No or ten-minute expiration SHALL discard the request without opening the editor or changing workflows. Late activation SHALL have no effect.
Yes SHALL open a new settings-agent conversation and immediately submit the full detailed prompt once. An existing open conversation or draft SHALL remain intact. Application shutdown SHALL cancel and remove pending notifications.
#### Scenario: Workflow request does not block the summary
- **WHEN** the agent requests a workflow change
- **THEN** the tool returns requested while approval remains pending
- **AND** the summary can complete without opening the settings agent
#### Scenario: Approved workflow request starts settings agent
- **GIVEN** the summary run has completed and its workflow request is pending
- **WHEN** the user chooses Yes
- **THEN** the settings-agent window opens and starts the detailed prompt exactly once
#### Scenario: Denied or expired workflow request does nothing
- **WHEN** the user declines or ignores a workflow request for ten minutes
- **THEN** its notification disappears and no editor or workflow change is started
@@ -1,35 +0,0 @@
## 1. Project catalog and metadata
- [x] 1.1 Test and implement full metadata catalog with legacy/malformed fallback and scoped file access.
- [x] 1.2 Test PROJECT.md metadata writes and enforce the description limit without damaging existing files.
- [x] 1.3 Include bound metadata beside AGENTS.md and instruct default/custom agents to maintain it and request missing associations.
## 2. Project approval
- [x] 2.1 Test blocking approval, immediate scope update, returned AGENTS.md and preservation of concurrent user note edits.
- [x] 2.2 Handle invalid IDs/reasons, existing associations, denial, timeout, cancellation and stale actions.
- [x] 2.3 Register the tool and native Yes/No notification with ten-minute validity and sequential summary tool execution.
- [x] 2.4 Keep summarization alive after the recording stop wait expires so approval does not trigger transcription cancellation or offline replay.
## 3. Workflow-change requests
- [x] 3.1 Test and implement nonblocking application-owned workflow requests that survive summary completion.
- [x] 3.2 Open a separate settings-agent conversation and automatically submit approved detail; preserve existing drafts and runs.
- [x] 3.3 Cover rejection, expiration, shutdown, invalid requests and exactly-once acceptance.
## 4. Verification
- [x] 4.1 Update workflow and configuration documentation.
- [x] 4.2 Review changed code and surrounding integration; fix actionable findings.
- [x] 4.3 Run relevant tests, full solution tests, Windows build and strict OpenSpec validation.
- [x] 4.4 Verify through the safest direct operational surface and document native UI/live-provider limits.
Verification (2026-09-25): all 552 solution tests pass, including 20 focused project/catalog/request tests. The solution builds the Windows target successfully with the four existing NAudio obsolete-API warnings. Strict validation passes for this change and the existing microphone change; `git diff --check` is clean.
Two independent review passes covered project metadata/access and notification/queue/UI lifecycle, including surrounding recording-stop integration. Incomplete metadata now consistently falls back to the project ID, and traversal coverage uses an associated project so it actually exercises path containment. Project association writes prepare a same-directory temporary file before replacing the meeting note; successful requests leave no temporary file behind.
Public-tool behavior tests inspect real generated meeting and project files with controlled approval/window services. They verify latest-note preservation, immediate scope changes, returned instructions, rejection/expiry/cancellation, nonblocking workflow requests, and automatic submission of approved detail to a fresh settings-agent conversation. A coordinator regression test verifies that the stop wait cannot cancel a summary awaiting approval or enqueue unnecessary offline replay.
The running application's recording-status endpoint was checked read-only and reported idle. The service was not updated or restarted. Native toast activation/removal, actual WPF window isolation, and a live model invoking these tools were not exercised, because those require running the new Windows build and interacting with its notifications. Sequential tool execution is configured and compiled; the model pipeline itself was not exercised with simultaneous tool calls. These operational checks remain required before acceptance or archival.
Authorized local start (2026-09-25): after confirming idle status and logs, the documented restart helper published and started the Release build in `tmp/meeting-assistant-runtime/run-20260925-115711`. `/health` reports `ok`, `/recording/status` reports idle, the tray icon was created, and Resemblyzer warm-up completed without stderr errors. The published application DLL matches the Release output by SHA-256. This verifies startup of the new build; native approval interactions and live-model tool invocation remain unverified.
@@ -1,2 +0,0 @@
schema: spec-driven
created: 2026-09-02
@@ -1,69 +0,0 @@
## Context
Meeting Assistant currently has one active `RecordingRun` that owns its speech-recognition pipeline, transcript session, temporary WAV, speaker mappings, and live speaker samples. Mixed audio chunks are written to both the WAV and the active pipeline. Transcript inactivity is tracked separately, and Windows inactivity toasts share a notification group but expose only stop/continue callbacks.
The Azure Speech SDK `ConversationTranscriber` has start and stop operations but no pause operation. Stopping ends the ongoing real-time recognition session, and recreating or restarting recognition can reset backend speaker IDs. Continuous recognition does support an open push stream containing silence, so the provider session can remain alive without receiving the meeting's actual audio during a pause.
## Goals / Non-Goals
**Goals:**
- Pause the transcription of an active meeting without finishing its recording run.
- Prevent actual paused audio from reaching either durable temporary audio or a remote/local transcription backend.
- Preserve the same recognition pipeline, Azure conversation transcriber, transcript session, speaker mappings, collected speaker samples, and artifacts.
- Keep normal finish, cancel/discard, and profile-switch controls usable while paused.
- Treat intentional pause as distinct from inactivity and dismiss notifications that become obsolete when transcript text resumes.
**Non-Goals:**
- Suspend microphone or loopback device capture at the operating-system layer.
- Disconnect or stop the configured speech-recognition backend while paused.
- Reduce Azure connection time or billing during a pause.
- Persist pause state across process restarts or stopped meeting backlog replay.
- Add a pause hotkey.
## Decisions
### Preserve the pipeline by substituting silence at the recording-run boundary
For each mixed audio chunk captured while paused, the coordinator will create an equal-length zeroed PCM chunk and route that chunk to the temporary WAV, live speaker audio buffer, and current speech-recognition pipeline. Real captured samples are discarded at that boundary.
This keeps audio duration and transcript timestamps aligned while keeping the Azure push stream active. Dropping chunks entirely was rejected because a sufficiently long input gap can stop or reconnect a provider session. Calling `StopTranscribingAsync` was rejected because it terminates the ongoing Azure operation and cannot guarantee stable backend speaker IDs when started again.
### Keep pause state on the active recording run
The run will expose a thread-safe paused flag and coordinator pause/unpause operations. `RecordingStatus` will expose the flag so tray rendering and the loopback status endpoint observe the same state. Pause requests outside an active capture will be harmless, and normal stop/abort paths will remain authoritative.
The paused flag will not become another post-recording process state: a paused run is still an active recording and continues to offer Finish and Cancel.
### Separate transcript inactivity from a maximum continuous pause
The inactivity safeguard will skip transcript-inactivity prompting and its ordinary auto-stop while the run is paused. Pausing will dismiss all outstanding inactivity notifications. A separate `MaximumPauseDuration`, defaulting to four hours, will normally stop a run that remains continuously paused for that duration without showing inactivity notifications. Unpausing will clear the paused-duration timer, set a new transcript-inactivity baseline, and advance the activity version so prompt thresholds start over rather than firing immediately.
### Make notification lifecycle part of the prompt-service contract
`IMeetingInactivityPromptService` will gain an operation to dismiss all active inactivity prompts. The Windows implementation will remove the notification group from Action Center and clear its pending callback registry. The no-op and test implementations will implement the same public contract.
The coordinator will call dismissal only after a non-empty live segment has been durably appended, matching the user's observable meaning of a new transcription being written. Notification callbacks will verify that their originating run is still current before changing recording state.
Transcript processing requests notification dismissal without awaiting it. Profile switching holds the coordinator gate while draining the old transcript reader, so awaiting dismissal from that reader would create a circular wait on the same gate. Deferred cleanup retains the gate and current-run check, uses the capture cancellation token, and handles cancellation when capture stops. Durable transcript activity still invalidates stale notification actions immediately, before deferred cleanup runs.
### Model pause as one toggling tray action
The active-recording tray menu will show `Pause transcription` while running and `Unpause transcription` while paused. It will stay in the fine-grained controls section below the dedicated `Finish meeting` section. The inactivity toast will add `Pause transcription` alongside the existing Yes/No stop controls.
## Risks / Trade-offs
- **Azure may still reconnect for unrelated transport failures during a long pause** → Reuse the existing Azure reconnect behavior; the application preserves meeting artifacts and resets backend speaker assumptions only if Azure actually creates a new SDK session.
- **Silence consumes provider connection time and temporary WAV space** → Accept this to preserve provider/session continuity and timestamp alignment; document that pause is not a cost-suspension mechanism.
- **A forgotten pause could otherwise keep capture and provider resources alive indefinitely** → Apply the separate four-hour maximum continuous pause while keeping the shorter transcript-inactivity prompts fully suppressed.
- **A transcript result already buffered by the provider can arrive just after pause** → Keep and write it because it represents audio submitted before the pause boundary.
- **A stale notification action could affect a later meeting** → Clear pending callbacks on dismissal and verify the originating run before applying stop or pause.
## Migration Plan
No data or configuration migration is required. Deploy the updated executable normally. Rollback restores the previous controls; existing meeting artifacts and speaker identities remain compatible.
## Open Questions
None.

Some files were not shown because too many files have changed in this diff Show More