Author SHA1 Message Date
dh 2af7933a1b feat(macos): enable meeting integrations
PR and Push Build/Test / build-and-test (pull_request) Failing after 19m49s
2026-08-06 08:19:26 +02:00
dh 9542c05ea4 feat(macos): add native desktop controls and hotkey integration
- Introduced MacOsDesktopControlManifest to manage desktop control configurations.
- Implemented MacOsDesktopControlService to handle the lifecycle of the macOS helper process.
- Created MacOsWorkflowRulesEditorWindowService for managing workflow rules through a native window.
- Developed a Swift helper for macOS that integrates with AppKit and Carbon for menu-bar controls and global hotkeys.
- Added support for recording lifecycle actions via HTTP endpoints, maintaining consistency with existing Windows functionality.
- Enhanced user experience by allowing macOS users to start/stop recordings and access settings through a menu-bar interface.
- Documented design decisions, implementation evidence, and verification tasks for the new macOS features.
2026-08-06 08:18:52 +02:00
dh 5f39923c27 docs: archive macOS meeting audio change 2026-08-06 08:16:55 +02:00
dh 13a70c2375 feat: detect host operating system at startup and provide platform-specific guidance 2026-08-06 08:16:55 +02:00
dh 59ba70ca11 feat: add macOS meeting audio capture 2026-08-06 08:16:54 +02:00
173 changed files with 5278 additions and 10089 deletions
@@ -1,7 +1,6 @@
using System.Net;
using System.Net.Http.Json;
using MeetingAssistant.Recording;
using MeetingAssistant.Workflow;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
@@ -43,14 +42,16 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
}
[Fact]
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
public async Task ApplicationStartupRegistersDetectedHostOperatingSystemOnce()
{
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
using var client = factory.CreateClient();
using var response = await client.GetAsync("/health");
var first = factory.Services.GetRequiredService<HostOperatingSystem>();
var second = factory.Services.GetRequiredService<HostOperatingSystem>();
var result = await playback.QueueAsync(1, [1, 2]);
Assert.Contains("Windows build", result);
Assert.DoesNotContain("Queued", result);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Same(first, second);
Assert.Equal(HostOperatingSystem.DetectCurrent(), first);
}
[Fact]
@@ -58,64 +58,6 @@ public sealed class LaunchProfileOptionsProviderTests
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
}
[Fact]
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
{
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
}
[Fact]
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
}
[Fact]
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
}
[Fact]
public void ContradictorySpeakerSampleDurationsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
Assert.Contains("MaximumSampleDuration", exception.Message);
}
[Fact]
public void DuplicateProfileHotkeysAreRejected()
{
@@ -2,8 +2,6 @@ using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Drawing;
using System.Drawing.Imaging;
using MeetingAssistant.Screenshots;
using Microsoft.Extensions.Logging.Abstractions;
@@ -284,15 +282,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
private static byte[] CreatePngBytes(int width, int height)
{
using var bitmap = new Bitmap(width, height);
using (var graphics = Graphics.FromImage(bitmap))
{
graphics.Clear(Color.White);
}
using var stream = new MemoryStream();
bitmap.Save(stream, ImageFormat.Png);
return stream.ToArray();
return PngTestFixture.Solid(width, height);
}
private static string CreateNonStreamingTextResponse(string text)
@@ -0,0 +1,89 @@
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.MacOs;
using MeetingAssistant.Workflow;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.DependencyInjection;
namespace MeetingAssistant.Tests;
public sealed class MacOsDesktopControlManifestTests
{
[Fact]
public void MacOsApplicationRegistersARealInteractiveAgentWindow()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
using var factory = new WebApplicationFactory<Program>();
var service = factory.Services.GetRequiredService<IWorkflowRulesEditorWindowService>();
Assert.IsType<MacOsWorkflowRulesEditorWindowService>(service);
}
[Fact]
public void ManifestPreservesConfiguredWindowsEquivalentShortcutsAndControlEndpoints()
{
var profiles = new[]
{
Profile("default", "Ctrl+Alt+M", "Ctrl+Alt+Z", "Ctrl+Alt+S"),
Profile("english", "Ctrl+Alt+L", "", "")
};
var hotkeys = new[]
{
new LaunchProfileHotkey("default", "Ctrl+Alt+M", LaunchProfileHotkeyAction.ToggleRecording),
new LaunchProfileHotkey("default", "Ctrl+Alt+Z", LaunchProfileHotkeyAction.AbortRecording),
new LaunchProfileHotkey("default", "Ctrl+Alt+S", LaunchProfileHotkeyAction.CaptureScreenshot),
new LaunchProfileHotkey("english", "Ctrl+Alt+L", LaunchProfileHotkeyAction.ToggleRecording)
};
var manifest = MacOsDesktopControlManifest.Create(
"http://127.0.0.1:5090",
profiles,
hotkeys);
Assert.Collection(
manifest.Hotkeys,
hotkey =>
{
Assert.Equal("Ctrl+Alt+M", hotkey.Shortcut);
Assert.Equal("/profiles/default/recording/toggle", hotkey.Endpoint);
},
hotkey =>
{
Assert.Equal("Ctrl+Alt+Z", hotkey.Shortcut);
Assert.Equal("/profiles/default/recording/abort", hotkey.Endpoint);
},
hotkey =>
{
Assert.Equal("Ctrl+Alt+S", hotkey.Shortcut);
Assert.Equal("/profiles/default/meetings/screenshot/capture", hotkey.Endpoint);
},
hotkey =>
{
Assert.Equal("Ctrl+Alt+L", hotkey.Shortcut);
Assert.Equal("/profiles/english/recording/toggle", hotkey.Endpoint);
});
Assert.Equal("http://127.0.0.1:5090/recording/status", manifest.StatusUrl);
Assert.Equal("http://127.0.0.1:5090/diagnostics/settings-and-logs", manifest.OpenAgentUrl);
}
private static LaunchProfile Profile(string name, string toggle, string abort, string screenshot)
{
return new LaunchProfile(name, new MeetingAssistantOptions
{
Hotkey = new HotkeyOptions
{
Toggle = toggle,
Abort = abort
},
Screenshots = new ScreenshotOptions
{
Hotkey = screenshot
}
});
}
}
@@ -0,0 +1,415 @@
using MeetingAssistant.Recording;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging.Abstractions;
using System.Net.Http.Json;
namespace MeetingAssistant.Tests;
public sealed class MacOsMeetingAudioSourceTests
{
[Fact]
public async Task MacOsMicrophoneStreamsNativePcmUsingRunFormat()
{
var pcm = Pcm16(1_000, -2_000, 3_000, -4_000);
var process = new StubMacOsAudioCaptureProcess(pcm, exitCode: 0);
var processFactory = new StubMacOsAudioCaptureProcessFactory(process);
IMeetingAudioSource source = new MacOsMicrophoneAudioSource(
processFactory,
NullLogger<MacOsMicrophoneAudioSource>.Instance);
var options = new MeetingAssistantOptions
{
Recording =
{
SampleRate = 24_000,
Channels = 2
}
};
var chunks = await ReadChunksAsync(source, options);
var chunk = Assert.Single(chunks);
Assert.Equal(pcm, chunk.Pcm);
Assert.Equal(24_000, chunk.SampleRate);
Assert.Equal(2, chunk.Channels);
Assert.Equal(
new MacOsAudioCaptureRequest(MacOsAudioCaptureKind.Microphone, 24_000, 2),
processFactory.Request);
Assert.True(process.Disposed);
}
[Fact]
public async Task MacOsSystemAudioReportsNativePermissionFailure()
{
var process = new StubMacOsAudioCaptureProcess(
[],
exitCode: 77,
standardError: "Screen Recording/System Audio permission is required.");
IMeetingAudioSource source = new MacOsSystemAudioSource(
new StubMacOsAudioCaptureProcessFactory(process),
NullLogger<MacOsSystemAudioSource>.Instance);
var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
await ReadChunksAsync(source, new MeetingAssistantOptions()));
Assert.Contains("Screen Recording/System Audio permission", exception.Message, StringComparison.Ordinal);
}
[Fact]
public void PortableRegistrationUsesMacOsCompositeOnlyForMacOs()
{
var macServices = new ServiceCollection().AddLogging();
var unsupportedServices = new ServiceCollection().AddLogging();
PortableMeetingAudioRegistration.Add(macServices, isMacOs: true);
PortableMeetingAudioRegistration.Add(unsupportedServices, isMacOs: false);
using var macProvider = macServices.BuildServiceProvider();
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
Assert.IsType<CompositeMeetingAudioSource>(macProvider.GetRequiredService<IMeetingAudioSource>());
Assert.IsType<UnavailableMeetingAudioSource>(unsupportedProvider.GetRequiredService<IMeetingAudioSource>());
}
[Fact]
public async Task RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses()
{
var testRoot = Path.Combine(Path.GetTempPath(), $"meeting-assistant-macos-endpoint-{Guid.NewGuid():N}");
var nativeProcesses = new EndpointMacOsAudioCaptureProcessFactory(
microphonePcm: Pcm16(2_000),
systemPcm: Pcm16(10_000));
var speechPipelines = new CapturingSpeechRecognitionPipelineFactory();
await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, configuration) =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["MeetingAssistant:Vault:BaseFolder"] = testRoot,
["MeetingAssistant:Vault:TranscriptsFolder"] = "Transcripts",
["MeetingAssistant:Vault:MeetingNotesFolder"] = "Notes",
["MeetingAssistant:Vault:AssistantContextFolder"] = "Context",
["MeetingAssistant:Vault:SummariesFolder"] = "Summaries",
["MeetingAssistant:Vault:ProjectsFolder"] = "Projects",
["MeetingAssistant:Recording:TemporaryRecordingsFolder"] = Path.Combine(testRoot, "Recordings"),
["MeetingAssistant:Recording:InactivitySafeguard:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(testRoot, "speakers.db"),
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false"
});
});
builder.ConfigureTestServices(services =>
{
services.RemoveAll<IMeetingAudioSource>();
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
PortableMeetingAudioRegistration.Add(services, isMacOs: true);
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
services.AddSingleton<IMacOsAudioCaptureProcessFactory>(nativeProcesses);
services.RemoveAll<ISpeechRecognitionPipelineFactory>();
services.AddSingleton<ISpeechRecognitionPipelineFactory>(speechPipelines);
services.RemoveAll<IMeetingNoteOpener>();
services.AddSingleton<IMeetingNoteOpener, NoopMeetingNoteOpener>();
services.RemoveAll<IMeetingSummaryPipeline>();
services.AddSingleton<IMeetingSummaryPipeline, NoopMeetingSummaryPipeline>();
});
});
try
{
using var client = factory.CreateClient();
using var startResponse = await client.PostAsync("/recording/start", content: null);
var started = await startResponse.Content.ReadFromJsonAsync<RecordingStatus>();
var mixedChunk = await speechPipelines.FirstChunk.Task.WaitAsync(TimeSpan.FromSeconds(5));
Assert.True(startResponse.IsSuccessStatusCode);
Assert.True(started?.IsRecording);
Assert.Equal(12_000, BitConverter.ToInt16(mixedChunk.Pcm));
using var stopResponse = await client.PostAsync("/recording/stop", content: null);
var stopped = await stopResponse.Content.ReadFromJsonAsync<RecordingStatus>();
Assert.True(stopResponse.IsSuccessStatusCode);
Assert.False(stopped?.IsRecording);
Assert.All(nativeProcesses.Processes, process => Assert.True(process.Terminated));
}
finally
{
if (Directory.Exists(testRoot))
{
Directory.Delete(testRoot, recursive: true);
}
}
}
private static async Task<IReadOnlyList<AudioChunk>> ReadChunksAsync(
IMeetingAudioSource source,
MeetingAssistantOptions options)
{
var chunks = new List<AudioChunk>();
await foreach (var chunk in source.CaptureAsync(options, CancellationToken.None))
{
chunks.Add(chunk);
}
return chunks;
}
private static byte[] Pcm16(params short[] samples)
{
var pcm = new byte[samples.Length * sizeof(short)];
Buffer.BlockCopy(samples, 0, pcm, 0, pcm.Length);
return pcm;
}
private sealed class StubMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
{
private readonly IMacOsAudioCaptureProcess process;
public StubMacOsAudioCaptureProcessFactory(IMacOsAudioCaptureProcess process)
{
this.process = process;
}
public MacOsAudioCaptureRequest? Request { get; private set; }
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
{
Request = request;
return process;
}
}
private sealed class StubMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
{
private readonly MemoryStream standardOutput;
private readonly string standardError;
public StubMacOsAudioCaptureProcess(byte[] pcm, int exitCode, string standardError = "")
{
standardOutput = new MemoryStream(pcm);
this.standardError = standardError;
ExitCode = exitCode;
}
public Stream StandardOutput => standardOutput;
public int ExitCode { get; }
public bool HasExited => true;
public bool Disposed { get; private set; }
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
{
return Task.FromResult(standardError);
}
public Task WaitForExitAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public void Terminate()
{
}
public ValueTask DisposeAsync()
{
Disposed = true;
standardOutput.Dispose();
return ValueTask.CompletedTask;
}
}
private sealed class EndpointMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
{
private readonly IReadOnlyDictionary<MacOsAudioCaptureKind, byte[]> pcmByKind;
public EndpointMacOsAudioCaptureProcessFactory(byte[] microphonePcm, byte[] systemPcm)
{
pcmByKind = new Dictionary<MacOsAudioCaptureKind, byte[]>
{
[MacOsAudioCaptureKind.Microphone] = microphonePcm,
[MacOsAudioCaptureKind.System] = systemPcm
};
}
public List<BlockingMacOsAudioCaptureProcess> Processes { get; } = [];
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
{
var process = new BlockingMacOsAudioCaptureProcess(pcmByKind[request.Kind]);
Processes.Add(process);
return process;
}
}
private sealed class BlockingMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
{
private readonly TaskCompletionSource terminated = new(TaskCreationOptions.RunContinuationsAsynchronously);
public BlockingMacOsAudioCaptureProcess(byte[] pcm)
{
StandardOutput = new InitialPcmThenBlockingStream(pcm, terminated.Task);
}
public Stream StandardOutput { get; }
public int ExitCode => Terminated ? 137 : throw new InvalidOperationException("Capture is still running.");
public bool HasExited => Terminated;
public bool Terminated => terminated.Task.IsCompleted;
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
{
return Task.FromResult("");
}
public Task WaitForExitAsync(CancellationToken cancellationToken)
{
return terminated.Task.WaitAsync(cancellationToken);
}
public void Terminate()
{
terminated.TrySetResult();
}
public ValueTask DisposeAsync()
{
StandardOutput.Dispose();
return ValueTask.CompletedTask;
}
}
private sealed class InitialPcmThenBlockingStream : Stream
{
private readonly byte[] pcm;
private readonly Task processExit;
private bool read;
public InitialPcmThenBlockingStream(byte[] pcm, Task processExit)
{
this.pcm = pcm;
this.processExit = processExit;
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default)
{
if (!read)
{
read = true;
pcm.AsSpan().CopyTo(buffer.Span);
return pcm.Length;
}
await processExit.WaitAsync(cancellationToken);
return 0;
}
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override void Flush()
{
}
}
private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
{
public TaskCompletionSource<AudioChunk> FirstChunk { get; } =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public ISpeechRecognitionPipeline Create()
{
return new CapturingSpeechRecognitionPipeline(FirstChunk);
}
}
private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
{
private readonly TaskCompletionSource<AudioChunk> firstChunk;
private readonly TaskCompletionSource completed = new(TaskCreationOptions.RunContinuationsAsynchronously);
public CapturingSpeechRecognitionPipeline(TaskCompletionSource<AudioChunk> firstChunk)
{
this.firstChunk = firstChunk;
}
public Task InitializeAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public Task InitializeAsync(
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken) => Task.CompletedTask;
public Task WaitUntilReadyAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
firstChunk.TrySetResult(chunk);
return ValueTask.CompletedTask;
}
public Task CompleteAsync(CancellationToken cancellationToken)
{
completed.TrySetResult();
return Task.CompletedTask;
}
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await completed.Task.WaitAsync(cancellationToken);
yield break;
}
public Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
string audioPath,
SpeechRecognitionPipelineOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<TranscriptionSegment>>([]);
}
public ValueTask DisposeAsync()
{
completed.TrySetResult();
return ValueTask.CompletedTask;
}
}
private sealed class NoopMeetingNoteOpener : IMeetingNoteOpener
{
public Task OpenAsync(string notePath, CancellationToken cancellationToken) => Task.CompletedTask;
}
private sealed class NoopMeetingSummaryPipeline : IMeetingSummaryPipeline
{
public Task<MeetingSummaryRunResult> RunAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken)
{
return Task.FromResult(new MeetingSummaryRunResult(artifacts.SummaryPath, "", Succeeded: true));
}
}
}
@@ -0,0 +1,436 @@
using System.Net.Http.Json;
using MeetingAssistant.Calendar;
using MeetingAssistant.MacOs;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class MacOsMeetingIntegrationTests
{
[Fact]
public void PortableRegistrationUsesRealMacOsProvidersOnlyOnMacOs()
{
var macServices = new ServiceCollection().AddLogging();
var unsupportedServices = new ServiceCollection().AddLogging();
MacOsMeetingIntegrationRegistration.Add(macServices, isMacOs: true);
MacOsMeetingIntegrationRegistration.Add(unsupportedServices, isMacOs: false);
using var macProvider = macServices.BuildServiceProvider();
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
Assert.IsType<MacOsMeetingMetadataProvider>(
macProvider.GetRequiredService<IMeetingMetadataProvider>());
Assert.IsType<MacOsCalendarMeetingProvider>(
macProvider.GetRequiredService<ICalendarMeetingProvider>());
Assert.IsType<MacOsMeetingStartPromptService>(
macProvider.GetRequiredService<IMeetingStartPromptService>());
Assert.IsType<MacOsDesktopIntegrationClient>(
macProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
Assert.IsType<MacOsDesktopIntegrationClient>(
macProvider.GetRequiredService<IScreenshotImageCropper>());
Assert.IsType<NoopMeetingMetadataProvider>(
unsupportedProvider.GetRequiredService<IMeetingMetadataProvider>());
Assert.IsType<NoopCalendarMeetingProvider>(
unsupportedProvider.GetRequiredService<ICalendarMeetingProvider>());
Assert.IsType<NoopMeetingStartPromptService>(
unsupportedProvider.GetRequiredService<IMeetingStartPromptService>());
Assert.IsType<UnavailableActiveWindowScreenshotCapture>(
unsupportedProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
Assert.IsType<UnavailableScreenshotImageCropper>(
unsupportedProvider.GetRequiredService<IScreenshotImageCropper>());
}
[Fact]
public async Task MacOsCalendarProvidersReturnTeamsMetadataAndPromptCandidate()
{
var startedAt = new DateTimeOffset(2026, 7, 25, 10, 5, 0, TimeSpan.FromHours(2));
var calendarEvent = new MacOsCalendarEvent(
"event-1",
"Project sync",
startedAt.AddMinutes(-5),
startedAt.AddMinutes(55),
"Microsoft Teams",
"Discuss launch readiness.\nJoin Microsoft Teams Meeting",
"https://teams.microsoft.com/l/meetup-join/example",
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
IsCanceled: false);
var client = new StubMacOsDesktopIntegrationClient(calendarEvent);
IMeetingMetadataProvider metadataProvider = new MacOsMeetingMetadataProvider(
client,
NullLogger<MacOsMeetingMetadataProvider>.Instance);
ICalendarMeetingProvider calendarProvider = new MacOsCalendarMeetingProvider(
client,
NullLogger<MacOsCalendarMeetingProvider>.Instance);
var metadata = await metadataProvider.GetCurrentMeetingAsync(
startedAt,
CancellationToken.None);
var candidates = await calendarProvider.GetRecordingPromptCandidatesForDayAsync(
DateOnly.FromDateTime(startedAt.Date),
CancellationToken.None);
Assert.NotNull(metadata);
Assert.Equal("Project sync", metadata.Title);
Assert.Equal(
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
metadata.Attendees);
Assert.Equal("Discuss launch readiness.", metadata.Agenda);
Assert.Equal(calendarEvent.End, metadata.ScheduledEnd);
var candidate = Assert.Single(candidates);
Assert.Equal("event-1", candidate.Id);
Assert.Equal("Project sync", candidate.Subject);
Assert.NotNull(candidate.Metadata);
Assert.Equal(metadata.Title, candidate.Metadata.Title);
Assert.Equal(metadata.Attendees, candidate.Metadata.Attendees);
Assert.Equal(metadata.Agenda, candidate.Metadata.Agenda);
Assert.Equal(metadata.ScheduledEnd, candidate.Metadata.ScheduledEnd);
}
[Fact]
public async Task MacOsPromptAndScreenshotUseNativeClientResults()
{
var screenshot = new ActiveWindowScreenshot([137, 80, 78, 71], "Meeting deck");
var client = new StubMacOsDesktopIntegrationClient(
screenshot: screenshot,
promptResponse: MeetingStartPromptResponse.Record);
IMeetingStartPromptService prompt = new MacOsMeetingStartPromptService(
client,
NullLogger<MacOsMeetingStartPromptService>.Instance);
IActiveWindowScreenshotCapture capture = client;
IScreenshotImageCropper cropper = client;
MeetingStartPromptResponse? callbackResponse = null;
var request = new MeetingStartPromptRequest(new CalendarMeeting(
"event-1",
"Project sync",
DateTimeOffset.Now,
DateTimeOffset.Now.AddHours(1)));
await prompt.ShowPromptAsync(
request,
(response, _) =>
{
callbackResponse = response;
return Task.CompletedTask;
},
CancellationToken.None);
var captured = await capture.CaptureAsync(CancellationToken.None);
var croppedPath = await cropper.SaveCroppedScreenshotAsync(
"meeting.png",
new ScreenshotCropCoordinates(1, 2, 3, 4),
CancellationToken.None);
Assert.Equal(MeetingStartPromptResponse.Record, callbackResponse);
Assert.Equal(screenshot, captured);
Assert.Equal("meeting.png.cropped.png", croppedPath);
}
[Fact]
public async Task MacOsCapabilityEndpointReportsEnabledRealProviders()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
await using var factory = new WebApplicationFactory<Program>();
using var client = factory.CreateClient();
var report = await client.GetFromJsonAsync<PlatformCapabilityReport>(
"/diagnostics/platform-capabilities");
Assert.NotNull(report);
Assert.Equal("macOS", report.Host);
Assert.Equal(nameof(MacOsMeetingMetadataProvider), report.MeetingMetadataProvider);
Assert.Equal(nameof(MacOsCalendarMeetingProvider), report.CalendarMeetingProvider);
Assert.Equal(nameof(MacOsMeetingStartPromptService), report.MeetingStartPromptProvider);
Assert.Equal(nameof(MacOsDesktopIntegrationClient), report.ScreenshotProvider);
Assert.Equal(nameof(MacOsWorkflowRulesEditorWindowService), report.WorkflowEditorProvider);
Assert.True(report.WorkflowEditorAvailable);
Assert.True(report.CalendarRecordingPromptsEnabled);
Assert.True(report.SpeakerIdentificationEnabled);
Assert.True(report.ScreenshotOcrEnabled);
Assert.True(report.FunAsrBackendEnabled);
Assert.True(report.FunAsrDiarizationEnabled);
Assert.True(report.WhisperLocalDiarizationEnabled);
}
[Fact]
public async Task NativeHelperAdvertisesCalendarPromptAndScreenshotFeatures()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
var runner = new ProcessCommandRunner(
NullLogger<ProcessCommandRunner>.Instance);
var result = await runner.RunAsync(
GetNativeHelperPath(),
["self-test"],
CancellationToken.None);
Assert.Equal(0, result.ExitCode);
using var json = System.Text.Json.JsonDocument.Parse(result.StandardOutput);
Assert.Equal("ok", json.RootElement.GetProperty("status").GetString());
Assert.Equal(
["calendar", "prompt", "screenshot", "crop"],
json.RootElement.GetProperty("features")
.EnumerateArray()
.Select(feature => feature.GetString()!)
.ToArray());
}
[Fact]
public async Task NativeHelperCropsPngUsingOcrPixelCoordinates()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
var testRoot = Path.Combine(
Path.GetTempPath(),
$"meeting-assistant-native-crop-{Guid.NewGuid():N}");
Directory.CreateDirectory(testRoot);
var bitmapPath = Path.Combine(testRoot, "input.bmp");
var inputPath = Path.Combine(testRoot, "input.png");
var outputPath = Path.Combine(testRoot, "output.png");
var outputBitmapPath = Path.Combine(testRoot, "output.bmp");
try
{
await File.WriteAllBytesAsync(
bitmapPath,
CreateTopRedBottomBlueBitmap());
var runner = new ProcessCommandRunner(
NullLogger<ProcessCommandRunner>.Instance);
var inputConversion = await runner.RunAsync(
"/usr/bin/sips",
["-s", "format", "png", bitmapPath, "--out", inputPath],
CancellationToken.None);
Assert.Equal(0, inputConversion.ExitCode);
var result = await runner.RunAsync(
GetNativeHelperPath(),
["crop", inputPath, outputPath, "0", "0", "2", "1"],
CancellationToken.None);
Assert.Equal(0, result.ExitCode);
var png = await File.ReadAllBytesAsync(outputPath);
Assert.Equal((2, 1), PngTestFixture.ReadDimensions(png));
var outputConversion = await runner.RunAsync(
"/usr/bin/sips",
["-s", "format", "bmp", outputPath, "--out", outputBitmapPath],
CancellationToken.None);
Assert.Equal(0, outputConversion.ExitCode);
Assert.Equal(
(Red: (byte)255, Green: (byte)0, Blue: (byte)0),
ReadFirstBitmapPixel(await File.ReadAllBytesAsync(outputBitmapPath)));
}
finally
{
Directory.Delete(testRoot, recursive: true);
}
}
[Fact]
public async Task CalendarProviderUsesActualLocalMidnightsAcrossDaylightSavingTransition()
{
if (TimeZoneInfo.Local.Id != "Europe/Berlin")
{
return;
}
var client = new StubMacOsDesktopIntegrationClient();
var provider = new MacOsCalendarMeetingProvider(
client,
NullLogger<MacOsCalendarMeetingProvider>.Instance);
await provider.GetRecordingPromptCandidatesForDayAsync(
new DateOnly(2026, 3, 29),
CancellationToken.None);
Assert.Equal(
DateTimeOffset.Parse("2026-03-29T00:00:00+01:00"),
client.CalendarFrom);
Assert.Equal(
DateTimeOffset.Parse("2026-03-30T00:00:00+02:00"),
client.CalendarTo);
Assert.Equal(TimeSpan.FromHours(23), client.CalendarTo - client.CalendarFrom);
}
[Fact]
public async Task CalendarClientFallsBackToCalendarAutomationWhenEventKitIsDenied()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
var runner = new SequenceCommandRunner(
new CommandResult(
MacOsIntegrationException.CalendarPermissionDeniedExitCode,
"",
"Calendar Full Access permission is required."),
new CommandResult(
0,
"""
[{"id":"fallback-event","subject":"Project sync","start":"2026-07-25T08:00:00Z","end":"2026-07-25T09:00:00Z","location":"Microsoft Teams","notes":"","url":"https://teams.microsoft.com/example","attendees":[],"isCanceled":false}]
""",
""));
IMacOsCalendarClient client = new MacOsDesktopIntegrationClient(
runner,
Path.GetDirectoryName(GetNativeHelperPath())!);
var events = await client.GetCalendarEventsAsync(
DateTimeOffset.Parse("2026-07-25T00:00:00Z"),
DateTimeOffset.Parse("2026-07-26T00:00:00Z"),
CancellationToken.None);
var calendarEvent = Assert.Single(events);
Assert.Equal("fallback-event", calendarEvent.Id);
Assert.Collection(
runner.Commands,
command => Assert.Equal("calendar", command.Arguments[0]),
command =>
{
Assert.Equal("/usr/bin/osascript", command.FileName);
Assert.Equal(["-l", "JavaScript"], command.Arguments.Take(2));
});
}
private static string GetNativeHelperPath()
{
var configuration = new DirectoryInfo(AppContext.BaseDirectory)
.Parent?.Name ?? "Debug";
return Path.GetFullPath(Path.Combine(
AppContext.BaseDirectory,
"..",
"..",
"..",
"..",
"MeetingAssistant",
"bin",
configuration,
"net10.0",
"Native",
"macos-meeting-integrations"));
}
private static byte[] CreateTopRedBottomBlueBitmap()
{
var bitmap = new byte[70];
bitmap[0] = (byte)'B';
bitmap[1] = (byte)'M';
BitConverter.GetBytes(bitmap.Length).CopyTo(bitmap, 2);
BitConverter.GetBytes(54).CopyTo(bitmap, 10);
BitConverter.GetBytes(40).CopyTo(bitmap, 14);
BitConverter.GetBytes(2).CopyTo(bitmap, 18);
BitConverter.GetBytes(2).CopyTo(bitmap, 22);
BitConverter.GetBytes((short)1).CopyTo(bitmap, 26);
BitConverter.GetBytes((short)24).CopyTo(bitmap, 28);
BitConverter.GetBytes(16).CopyTo(bitmap, 34);
// BMP rows are bottom-up and padded to four-byte boundaries.
bitmap[54] = 255;
bitmap[57] = 255;
bitmap[64] = 255;
bitmap[67] = 255;
return bitmap;
}
private static (byte Red, byte Green, byte Blue) ReadFirstBitmapPixel(byte[] bitmap)
{
var pixelOffset = BitConverter.ToInt32(bitmap, 10);
return (
Red: bitmap[pixelOffset + 2],
Green: bitmap[pixelOffset + 1],
Blue: bitmap[pixelOffset]);
}
private sealed class StubMacOsDesktopIntegrationClient :
IMacOsCalendarClient,
IMacOsMeetingPromptClient,
IActiveWindowScreenshotCapture,
IScreenshotImageCropper
{
private readonly IReadOnlyList<MacOsCalendarEvent> events;
private readonly ActiveWindowScreenshot screenshot;
private readonly MeetingStartPromptResponse promptResponse;
public StubMacOsDesktopIntegrationClient(
MacOsCalendarEvent? calendarEvent = null,
ActiveWindowScreenshot? screenshot = null,
MeetingStartPromptResponse promptResponse = MeetingStartPromptResponse.Skip)
{
events = calendarEvent is null ? [] : [calendarEvent];
this.screenshot = screenshot ?? new ActiveWindowScreenshot([], null);
this.promptResponse = promptResponse;
}
public Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
DateTimeOffset from,
DateTimeOffset to,
CancellationToken cancellationToken)
{
CalendarFrom = from;
CalendarTo = to;
return Task.FromResult(events);
}
public DateTimeOffset CalendarFrom { get; private set; }
public DateTimeOffset CalendarTo { get; private set; }
public Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
MeetingStartPromptRequest request,
CancellationToken cancellationToken)
{
return Task.FromResult(promptResponse);
}
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
{
return Task.FromResult(screenshot);
}
public Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
return Task.FromResult<string?>($"{screenshotPath}.cropped.png");
}
}
private sealed class SequenceCommandRunner : ICommandRunner
{
private readonly Queue<CommandResult> results;
public SequenceCommandRunner(params CommandResult[] results)
{
this.results = new Queue<CommandResult>(results);
}
public List<(string FileName, IReadOnlyList<string> Arguments)> Commands { get; } = [];
public Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
Commands.Add((fileName, arguments));
return Task.FromResult(results.Dequeue());
}
}
}
@@ -8,11 +8,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="10.1.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
<PackageReference Include="coverlet.collector" Version="10.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.10" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
</ItemGroup>
<ItemGroup>
@@ -4,8 +4,6 @@ using MeetingAssistant.Speakers;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System.Drawing;
using System.Drawing.Imaging;
#pragma warning disable CA1416
@@ -117,11 +115,7 @@ public sealed class MeetingScreenshotServiceTests
var croppedPath = Assert.Single(Directory.GetFiles(
Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"),
"*-cropped.png"));
using (var cropped = new Bitmap(croppedPath))
{
Assert.Equal(2, cropped.Width);
Assert.Equal(2, cropped.Height);
}
Assert.Equal((2, 2), PngTestFixture.ReadDimensions(await File.ReadAllBytesAsync(croppedPath)));
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
var screenshotIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal);
@@ -133,6 +127,38 @@ public sealed class MeetingScreenshotServiceTests
Assert.Contains("Shared screen text", context);
}
[Fact]
public async Task CaptureKeepsSuccessfulOcrTextWhenCropperRejectsCoordinates()
{
var fixture = await ScreenshotFixture.CreateAsync(options =>
{
options.Screenshots.Ocr.Enabled = true;
});
var ocr = new CapturingScreenshotOcrClient(
"Readable text survives.",
new ScreenshotCropCoordinates(99, 99, 10, 10));
var service = fixture.CreateService(
new FixedScreenshotCapture(CreatePngBytes(4, 4)),
ocr,
screenshotImageCropper: new RejectingScreenshotImageCropper());
await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(
fixture.Artifacts,
TimeSpan.FromSeconds(5),
CancellationToken.None);
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
Assert.Contains("Readable text survives.", context);
Assert.DoesNotContain("_OCR failed", context);
Assert.DoesNotContain("![Cropped screenshot]", context);
}
[Fact]
public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer()
{
@@ -454,7 +480,8 @@ public sealed class MeetingScreenshotServiceTests
IActiveWindowScreenshotCapture capture,
IScreenshotOcrClient ocrClient,
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
IScreenshotImageCropper? screenshotImageCropper = null)
{
return new MeetingScreenshotService(
capture,
@@ -462,12 +489,28 @@ public sealed class MeetingScreenshotServiceTests
NoteStore,
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
ocrClient,
screenshotImageCropper ??
capture as IScreenshotImageCropper ??
new UnavailableScreenshotImageCropper(),
NullLogger<MeetingScreenshotService>.Instance,
meetingWorkflowEngine);
}
}
private sealed class FixedScreenshotCapture : IActiveWindowScreenshotCapture
private sealed class RejectingScreenshotImageCropper : IScreenshotImageCropper
{
public Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
throw new InvalidDataException("The crop is outside image bounds.");
}
}
private sealed class FixedScreenshotCapture :
IActiveWindowScreenshotCapture,
IScreenshotImageCropper
{
private readonly byte[] bytes;
@@ -480,6 +523,21 @@ public sealed class MeetingScreenshotServiceTests
{
return Task.FromResult(new ActiveWindowScreenshot(bytes, "Demo Window"));
}
public async Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
var croppedPath = Path.Combine(
Path.GetDirectoryName(screenshotPath)!,
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
await File.WriteAllBytesAsync(
croppedPath,
PngTestFixture.Solid(2, 2),
cancellationToken);
return croppedPath;
}
}
private sealed class CapturingScreenshotOcrClient : IScreenshotOcrClient
@@ -614,15 +672,7 @@ public sealed class MeetingScreenshotServiceTests
private static byte[] CreatePngBytes(int width, int height)
{
using var bitmap = new Bitmap(width, height);
using (var graphics = Graphics.FromImage(bitmap))
{
graphics.Clear(Color.White);
}
using var stream = new MemoryStream();
bitmap.Save(stream, ImageFormat.Png);
return stream.ToArray();
return PngTestFixture.Solid(width, height);
}
private static string ExtractScreenshotOcrId(string context)
@@ -7,37 +7,6 @@ namespace MeetingAssistant.Tests;
public sealed class MeetingSummaryInstructionBuilderTests
{
[Fact]
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
var artifacts = CreateArtifacts(root);
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
{
Agent = new AgentOptions { InitialPrompt = "Custom base." },
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
}));
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Custom base.", instructions);
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha Platform", instructions);
Assert.Contains("Builds meeting automation", instructions);
Assert.Contains("PROJECT.md", instructions);
Assert.Contains("256", instructions);
Assert.Contains("request_project_association", instructions);
Assert.Contains("request_workflow_change", instructions);
Assert.DoesNotContain("Internal notes", instructions);
Assert.DoesNotContain("Do not expose", instructions);
}
[Fact]
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
{
@@ -106,8 +75,7 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Custom summarizer instructions.", instructions);
Assert.Contains("request_project_association", instructions);
Assert.Equal("Custom summarizer instructions.", instructions);
}
[Fact]
@@ -128,12 +96,21 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Base.", instructions);
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha instructions.", instructions);
Assert.Contains("# Beta", instructions);
Assert.Contains("Beta instructions.", instructions);
var expected = """
Base.
---
projects:
# Alpha
Alpha instructions.
# Beta
Beta instructions.
""";
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
}
private static MeetingSessionArtifacts CreateArtifacts(string root)
@@ -50,145 +50,6 @@ public sealed class MicrophoneAudioSourceTests
await cancellation.CancelAsync();
}
[Fact]
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SelectableCaptureDevices(selection);
var prompts = new AcceptingPrompt();
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
var nextChunk = chunks.MoveNextAsync().AsTask();
Assert.False(nextChunk.IsCompleted);
Assert.Null(selection.SelectedDeviceId);
prompts.Response.SetResult(true);
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.Equal("new", selection.SelectedDeviceId);
Assert.False(lifetime.IsCancellationRequested);
}
finally
{
await lifetime.CancelAsync();
}
}
[Fact]
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var abandonedCapture = new DisposableAudioSource();
var factory = new CallbackCaptureFactory(() =>
{
lifetime.Cancel();
return abandonedCapture;
});
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
Assert.False(await chunks.MoveNextAsync());
Assert.True(abandonedCapture.Disposed);
}
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
{
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
}
[Fact]
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SwitchDuringCreationDevices(selection);
var prompts = new AcceptingPrompt();
prompts.Response.SetResult(true);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal("new", selection.SelectedDeviceId);
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.True(devices.AbandonedCapture.Disposed);
}
finally
{
await lifetime.CancelAsync();
}
}
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
private MicrophoneDevice[] available = [new("old", "Built-in")];
private int creations;
private volatile bool switchObserved;
public DisposableAudioSource AbandonedCapture { get; } = new();
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
if (selection.SelectedDeviceId == "new") switchObserved = true;
return Volatile.Read(ref available);
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(GetAvailableMicrophones(), available[0]);
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
// Let the acceptance callback finish before returning the already-opened source.
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
throw new TimeoutException("The new microphone was not selected during capture creation.");
return AbandonedCapture;
}
}
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
{
public bool Disposed { get; private set; }
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
public void Dispose() => Disposed = true;
}
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
{
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
Shown.TrySetResult(microphone);
return Response.Task.WaitAsync(cancellationToken);
}
}
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
}
private static byte[] Pcm16(short sample)
{
return BitConverter.GetBytes(sample);
@@ -1,163 +0,0 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class MicrophoneSwitchMonitorTests
{
[Fact]
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var devices = new MutableDevices();
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var switched = false;
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
devices.Available = [new("usb", "USB microphone")];
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(prompts.WasShown);
Assert.False(switched);
}
[Fact]
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
{
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
var newDevice = new MicrophoneDevice("new", "USB headset");
var devices = new MutableDevices { Available = [oldDevice] };
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts);
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
try
{
devices.Available = [oldDevice, newDevice];
var prompt = await prompts.NextAsync();
Assert.Equal(newDevice, prompt.Device);
Assert.False(switched.Task.IsCompleted);
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
prompt.Response.SetResult(true);
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
[Theory]
[InlineData("declined")]
[InlineData("removed")]
[InlineData("expired")]
[InlineData("ended")]
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
{
var device = new MicrophoneDevice("usb", "USB microphone");
var devices = new MutableDevices();
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts,
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
var switched = false;
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
try
{
devices.Available = [device];
var prompt = await prompts.NextAsync();
if (outcome == "declined")
{
prompt.Response.SetResult(false);
await devices.WaitForPollsAsync();
}
else
{
if (outcome == "removed") devices.Available = [];
if (outcome == "ended") await lifetime.CancelAsync();
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
await lifetime.CancelAsync();
await monitoring;
Assert.False(switched);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
TimeSpan? promptLifetime = null) => new(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
promptLifetime ?? TimeSpan.FromMinutes(1));
private sealed class MutableDevices : IMicrophoneDeviceProvider
{
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
public volatile MicrophoneDevice[] Available = [];
public Action? BeforeRead { get; set; }
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
BeforeRead?.Invoke();
polls.Writer.TryWrite(true);
return Available;
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, Available.FirstOrDefault());
public async Task WaitForPollsAsync()
{
while (polls.Reader.TryRead(out _)) { }
for (var i = 0; i < 3; i++)
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
}
}
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
{
public bool WasShown { get; private set; }
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
WasShown = true;
beforeAcceptance();
return Task.FromResult(true);
}
}
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
{
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
var prompt = new Prompt(microphone, cancellationToken);
requests.Writer.TryWrite(prompt);
return await prompt.Response.Task.WaitAsync(cancellationToken);
}
}
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -25,16 +25,4 @@ public sealed class NotificationActivationArgumentsTests
Assert.Equal("abc123", arguments["promptId"]);
Assert.Equal("continue", arguments["response"]);
}
[Fact]
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
{
Assert.Contains(
MeetingInactivityPromptActions.All,
action => action.Content == "Pause transcription" &&
action.ResponseArgument == "pause");
Assert.Equal(
MeetingInactivityPromptResponse.Pause,
MeetingInactivityPromptActions.ParseResponse("pause"));
}
}
+38
View File
@@ -0,0 +1,38 @@
namespace MeetingAssistant.Tests;
internal static class PngTestFixture
{
private const string Solid2X2 =
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAAqADAAQAAAABAAAAAgAAAADO0J6QAAAAEklEQVQIHWP8DwQMQMAEIkAAAD34BACALvQ5AAAAAElFTkSuQmCC";
private const string Solid4X4 =
"iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAABKADAAQAAAABAAAABAAAAADmpNw4AAAAFUlEQVQIHWP8DwQMSIAJiQ1mEhYAAAZdBAQjjGwcAAAAAElFTkSuQmCC";
private const string Solid8X6 =
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAYAAAD+Bd/7AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAACKADAAQAAAABAAAABgAAAAAJfOcJAAAAFklEQVQIHWP8DwQMeAATHjmw1GBQAAAQiQQIVmhjoQAAAABJRU5ErkJggg==";
public static byte[] Solid(int width, int height)
{
var encoded = (width, height) switch
{
(2, 2) => Solid2X2,
(4, 4) => Solid4X4,
(8, 6) => Solid8X6,
_ => throw new ArgumentOutOfRangeException(
nameof(width),
$"No deterministic PNG fixture exists for {width}x{height}.")
};
return Convert.FromBase64String(encoded);
}
public static (int Width, int Height) ReadDimensions(byte[] png)
{
if (!png.AsSpan(0, 8).SequenceEqual(
new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }))
{
throw new InvalidDataException("The fixture is not a PNG.");
}
return (
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(16, 4)),
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(20, 4)));
}
}
@@ -1,90 +1,10 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class ProjectKnowledgeToolTests
{
[Theory]
[InlineData("name: Incomplete project")]
[InlineData("description: Missing name")]
[InlineData("name: ' '\ndescription: Missing name")]
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
using var catalog = JsonDocument.Parse(await tools.ListProjects());
var project = Assert.Single(catalog.RootElement.EnumerateArray());
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
Assert.Equal("", project.GetProperty("description").GetString());
}
[Fact]
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
var tools = new MeetingSummaryTools(artifacts,
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
$"description: {new string('b', 257)}", from: 3, to: 3);
Assert.StartsWith("Refused:", rejected);
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
"description: Updated project purpose", from: 3, to: 3));
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
Assert.Contains("owner: Ada", updated);
Assert.Contains("User-authored project notes.", updated);
Assert.Contains("Updated project purpose", await tools.ListProjects());
}
[Fact]
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
var catalog = await tools.ListProjects();
using var json = JsonDocument.Parse(catalog);
var projects = json.RootElement.EnumerateArray().ToArray();
Assert.Equal(3, projects.Length);
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
Assert.Equal("", projects[1].GetProperty("description").GetString());
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
Assert.Equal("", projects[2].GetProperty("description").GetString());
Assert.DoesNotContain("Private", catalog);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
{
@@ -128,7 +48,7 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Contains("MeetingAssistant", await tools.ListProjects());
Assert.Equal("MeetingAssistant", await tools.ListProjects());
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
@@ -136,15 +56,15 @@ public sealed class ProjectKnowledgeToolTests
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.Equal(
"README.md:2 Second alpha line",
await tools.Search("alpha"));
}
[Fact]
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
@@ -168,8 +88,7 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
Assert.Equal("MeetingAssistant", await tools.ListProjects());
}
[Fact]
@@ -181,11 +100,8 @@ public sealed class ProjectKnowledgeToolTests
Directory.CreateDirectory(projectRoot);
var projectFile = Path.Combine(projectRoot, "notes.md");
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var tools = new MeetingSummaryTools(
artifacts,
CreateArtifacts(root),
new MeetingAssistantOptions
{
Vault =
@@ -216,13 +132,8 @@ public sealed class ProjectKnowledgeToolTests
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var outsidePath = Path.Combine(projectsRoot, "outside.md");
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
var tools = new MeetingSummaryTools(
artifacts,
CreateArtifacts(root),
new MeetingAssistantOptions
{
Vault =
@@ -233,7 +144,6 @@ public sealed class ProjectKnowledgeToolTests
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
@@ -18,8 +18,31 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
Enabled = true,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote-validation:local",
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
}),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
@@ -31,109 +54,23 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
Assert.True(commandRunner.RunCancellationWasObserved);
}
[Fact]
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(
[
new LaunchProfile("english", namedOptions),
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
]),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await commandRunner.WaitForRunAsync();
await service.StopAsync(CancellationToken.None);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
}
[Fact]
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(options),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromMilliseconds(50));
await service.StopAsync(CancellationToken.None);
Assert.Empty(commandRunner.Commands);
}
private static MeetingAssistantOptions CreateValidationOptions(string image)
{
return new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
DockerCommand = "docker",
Image = image,
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
};
}
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
{
private readonly IReadOnlyList<LaunchProfile> profiles;
private readonly MeetingAssistantOptions options;
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
{
}
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
{
this.profiles = profiles;
this.options = options;
}
public LaunchProfile GetRequiredProfile(string? name)
{
var profileName = string.IsNullOrWhiteSpace(name)
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
: name;
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
}
public IReadOnlyList<LaunchProfile> GetProfiles()
{
return profiles;
return [GetRequiredProfile(null)];
}
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
@@ -2,7 +2,6 @@ using MeetingAssistant;
using MeetingAssistant.Recording;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NAudio.Wave;
@@ -58,49 +57,17 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Assert.Empty(commandRunner.Commands);
}
[Fact]
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
{
var commandRunner = new CapturingCommandRunner(
"""
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
""");
var modelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
})
.Build();
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
var validator = CreateValidator(commandRunner, configuredOptions);
var valid = await validator.ValidateSampleAsync(
CreateWav(TimeSpan.FromSeconds(20)),
CancellationToken.None);
Assert.True(valid);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
bool enabled = true)
{
return CreateValidator(
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
new MeetingAssistantOptions
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
@@ -109,8 +76,9 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Enabled = enabled,
MinimumSingleSpeakerCoverage = 0.90,
MinimumMatchingKnownSnippetRatio = 1,
Diarization = new PyannoteRuntimeOptions
Diarization = new PyannoteDiarizationOptions
{
Enabled = true,
BuildImage = false,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote:local",
@@ -126,20 +94,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
}
}
}
});
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
MeetingAssistantOptions configuredOptions)
{
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(configuredOptions),
}),
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
}
@@ -156,8 +156,9 @@ public sealed class PyannoteTranscriptFinalizerTests
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var explicitDiarization = new PyannoteRuntimeOptions
var explicitDiarization = new PyannoteDiarizationOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-azure:local",
@@ -168,7 +169,7 @@ public sealed class PyannoteTranscriptFinalizerTests
CommandTimeout = TimeSpan.FromMinutes(1)
};
await finalizer.FinalizeEnabledAsync(
await finalizer.FinalizeAsync(
audioPath,
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
explicitDiarization,
@@ -185,8 +186,9 @@ public sealed class PyannoteTranscriptFinalizerTests
{
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
var diarization = new PyannoteRuntimeOptions
var diarization = new PyannoteDiarizationOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-warmup:local",
@@ -216,8 +218,9 @@ public sealed class PyannoteTranscriptFinalizerTests
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: null);
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
{
Enabled = true,
Token = "",
TokenEnv = "",
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
File diff suppressed because it is too large Load Diff
@@ -1,596 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentificationServiceTests
{
[Fact]
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(7, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Single(fixture.Encoder.Requests);
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
}
[Fact]
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Sabrina", saved.CanonicalName);
Assert.Equal(3, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Null(saved.CanonicalName);
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
}
[Fact]
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(8, saved.VoiceVectors.Count);
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
{
await using var fixture = await Fixture.CreateAsync(options =>
{
options.OutlierPruningMinimumVectors = 20;
options.OutlierPruningNeighborSimilarity = 0.90;
options.OutlierPruningMinimumNeighbors = 3;
options.OutlierPruningMinimumClusterRatio = 0.60;
});
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(16, saved.VoiceVectors.Count);
Assert.All(
saved.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
"Chris",
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
var currentRunVectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
var service = fixture.CreateService();
var liveResult = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
fixture.Encoder.Vectors = currentRunVectors;
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
{
KnownSpeakerMappings = liveResult.SpeakerMappings
};
if (transcriptAlreadyRelabeled)
{
finalRequest = finalRequest with { Segments = liveResult.Segments };
}
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(10, saved.VoiceVectors.Count);
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
}
[Fact]
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
{
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors =
[
UnitVector(0),
UnitVector(0, 2, 0.01f),
UnitVector(0, 3, 0.02f),
UnitVector(0, 4, 0.03f),
UnitVector(0, 5, 0.04f)
];
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(6, saved.VoiceVectors.Count);
Assert.Single(saved.References);
}
[Fact]
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyFinishedSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments[0].Speaker);
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
Assert.All(
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal("John", saved.CanonicalName);
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
Assert.Equal("John", result.Segments.Single().Speaker);
Assert.All(saved.References, reference =>
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
}
[Fact]
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Null(saved.CanonicalName);
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
}
[Fact]
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Manuel", saved.CanonicalName);
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
}
[Fact]
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
{
await using var fixture = await Fixture.CreateAsync(
configureSpeaker: options => options.MaxMatchCandidates = 1);
await fixture.AddIdentityAsync("Legacy WAV identity", []);
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
}
[Fact]
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
{
await using var fixture = await Fixture.CreateAsync();
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.02f))
.ToList();
var service = fixture.CreateService();
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(target.Id);
Assert.Single(saved.Snippets);
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
Assert.Single(await context.SpeakerIdentities.ToListAsync());
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private sealed class Fixture : IAsyncDisposable
{
private readonly string directory;
private readonly string databasePath;
private readonly SpeakerIdentificationOptions speakerOptions;
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
{
this.directory = directory;
this.databasePath = databasePath;
this.speakerOptions = speakerOptions;
}
public FakeEncoder Encoder { get; } = new();
public FakeSnippetExtractor SnippetExtractor { get; } = new();
public string DatabasePath => databasePath;
public static async Task<Fixture> CreateAsync(
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
{
var directory = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
var databasePath = Path.Combine(directory, "speaker-identities.db");
var speakerOptions = new SpeakerIdentificationOptions
{
DatabasePath = databasePath,
MatchBatchSize = 6,
MaxMatchCandidates = 100,
MatchIdentityActiveAge = TimeSpan.FromDays(365),
MinimumSampleSpeechDuration = TimeSpan.Zero,
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
}
};
configure?.Invoke(speakerOptions.Resemblyzer);
configureSpeaker?.Invoke(speakerOptions);
await using var context = new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
return new Fixture(directory, databasePath, speakerOptions);
}
public ResemblyzerSpeakerIdentificationService CreateService()
{
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
return new ResemblyzerSpeakerIdentificationService(
new TestDbContextFactory(databasePath),
SnippetExtractor,
Encoder,
new ResemblyzerVoiceClusterMatcher(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(appOptions),
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
}
public SpeakerIdentificationRequest CreateRequest(
string speaker,
int sampleCount,
IReadOnlyList<string>? attendees = null,
int segmentCount = 1)
{
var transcriptPath = Path.Combine(directory, "transcript.md");
File.WriteAllText(transcriptPath, "Transcript");
var segments = Enumerable.Range(0, segmentCount)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 30),
TimeSpan.FromSeconds((index + 1) * 30),
speaker,
"enough useful words for a speaker sample"))
.ToList();
var segment = segments[0];
return new SpeakerIdentificationRequest(
Path.Combine(directory, "meeting.wav"),
new MeetingNote(
Path.Combine(directory, "meeting.md"),
new MeetingNoteFrontmatter
{
Title = "Test",
Attendees = attendees?.ToList() ?? ["Chris"],
Transcript = transcriptPath,
AssistantContext = Path.Combine(directory, "context.md"),
Summary = Path.Combine(directory, "summary.md")
},
""),
segments,
Enumerable.Range(0, sampleCount)
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
.ToList());
}
public async Task<SpeakerIdentity> AddIdentityAsync(
string? name,
IReadOnlyList<float[]> vectors,
IReadOnlyList<string>? candidates = null,
IReadOnlyList<string>? aliases = null)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = speakerOptions.Resemblyzer.ModelId,
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"known-{index}",
CreatedAt = now.AddMinutes(index)
}).ToList()
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
return identity;
}
public async Task AddCurrentRunCandidateAsync(
SpeakerIdentificationRequest request,
string candidateName)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CreatedAt = now,
UpdatedAt = now,
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "older-model",
Dimensions = 256,
VectorBytes = ToBytes(UnitVector(1)),
Fingerprint = "older-model-vector",
CreatedAt = now
}
],
References =
[
SpeakerIdentityReferences.Create(
request.MeetingNote.Path,
request.MeetingNote.Frontmatter.Transcript,
now)
]
});
await context.SaveChangesAsync();
}
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Aliases)
.Include(identity => identity.References)
.SingleAsync(identity => identity.Id == id);
}
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.References)
.SingleAsync();
}
public ValueTask DisposeAsync()
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
return ValueTask.CompletedTask;
}
}
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
{
public IReadOnlyList<float[]> Vectors { get; set; } = [];
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
{
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
public Task<byte[]> ExtractSnippetAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> speakerSegments,
CancellationToken cancellationToken)
{
Requests.Add(speakerSegments.ToList());
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
}
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -1,174 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
{
[Fact]
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
var databasePath = Path.Combine(directory, "identities.db");
var factory = new TestDbContextFactory(databasePath);
await using (var context = factory.CreateDbContext())
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(CreateIdentity(
"Chris",
DateTimeOffset.UtcNow.AddMonths(-2),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 20, 0.02f))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Chris duplicate",
DateTimeOffset.UtcNow.AddDays(-1),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.02f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 40, 0.02f)))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Decoy",
DateTimeOffset.UtcNow.AddMonths(-1),
[UnitVector(0)],
directory));
await context.SaveChangesAsync();
}
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
};
var service = new ResemblyzerSpeakerIdentityMergeService(
factory,
new ResemblyzerVoiceClusterMatcher(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
MergeRecentIdentityAge = TimeSpan.FromDays(14),
MaxMatchCandidates = 1,
MaxSnippetsPerSpeaker = 3,
Resemblyzer = resemblyzerOptions
}
}),
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
Assert.Equal(2, result.MatchAttempts);
Assert.Equal(1, result.MergedPairs);
await using var verification = factory.CreateDbContext();
var saved = await verification.SpeakerIdentities
.Include(identity => identity.Aliases)
.Include(identity => identity.VoiceVectors)
.SingleAsync(identity => identity.CanonicalName == "Chris");
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
Assert.Equal("Chris", saved.CanonicalName);
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
Assert.Equal(20, saved.VoiceVectors.Count);
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
}
finally
{
Directory.Delete(directory, recursive: true);
}
}
private static SpeakerIdentity CreateIdentity(
string name,
DateTimeOffset createdAt,
IReadOnlyList<float[]> vectors,
string directory)
{
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
File.WriteAllText(transcriptPath, "Transcript");
return new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = createdAt,
UpdatedAt = createdAt,
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"{name}-{index}",
CreatedAt = createdAt.AddMinutes(index)
}).ToList(),
References =
[
new SpeakerIdentityReference
{
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
TranscriptPath = transcriptPath,
CreatedAt = createdAt
}
]
};
}
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
{
var vector = new float[256];
vector[primary] = 1;
if (secondary is { } index)
{
vector[index] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private static float[] ToVector(byte[] bytes)
{
var vector = new float[bytes.Length / sizeof(float)];
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
return vector;
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -1,122 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceClusterMatcherTests
{
[Fact]
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
{
var matcher = CreateMatcher();
var query = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
.ToList();
var candidates = new[]
{
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
};
var result = matcher.Match(query, candidates);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
Assert.True(result.Cohesion >= 0.99);
Assert.True(result.BestSimilarity >= 0.99);
Assert.True(result.RunnerUpSimilarity < 0.1);
}
[Fact]
public void FourVectorsDoNotTriggerAutomaticMatching()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 4).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("4/5", result.Reason);
}
[Fact]
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
{
var result = CreateMatcher().Match(
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("cohesion", result.Reason);
Assert.Null(result.BestSimilarity);
}
[Fact]
public void SimilarityBelowThresholdIsRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
Assert.False(result.Accepted);
Assert.Contains("best similarity", result.Reason);
Assert.Equal(0, result.BestSimilarity);
}
[Fact]
public void SimilarCandidatesWithinMarginAreRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
]);
Assert.False(result.Accepted);
Assert.Contains("margin", result.Reason);
Assert.NotNull(result.RunnerUpSimilarity);
}
[Fact]
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
]);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
}
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
{
return new ResemblyzerVoiceClusterMatcher(
new ResemblyzerSpeakerRecognitionOptions
{
RequiredVectorsPerSpeaker = 5,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05
},
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
}
@@ -1,141 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
{
[Fact]
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(16, identity.VoiceVectors.Count);
Assert.All(
identity.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Fact]
public void AmbiguousDenseClustersPreserveAllVectors()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 10)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(20, identity.VoiceVectors.Count);
}
[Fact]
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 15)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(19, identity.VoiceVectors.Count);
}
[Fact]
public void PruningPreservesOtherModelsAndMalformedStoredRows()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
SpeakerVoiceVectors.AddDistinct(
identity,
[UnitVector(2, 60, 0.04f)],
"older-model",
1000,
DateTimeOffset.UtcNow);
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = [1],
Fingerprint = "malformed",
CreatedAt = DateTimeOffset.UtcNow
});
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "zero-magnitude",
CreatedAt = DateTimeOffset.UtcNow
});
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(19, identity.VoiceVectors.Count);
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
}
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
{
return new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions
{
ModelId = "resemblyzer-0.1.4-pretrained",
OutlierPruningMinimumVectors = 20,
OutlierPruningNeighborSimilarity = 0.90,
OutlierPruningMinimumNeighbors = 3,
OutlierPruningMinimumClusterRatio = 0.60
},
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
}
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
{
var identity = new SpeakerIdentity();
SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
"resemblyzer-0.1.4-pretrained",
1000,
DateTimeOffset.UtcNow);
return identity;
}
private static float[] UnitVector(
int primaryDimension,
int secondaryDimension,
float secondaryValue)
{
var vector = new float[256];
vector[primaryDimension] = 1;
vector[secondaryDimension] = secondaryValue;
return vector;
}
}
@@ -1,78 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerWarmupHostedServiceTests
{
[Fact]
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
{
var encoder = new BlockingEncoder();
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(options),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await encoder.WaitForWarmupAsync();
await service.StopAsync(CancellationToken.None);
Assert.True(encoder.CancellationObserved);
}
[Fact]
public async Task DisabledWarmupDoesNotInvokeEncoder()
{
var encoder = new BlockingEncoder();
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(new MeetingAssistantOptions()),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await service.StopAsync(CancellationToken.None);
Assert.False(encoder.WarmupStarted);
}
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
{
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool WarmupStarted { get; private set; }
public bool CancellationObserved { get; private set; }
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
WarmupStarted = true;
started.TrySetResult();
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
CancellationObserved = true;
}
}
public Task WaitForWarmupAsync()
{
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
}
}
}
@@ -8,22 +8,13 @@ namespace MeetingAssistant.Tests;
public sealed class SpeakerAudioSampleCollectorTests
{
[Fact]
public void DefaultSampleDurationsAreTenAndSixtySeconds()
{
var options = new SpeakerIdentificationOptions();
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
}
[Fact]
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
@@ -46,8 +37,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
@@ -57,61 +48,6 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
}
[Fact]
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
var sample = Assert.Single(collector.Snapshot());
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
}
[Fact]
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
Assert.Empty(collector.Snapshot());
}
[Fact]
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
var sample = collector.TryAdd(Segment(
0,
70,
"Guest01",
"one two three four five six seven eight nine ten."));
Assert.NotNull(sample);
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
}
[Fact]
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
{
@@ -119,8 +55,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60),
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1),
logger: logger);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
@@ -133,47 +69,6 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.Contains("minimum duration", message);
}
[Fact]
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
Assert.Collection(
samples,
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
}
[Fact]
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
Assert.Single(collector.Snapshot());
}
private static TranscriptionSegment Segment(
double start,
double end,
@@ -107,9 +107,6 @@ public sealed class SpeakerIdentityMergeServiceTests
return new SpeakerIdentityMergeService(
new TestSpeakerIdentityDbContextFactory(dbPath),
Matcher,
new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions(),
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
@@ -1,47 +0,0 @@
using MeetingAssistant.Speakers;
namespace MeetingAssistant.Tests;
public sealed class SpeakerIdentityMergerTests
{
[Fact]
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
{
var now = DateTimeOffset.UtcNow;
var target = new SpeakerIdentity
{
CanonicalName = "Chris",
VoiceVectors =
[
Vector("oldest", 1, now.AddMinutes(-4)),
Vector("duplicate", 2, now.AddMinutes(-3))
]
};
var source = new SpeakerIdentity
{
VoiceVectors =
[
Vector("duplicate", 2, now.AddMinutes(-2)),
Vector("newer", 3, now.AddMinutes(-1)),
Vector("newest", 4, now)
]
};
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
}
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
{
return new SpeakerVoiceVector
{
ModelId = "model",
Dimensions = 256,
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
Fingerprint = fingerprint,
CreatedAt = createdAt
};
}
}
@@ -1,80 +0,0 @@
using MeetingAssistant.Speakers;
using Microsoft.Data.Sqlite;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace MeetingAssistant.Tests;
public sealed class SpeakerRecognitionFeatureSelectionTests
{
[Theory]
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
bool enabled,
Type expectedIdentificationType,
Type expectedMergeType,
int expectedMinimumSamples,
bool expectedNonOverlappingSamples)
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
await using var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, configuration) =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
});
});
builder.ConfigureTestServices(services =>
{
services.RemoveAll<IResemblyzerVoiceEncoder>();
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
});
});
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
Assert.IsType(expectedIdentificationType, service);
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
Assert.IsType(expectedMergeType, mergeService);
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
}
finally
{
SqliteConnection.ClearAllPools();
Directory.Delete(directory, recursive: true);
}
}
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
{
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}
@@ -1,142 +0,0 @@
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Tests;
public sealed class SpeakerSampleSpanSelectorTests
{
[Fact]
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(7, 13),
Segment(20, 70)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Equal(3, span.Count);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
[Fact]
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Empty(span);
}
[Fact]
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
{
var segments = Enumerable.Range(0, 6)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 10),
TimeSpan.FromSeconds((index + 1) * 10),
"Guest01",
$"segment {index} has enough words"))
.ToList();
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
segments,
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(5, spans.Count);
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
Assert.All(
spans.Zip(spans.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
}
[Fact]
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 10),
Segment(9, 19),
Segment(20, 30)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(2, spans.Count);
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
}
[Fact]
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(7, 13)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
}
[Fact]
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Empty(spans);
}
[Fact]
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[Segment(0, 70)],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
private static TranscriptionSegment Segment(double start, double end)
{
return new TranscriptionSegment(
TimeSpan.FromSeconds(start),
TimeSpan.FromSeconds(end),
"Guest01",
"enough words for this segment");
}
}
@@ -1,106 +0,0 @@
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
namespace MeetingAssistant.Tests;
public sealed class SpeakerVoiceVectorPersistenceTests
{
[Fact]
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using (var context = new SpeakerIdentityDbContext(options))
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-a",
CreatedAt = DateTimeOffset.UtcNow
}
]
});
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
var identity = await context.SpeakerIdentities
.Include(candidate => candidate.VoiceVectors)
.SingleAsync();
var vector = Assert.Single(identity.VoiceVectors);
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
Assert.Equal(256, vector.Dimensions);
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
context.SpeakerIdentities.Remove(identity);
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
}
}
[Fact]
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using var context = new SpeakerIdentityDbContext(options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
var identity = new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
context.SpeakerVoiceVectors.AddRange(
CreateVector(identity.Id, "same-vector"),
CreateVector(identity.Id, "same-vector"));
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
}
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
{
return new SpeakerVoiceVector
{
SpeakerIdentityId = identityId,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = fingerprint,
CreatedAt = DateTimeOffset.UtcNow
};
}
}
@@ -1,209 +0,0 @@
using System.Threading.Channels;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Summary;
using Microsoft.Extensions.Logging.Abstractions;
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Tests;
public sealed class SummaryAgentRequestTests
{
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("shutdown")]
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
{
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
finally { await requests.StopAsync(CancellationToken.None); }
Assert.Empty(editor.Prompts);
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
}
[Fact]
public async Task InvalidRequestsAreRefusedBeforeNotification()
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
Assert.False(prompts.HasMore);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
}
[Fact]
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
Assert.StartsWith("Requested", result);
var prompt = await prompts.NextAsync();
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
Assert.Empty(editor.Prompts);
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
Assert.True(tools.SummaryWasWritten);
prompt.Response.SetResult(true);
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.False(prompt.Response.TrySetResult(true));
Assert.Single(editor.Prompts);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
}
finally { await requests.StopAsync(CancellationToken.None); }
}
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("canceled")]
[InlineData("removed")]
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
using var cancellation = new CancellationTokenSource();
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "canceled") await cancellation.CancelAsync();
if (outcome == "removed")
{
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
prompt.Response.SetResult(true);
}
if (outcome == "canceled")
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
else
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
var prompt = await prompts.NextAsync();
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
Assert.False(request.IsCompleted);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
prompt.Response.SetResult(true);
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
Assert.StartsWith("Approved", result);
Assert.Contains("Alpha instructions", result);
Assert.Contains("Alpha Platform", result);
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
Assert.Contains("Edited by user", note);
Assert.Contains("custom: keep-me", note);
Assert.Contains("User's latest notes.", note);
Assert.Contains("Existing", note);
Assert.Contains("Alpha", note);
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
Assert.False(prompts.HasMore);
}
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var project = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(project);
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
}
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
{
public List<string> Prompts { get; } = [];
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Show() { }
public void ShowWithPrompt(string initialPrompt)
{
Prompts.Add(initialPrompt);
Started.TrySetResult(initialPrompt);
}
}
private sealed class ControlledPrompts : IAgentApprovalPromptService
{
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
var request = new PendingPrompt(prompt, cancellationToken);
requests.Writer.TryWrite(request);
try { return await request.Response.Task.WaitAsync(cancellationToken); }
finally { request.Removed.TrySetResult(); }
}
}
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
+4 -29
View File
@@ -74,8 +74,7 @@ public sealed class TaskbarIconTests
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
("Exit", MeetingTaskbarAction.Exit, true));
@@ -95,32 +94,10 @@ public sealed class TaskbarIconTests
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, true),
("Exit", MeetingTaskbarAction.Exit, true));
}
[Fact]
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
{
var runningMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
[Profile("default")]);
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
[Profile("default")]);
Assert.Contains(runningMenu.Items, item =>
item.Action == MeetingTaskbarAction.PauseTranscription &&
item.Text == "Pause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
item.Text == "Unpause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.StopRecording &&
item.Text == "Finish meeting");
}
[Fact]
public void ProcessingMenuShowsSummarizingButAllowsStartingNewRecordings()
{
@@ -223,8 +200,7 @@ public sealed class TaskbarIconTests
private static RecordingStatus Status(
bool isRecording = false,
RecordingProcessState state = RecordingProcessState.Idle,
string? profile = null,
bool isPaused = false)
string? profile = null)
{
return new RecordingStatus(
isRecording,
@@ -233,8 +209,7 @@ public sealed class TaskbarIconTests
state == RecordingProcessState.Idle ? null : "context.md",
state == RecordingProcessState.Idle ? null : "summary.md",
state,
profile,
isPaused);
profile);
}
[SupportedOSPlatform("windows")]
@@ -1,249 +0,0 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class VenvResemblyzerVoiceEncoderTests
{
[Fact]
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
{
using var fixture = new EncoderFixture();
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
Assert.DoesNotContain(
fixture.Runner.Commands,
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
Assert.Contains(
fixture.Runner.Commands,
command => command.FileName == "python-test"
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("torch==2.14.0+cpu")
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("Resemblyzer==0.1.4")
&& command.Arguments.Contains("--no-deps"));
}
[Fact]
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
{
using var fixture = new EncoderFixture(
warmupExitCode: 21,
warmupError: "incompatible environment");
await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
Assert.Empty(Directory.EnumerateFiles(
fixture.RuntimeFolder,
".ready",
SearchOption.AllDirectories));
}
[Fact]
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
{
var first = new float[256];
first[0] = 2;
var second = new float[256];
second[1] = 3;
using var fixture = new EncoderFixture(VectorOutput([first, second]));
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
Assert.Equal(2, vectors.Count);
Assert.Equal(1f, vectors[0][0], 5);
Assert.Equal(1f, vectors[1][1], 5);
var encodingCommand = Assert.Single(
fixture.Runner.Commands,
command => command.Arguments.Any(
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
Assert.EndsWith(
Path.Combine("Scripts", "python.exe"),
encodingCommand.FileName,
StringComparison.OrdinalIgnoreCase);
Assert.Equal(2, encodingCommand.Arguments.Count);
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
}
[Fact]
public async Task EncodeAsyncRejectsWrongVectorDimension()
{
var vector = new float[255];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("255 dimensions", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsMalformedVectorJson()
{
using var fixture = new EncoderFixture(
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsNonFiniteVector()
{
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
using var fixture = new EncoderFixture(
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsWrongResultCount()
{
var vector = new float[256];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
{
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("zero magnitude", error.Message);
}
[Fact]
public async Task EncodeAsyncReportsFailedLocalCommand()
{
using var fixture = new EncoderFixture("", 17, "runtime failed");
var error = await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("exit code 17", error.Message);
Assert.Contains("runtime failed", error.Message);
}
private static string VectorOutput(IReadOnlyList<float[]> vectors)
{
return $"""
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
{JsonSerializer.Serialize(vectors)}
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
""";
}
private sealed class EncoderFixture : IDisposable
{
public EncoderFixture(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
RuntimeFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"resemblyzer");
Runner = new CapturingCommandRunner(
encodingOutput,
encodingExitCode,
encodingError,
warmupExitCode,
warmupError);
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
Encoder = new VenvResemblyzerVoiceEncoder(
Runner,
Options.Create(options),
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
}
public CapturingCommandRunner Runner { get; }
public VenvResemblyzerVoiceEncoder Encoder { get; }
public string RuntimeFolder { get; }
public void Dispose()
{
if (Directory.Exists(RuntimeFolder))
{
Directory.Delete(RuntimeFolder, recursive: true);
}
}
}
private sealed class CapturingCommandRunner : ICommandRunner
{
private readonly string encodingOutput;
private readonly int encodingExitCode;
private readonly string encodingError;
private readonly int warmupExitCode;
private readonly string warmupError;
public CapturingCommandRunner(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
this.encodingOutput = encodingOutput;
this.encodingExitCode = encodingExitCode;
this.encodingError = encodingError;
this.warmupExitCode = warmupExitCode;
this.warmupError = warmupError;
}
public List<CapturedCommand> Commands { get; } = [];
public Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
if (isEncoding)
{
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
}
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
return Task.FromResult(isWarmup
? new CommandResult(warmupExitCode, string.Empty, warmupError)
: new CommandResult(0, string.Empty, string.Empty));
}
}
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
}
@@ -732,6 +732,7 @@ public sealed class WorkflowRulesEditorTests
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var rulesPath = Path.Combine(root, "rules.yaml");
var builder = new WorkflowRulesEditorInstructionBuilder(
HostOperatingSystem.DetectCurrent(),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions
@@ -749,10 +750,61 @@ public sealed class WorkflowRulesEditorTests
Assert.Contains("list_recent_summaries", instructions);
}
[Fact]
public async Task InstructionBuilderUsesWindowsCommandsAndConceptsOnWindows()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
new HostOperatingSystem(HostOperatingSystemKind.Windows),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
Assert.Contains("Detected host operating system: Windows", instructions);
Assert.Contains("PowerShell", instructions);
Assert.Contains("Windows-style paths", instructions);
Assert.Contains("Windows services", instructions);
Assert.Contains("Task Manager", instructions);
Assert.Contains("Do not use macOS-only commands or concepts", instructions);
Assert.Contains("does not make unavailable integrations available", instructions);
}
[Fact]
public async Task InstructionBuilderUsesMacOSCommandsAndConceptsOnMacOS()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
Assert.Contains("Detected host operating system: macOS", instructions);
Assert.Contains("zsh", instructions);
Assert.Contains("POSIX-style paths", instructions);
Assert.Contains("launchd and LaunchAgents", instructions);
Assert.Contains("Activity Monitor", instructions);
Assert.Contains("Do not use Windows-only commands or concepts", instructions);
Assert.Contains("does not make unavailable integrations available", instructions);
}
[Fact]
public async Task InstructionBuilderUsesPortableConceptsOnUnsupportedHosts()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
new HostOperatingSystem(HostOperatingSystemKind.Unsupported),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
Assert.Contains("Detected host operating system: unsupported", instructions);
Assert.Contains("portable .NET concepts", instructions);
Assert.Contains("Do not assume Windows or macOS", instructions);
}
[Fact]
public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
HostOperatingSystem.DetectCurrent(),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
@@ -767,6 +819,7 @@ public sealed class WorkflowRulesEditorTests
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var options = new MeetingAssistantOptions
{
@@ -784,6 +837,8 @@ public sealed class WorkflowRulesEditorTests
Assert.Contains("read the matching assistant context", instructions);
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
Assert.Contains("Detected host operating system: macOS", instructions);
Assert.Contains("zsh", instructions);
}
[Fact]
@@ -809,32 +864,6 @@ public sealed class WorkflowRulesEditorTests
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
[Fact]
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
{
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
{
SpeakerIdentityId = identity.Id,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-1",
CreatedAt = DateTimeOffset.UtcNow
});
await fixture.Context.SaveChangesAsync();
var tools = fixture.CreateTools();
var searchResult = await tools.SearchIdentities("Sabrina");
var readResult = await tools.ReadIdentity(identity.Id);
Assert.Contains("\"sampleCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", readResult);
Assert.DoesNotContain("vector-1", readResult);
}
[Fact]
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
{
@@ -843,7 +872,7 @@ public sealed class WorkflowRulesEditorTests
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
@@ -940,30 +969,6 @@ public sealed class WorkflowRulesEditorTests
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
}
[Fact]
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
{
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
{
Draft = "My unfinished draft"
};
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
var task = requested.SendPromptAsync(prompt);
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(requested.IsThinking);
Assert.Empty(pipeline.LastConversation);
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
Assert.Equal("My unfinished draft", existing.Draft);
Assert.Empty(existing.Messages);
pipeline.Release.SetResult();
await task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(requested.IsThinking);
}
[Fact]
public async Task ViewModelShowsAgentErrorWhenRequestTimesOut()
{
@@ -52,14 +52,14 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
cachedMeetingDay = today;
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
logger.LogInformation(
"Synced {MeetingCount} Outlook Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
"Synced {MeetingCount} Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
cachedMeetings.Count,
today,
nextSyncAt);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(exception, "Could not read today's Outlook Teams meetings for recording prompts");
logger.LogWarning(exception, "Could not read today's Teams calendar meetings for recording prompts");
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
return;
}
@@ -32,14 +32,7 @@ Calendar-Feature is refined, Golem feature needs more work.
PROJECT.md should use this structure:
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
```markdown
---
name: Project display name
description: Concise project purpose and distinguishing context.
---
# Executive Summary
<Elevator Pitch>
+26
View File
@@ -0,0 +1,26 @@
namespace MeetingAssistant;
public enum HostOperatingSystemKind
{
Windows,
MacOS,
Unsupported
}
public sealed record HostOperatingSystem(HostOperatingSystemKind Kind)
{
public static HostOperatingSystem DetectCurrent()
{
if (OperatingSystem.IsWindows())
{
return new HostOperatingSystem(HostOperatingSystemKind.Windows);
}
if (OperatingSystem.IsMacOS())
{
return new HostOperatingSystem(HostOperatingSystemKind.MacOS);
}
return new HostOperatingSystem(HostOperatingSystemKind.Unsupported);
}
}
@@ -1,5 +1,4 @@
using Microsoft.Extensions.Configuration;
using MeetingAssistant.Speakers;
namespace MeetingAssistant.LaunchProfiles;
@@ -49,7 +48,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
var options = BindDefaultOptions();
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
ValidateSpeakerSampleDurations(options, DefaultProfileName);
return new LaunchProfile(DefaultProfileName, options);
}
@@ -61,7 +59,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
profileSection.Bind(options);
ApplyArrayOverrides(profileSection, options);
ValidateSpeakerSampleDurations(options, profileName);
return new LaunchProfile(profileName, options);
}
@@ -152,18 +149,6 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
: name.Trim();
}
private static void ValidateSpeakerSampleDurations(
MeetingAssistantOptions options,
string profileName)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
options.SpeakerIdentification,
$"Launch profile '{profileName}' has invalid speaker sample durations");
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
options.SpeakerIdentification.Resemblyzer,
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
}
private static void ApplyArrayOverrides(
IConfigurationSection profileSection,
MeetingAssistantOptions options)
@@ -0,0 +1,47 @@
using MeetingAssistant.LaunchProfiles;
namespace MeetingAssistant.MacOs;
public sealed record MacOsDesktopControlManifest(
string BaseUrl,
string StatusUrl,
string OpenAgentUrl,
IReadOnlyList<MacOsDesktopProfile> Profiles,
IReadOnlyList<MacOsDesktopHotkey> Hotkeys)
{
public static MacOsDesktopControlManifest Create(
string baseUrl,
IReadOnlyList<LaunchProfile> profiles,
IReadOnlyList<LaunchProfileHotkey> hotkeys)
{
var normalizedBaseUrl = baseUrl.TrimEnd('/');
return new MacOsDesktopControlManifest(
normalizedBaseUrl,
$"{normalizedBaseUrl}/recording/status",
$"{normalizedBaseUrl}/diagnostics/settings-and-logs",
profiles
.Select(profile => new MacOsDesktopProfile(
profile.Name,
profile.Options.Hotkey.Toggle))
.ToArray(),
hotkeys.Select(CreateHotkey).ToArray());
}
private static MacOsDesktopHotkey CreateHotkey(LaunchProfileHotkey hotkey)
{
var endpoint = hotkey.Action switch
{
LaunchProfileHotkeyAction.ToggleRecording => "recording/toggle",
LaunchProfileHotkeyAction.AbortRecording => "recording/abort",
LaunchProfileHotkeyAction.CaptureScreenshot => "meetings/screenshot/capture",
_ => throw new ArgumentOutOfRangeException(nameof(hotkey))
};
return new MacOsDesktopHotkey(
hotkey.Hotkey,
$"/profiles/{Uri.EscapeDataString(hotkey.ProfileName)}/{endpoint}");
}
}
public sealed record MacOsDesktopProfile(string Name, string ToggleShortcut);
public sealed record MacOsDesktopHotkey(string Shortcut, string Endpoint);
@@ -0,0 +1,107 @@
using System.Diagnostics;
using System.Text.Json;
using MeetingAssistant.LaunchProfiles;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.MacOs;
public sealed class MacOsDesktopControlService : IHostedService, IDisposable
{
private readonly ILaunchProfileOptionsProvider launchProfiles;
private readonly IOptions<MeetingAssistantOptions> options;
private readonly IHostApplicationLifetime applicationLifetime;
private readonly ILogger<MacOsDesktopControlService> logger;
private Process? helper;
private string? manifestPath;
public MacOsDesktopControlService(
ILaunchProfileOptionsProvider launchProfiles,
IOptions<MeetingAssistantOptions> options,
IHostApplicationLifetime applicationLifetime,
ILogger<MacOsDesktopControlService> logger)
{
this.launchProfiles = launchProfiles;
this.options = options;
this.applicationLifetime = applicationLifetime;
this.logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
if (!OperatingSystem.IsMacOS())
{
return Task.CompletedTask;
}
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-desktop-controls");
if (!File.Exists(helperPath))
{
logger.LogWarning("macOS desktop controls helper was not found at {HelperPath}", helperPath);
return Task.CompletedTask;
}
var manifest = MacOsDesktopControlManifest.Create(
options.Value.Api.PublicBaseUrl,
launchProfiles.GetProfiles(),
launchProfiles.GetHotkeys());
manifestPath = Path.Combine(
Path.GetTempPath(),
$"meeting-assistant-desktop-{Environment.ProcessId}.json");
File.WriteAllText(
manifestPath,
JsonSerializer.Serialize(
manifest,
new JsonSerializerOptions(JsonSerializerDefaults.Web)));
helper = Process.Start(new ProcessStartInfo
{
FileName = helperPath,
ArgumentList = { manifestPath },
UseShellExecute = false,
CreateNoWindow = true
});
if (helper is null)
{
throw new InvalidOperationException("Could not start the macOS desktop controls helper.");
}
helper.EnableRaisingEvents = true;
helper.Exited += (_, _) =>
{
if (!applicationLifetime.ApplicationStopping.IsCancellationRequested)
{
logger.LogWarning("macOS desktop controls helper exited unexpectedly");
}
};
logger.LogInformation("Started macOS menu-bar controls with {HotkeyCount} global hotkeys", manifest.Hotkeys.Count);
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
if (helper is { HasExited: false })
{
helper.Kill();
helper.WaitForExit(TimeSpan.FromSeconds(2));
}
DeleteManifest();
return Task.CompletedTask;
}
public void Dispose()
{
helper?.Dispose();
DeleteManifest();
}
private void DeleteManifest()
{
if (manifestPath is not null && File.Exists(manifestPath))
{
File.Delete(manifestPath);
manifestPath = null;
}
}
}
@@ -0,0 +1,447 @@
using System.Text.Json;
using MeetingAssistant.Calendar;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace MeetingAssistant.MacOs;
internal sealed record MacOsCalendarEvent(
string Id,
string Subject,
DateTimeOffset Start,
DateTimeOffset End,
string Location,
string Notes,
string Url,
IReadOnlyList<string> Attendees,
bool IsCanceled);
internal interface IMacOsCalendarClient
{
Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
DateTimeOffset from,
DateTimeOffset to,
CancellationToken cancellationToken);
}
internal interface IMacOsMeetingPromptClient
{
Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
MeetingStartPromptRequest request,
CancellationToken cancellationToken);
}
internal sealed class MacOsDesktopIntegrationClient :
IMacOsCalendarClient,
IMacOsMeetingPromptClient,
IActiveWindowScreenshotCapture,
IScreenshotImageCropper
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly ICommandRunner commandRunner;
private readonly string nativeDirectory;
public MacOsDesktopIntegrationClient(ICommandRunner commandRunner)
: this(commandRunner, Path.Combine(AppContext.BaseDirectory, "Native"))
{
}
internal MacOsDesktopIntegrationClient(
ICommandRunner commandRunner,
string nativeDirectory)
{
this.commandRunner = commandRunner;
this.nativeDirectory = nativeDirectory;
}
public async Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
DateTimeOffset from,
DateTimeOffset to,
CancellationToken cancellationToken)
{
var fromValue = from.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
var toValue = to.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
CommandResult result;
try
{
result = await RunHelperAsync(
["calendar", fromValue, toValue],
cancellationToken);
}
catch (MacOsIntegrationException exception)
when (exception.ExitCode == MacOsIntegrationException.CalendarPermissionDeniedExitCode)
{
result = await RunCalendarAutomationFallbackAsync(
fromValue,
toValue,
cancellationToken);
}
return JsonSerializer.Deserialize<MacOsCalendarEvent[]>(result.StandardOutput, JsonOptions) ?? [];
}
public async Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
MeetingStartPromptRequest request,
CancellationToken cancellationToken)
{
var requestPath = Path.Combine(
Path.GetTempPath(),
$"meeting-assistant-prompt-{Guid.NewGuid():N}.json");
try
{
await File.WriteAllTextAsync(
requestPath,
JsonSerializer.Serialize(new
{
subject = request.Meeting.Subject,
start = request.Meeting.Start
}, JsonOptions),
cancellationToken);
var result = await RunHelperAsync(["prompt", requestPath], cancellationToken);
var response = JsonSerializer.Deserialize<MacOsPromptResult>(
result.StandardOutput,
JsonOptions);
return string.Equals(response?.Response, "record", StringComparison.OrdinalIgnoreCase)
? MeetingStartPromptResponse.Record
: MeetingStartPromptResponse.Skip;
}
finally
{
File.Delete(requestPath);
}
}
public async Task<ActiveWindowScreenshot> CaptureAsync(
CancellationToken cancellationToken)
{
var outputPath = Path.Combine(
Path.GetTempPath(),
$"meeting-assistant-screenshot-{Guid.NewGuid():N}.png");
try
{
var result = await RunHelperAsync(["screenshot", outputPath], cancellationToken);
var response = JsonSerializer.Deserialize<MacOsScreenshotResult>(
result.StandardOutput,
JsonOptions);
return new ActiveWindowScreenshot(
await File.ReadAllBytesAsync(outputPath, cancellationToken),
response?.WindowTitle);
}
finally
{
File.Delete(outputPath);
}
}
public async Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
var croppedPath = Path.Combine(
Path.GetDirectoryName(screenshotPath)!,
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
try
{
await RunHelperAsync(
[
"crop",
screenshotPath,
croppedPath,
crop.X.ToString(System.Globalization.CultureInfo.InvariantCulture),
crop.Y.ToString(System.Globalization.CultureInfo.InvariantCulture),
crop.Width.ToString(System.Globalization.CultureInfo.InvariantCulture),
crop.Height.ToString(System.Globalization.CultureInfo.InvariantCulture)
],
cancellationToken);
}
catch (MacOsIntegrationException exception)
{
throw new InvalidDataException(
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
exception);
}
return croppedPath;
}
private async Task<CommandResult> RunHelperAsync(
IReadOnlyList<string> arguments,
CancellationToken cancellationToken)
{
if (!OperatingSystem.IsMacOS())
{
throw new PlatformNotSupportedException(
"The native macOS desktop integrations can only run on macOS.");
}
var helperPath = Path.Combine(nativeDirectory, "macos-meeting-integrations");
if (!File.Exists(helperPath))
{
throw new FileNotFoundException(
$"The macOS meeting integrations helper was not found at '{helperPath}'. Build or publish Meeting Assistant on macOS.",
helperPath);
}
var result = await commandRunner.RunAsync(helperPath, arguments, cancellationToken);
if (result.ExitCode != 0)
{
var detail = string.IsNullOrWhiteSpace(result.StandardError)
? "The helper did not provide an error message."
: result.StandardError.Trim();
throw new MacOsIntegrationException(
result.ExitCode,
$"macOS meeting integration '{arguments[0]}' failed with exit code {result.ExitCode}. {detail}");
}
return result;
}
private async Task<CommandResult> RunCalendarAutomationFallbackAsync(
string from,
string to,
CancellationToken cancellationToken)
{
var scriptPath = Path.Combine(nativeDirectory, "macos-calendar-events.js");
if (!File.Exists(scriptPath))
{
throw new FileNotFoundException(
$"The macOS Calendar automation fallback was not found at '{scriptPath}'.",
scriptPath);
}
var result = await commandRunner.RunAsync(
"/usr/bin/osascript",
["-l", "JavaScript", scriptPath, from, to],
cancellationToken);
if (result.ExitCode != 0)
{
var detail = string.IsNullOrWhiteSpace(result.StandardError)
? "Calendar automation did not provide an error message."
: result.StandardError.Trim();
throw new InvalidOperationException(
$"macOS calendar access failed through both EventKit and Calendar automation. {detail}");
}
return result;
}
private sealed record MacOsPromptResult(string Response);
private sealed record MacOsScreenshotResult(string? WindowTitle);
}
internal sealed class MacOsIntegrationException : InvalidOperationException
{
public const int CalendarPermissionDeniedExitCode = 77;
public MacOsIntegrationException(int exitCode, string message)
: base(message)
{
ExitCode = exitCode;
}
public int ExitCode { get; }
}
internal sealed class MacOsMeetingMetadataProvider :
IMeetingMetadataProvider,
IMeetingMetadataDiagnosticProvider
{
private readonly IMacOsCalendarClient client;
private readonly ILogger<MacOsMeetingMetadataProvider> logger;
public MacOsMeetingMetadataProvider(
IMacOsCalendarClient client,
ILogger<MacOsMeetingMetadataProvider> logger)
{
this.client = client;
this.logger = logger;
}
public async Task<MeetingMetadata?> GetCurrentMeetingAsync(
DateTimeOffset startedAt,
CancellationToken cancellationToken)
{
try
{
return await GetCurrentMeetingCoreAsync(startedAt, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogDebug(exception, "macOS calendar meeting metadata was not available");
return null;
}
}
public async Task<MeetingMetadataDiagnosticResult> DiagnoseCurrentMeetingAsync(
DateTimeOffset startedAt,
TimeSpan timeout,
CancellationToken cancellationToken)
{
return await MeetingMetadataDiagnosticRunner.RunAsync(
nameof(MacOsMeetingMetadataProvider),
startedAt,
timeout,
token => GetCurrentMeetingCoreAsync(startedAt, token),
"No matching Teams calendar event was found.",
cancellationToken);
}
private async Task<MeetingMetadata?> GetCurrentMeetingCoreAsync(
DateTimeOffset startedAt,
CancellationToken cancellationToken)
{
var events = await client.GetCalendarEventsAsync(
startedAt.AddHours(-4),
startedAt.AddMinutes(5),
cancellationToken);
var candidates = events
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
.Where(calendarEvent => !calendarEvent.IsCanceled)
.ToList();
var selected = OutlookMeetingCandidateSelector.Select(
candidates,
startedAt.LocalDateTime,
calendarEvent => calendarEvent.Start.LocalDateTime,
calendarEvent => calendarEvent.End.LocalDateTime);
return selected is null ? null : MacOsCalendarEventMapper.CreateMetadata(selected);
}
}
internal sealed class MacOsCalendarMeetingProvider : ICalendarMeetingProvider
{
private readonly IMacOsCalendarClient client;
private readonly ILogger<MacOsCalendarMeetingProvider> logger;
public MacOsCalendarMeetingProvider(
IMacOsCalendarClient client,
ILogger<MacOsCalendarMeetingProvider> logger)
{
this.client = client;
this.logger = logger;
}
public async Task<IReadOnlyList<CalendarMeeting>> GetRecordingPromptCandidatesForDayAsync(
DateOnly day,
CancellationToken cancellationToken)
{
var dayStart = AtLocalMidnight(day);
var dayEnd = AtLocalMidnight(day.AddDays(1));
try
{
var events = await client.GetCalendarEventsAsync(dayStart, dayEnd, cancellationToken);
return events
.Where(calendarEvent => !calendarEvent.IsCanceled)
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
.Where(calendarEvent => calendarEvent.Start >= dayStart && calendarEvent.Start < dayEnd)
.OrderBy(calendarEvent => calendarEvent.Start)
.Select(calendarEvent => new CalendarMeeting(
calendarEvent.Id,
CalendarAppointmentMetadata.ReadSubject(calendarEvent.Subject),
calendarEvent.Start,
calendarEvent.End,
MacOsCalendarEventMapper.CreateMetadata(calendarEvent),
calendarEvent.IsCanceled))
.ToList();
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogDebug(exception, "macOS calendar meetings were not available");
return [];
}
}
private static DateTimeOffset AtLocalMidnight(DateOnly day)
{
var localMidnight = day.ToDateTime(TimeOnly.MinValue);
return new DateTimeOffset(
localMidnight,
TimeZoneInfo.Local.GetUtcOffset(localMidnight));
}
}
internal sealed class MacOsMeetingStartPromptService : IMeetingStartPromptService
{
private readonly IMacOsMeetingPromptClient client;
private readonly ILogger<MacOsMeetingStartPromptService> logger;
public MacOsMeetingStartPromptService(
IMacOsMeetingPromptClient client,
ILogger<MacOsMeetingStartPromptService> logger)
{
this.client = client;
this.logger = logger;
}
public async Task ShowPromptAsync(
MeetingStartPromptRequest request,
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
try
{
var response = await client.ShowMeetingStartPromptAsync(request, cancellationToken);
await handleResponseAsync(response, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(exception, "Failed to display native macOS meeting-start prompt");
}
}
}
public static class MacOsMeetingIntegrationRegistration
{
public static IServiceCollection Add(
IServiceCollection services,
bool isMacOs)
{
if (isMacOs)
{
services.TryAddSingleton<ICommandRunner, ProcessCommandRunner>();
services.TryAddSingleton<MacOsDesktopIntegrationClient>();
services.TryAddSingleton<IMacOsCalendarClient>(services =>
services.GetRequiredService<MacOsDesktopIntegrationClient>());
services.TryAddSingleton<IMacOsMeetingPromptClient>(services =>
services.GetRequiredService<MacOsDesktopIntegrationClient>());
services.AddSingleton<IMeetingMetadataProvider, MacOsMeetingMetadataProvider>();
services.AddSingleton<ICalendarMeetingProvider, MacOsCalendarMeetingProvider>();
services.AddSingleton<IMeetingStartPromptService, MacOsMeetingStartPromptService>();
services.AddSingleton<IActiveWindowScreenshotCapture>(services =>
services.GetRequiredService<MacOsDesktopIntegrationClient>());
services.AddSingleton<IScreenshotImageCropper>(services =>
services.GetRequiredService<MacOsDesktopIntegrationClient>());
return services;
}
services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
services.AddSingleton<IScreenshotImageCropper, UnavailableScreenshotImageCropper>();
return services;
}
}
internal static class MacOsCalendarEventMapper
{
public static bool IsTeamsEvent(MacOsCalendarEvent calendarEvent)
{
return CalendarAppointmentMetadata.IsTeamsAppointment(
calendarEvent.Subject,
calendarEvent.Location,
calendarEvent.Notes,
calendarEvent.Url);
}
public static MeetingMetadata CreateMetadata(MacOsCalendarEvent calendarEvent)
{
return CalendarAppointmentMetadata.Create(
calendarEvent.Subject,
calendarEvent.Attendees,
calendarEvent.Notes,
calendarEvent.End);
}
}
@@ -0,0 +1,161 @@
using System.Diagnostics;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.MacOs;
public sealed class MacOsWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
{
private readonly IOptions<MeetingAssistantOptions> options;
public MacOsWorkflowRulesEditorWindowService(IOptions<MeetingAssistantOptions> options)
{
this.options = options;
}
public void Show()
{
if (!OperatingSystem.IsMacOS())
{
return;
}
var url = $"{options.Value.Api.PublicBaseUrl.TrimEnd('/')}/diagnostics/settings-and-logs";
Process.Start(new ProcessStartInfo
{
FileName = "/usr/bin/open",
ArgumentList = { url },
UseShellExecute = false
});
}
}
public sealed class MacOsWorkflowRulesEditorSession
{
private readonly WorkflowRulesEditorChatViewModel viewModel;
private readonly SemaphoreSlim sendLock = new(1, 1);
public MacOsWorkflowRulesEditorSession(WorkflowRulesEditorChatViewModel viewModel)
{
this.viewModel = viewModel;
}
public async Task<MacOsWorkflowRulesEditorChatResponse> SendAsync(
string? message,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(message))
{
return Snapshot();
}
await sendLock.WaitAsync(cancellationToken);
try
{
viewModel.Draft = message;
await viewModel.SendAsync(cancellationToken);
return Snapshot();
}
finally
{
sendLock.Release();
}
}
private MacOsWorkflowRulesEditorChatResponse Snapshot()
{
return new MacOsWorkflowRulesEditorChatResponse(
viewModel.Messages
.Where(item => item.Kind == WorkflowRulesEditorConversationItemKind.Message)
.Select(item => new MacOsWorkflowRulesEditorMessage(
item.Message!.Role.ToString().ToLowerInvariant(),
item.Message.Content))
.ToArray());
}
}
public sealed record MacOsWorkflowRulesEditorChatRequest(string? Message);
public sealed record MacOsWorkflowRulesEditorChatResponse(
IReadOnlyList<MacOsWorkflowRulesEditorMessage> Messages);
public sealed record MacOsWorkflowRulesEditorMessage(string Role, string Content);
public static class MacOsWorkflowRulesEditorPage
{
public const string Html = """
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Meeting Summary Agent</title>
<style>
:root { color-scheme: dark; font: 15px -apple-system, BlinkMacSystemFont, sans-serif; }
* { box-sizing: border-box; }
body { margin: 0; background: #171719; color: #f5f5f7; }
main { height: 100vh; display: grid; grid-template-rows: auto 1fr auto; padding: 18px; gap: 14px; }
h1 { margin: 0; font-size: 18px; }
#messages { overflow: auto; display: flex; flex-direction: column; gap: 10px; }
.message { max-width: 82%; padding: 10px 12px; border: 1px solid #414147; border-radius: 10px; white-space: pre-wrap; }
.user { align-self: flex-end; background: #293650; }
.agent { align-self: flex-start; background: #232326; }
form { display: grid; grid-template-columns: 1fr auto; gap: 10px; }
textarea { min-height: 78px; resize: vertical; border-radius: 9px; border: 1px solid #55555d; background: #222225; color: inherit; padding: 10px; font: inherit; }
button { min-width: 96px; border: 0; border-radius: 9px; background: #0a84ff; color: white; font-weight: 600; }
button:disabled { opacity: .55; }
</style>
</head>
<body>
<main>
<h1>Meeting Summary Agent</h1>
<section id="messages"></section>
<form id="chat">
<textarea id="prompt" placeholder="ask me what I can do for you"></textarea>
<button id="send">Send</button>
</form>
</main>
<script>
const form = document.querySelector('#chat');
const prompt = document.querySelector('#prompt');
const send = document.querySelector('#send');
const messages = document.querySelector('#messages');
const render = items => {
messages.replaceChildren(...items.map(item => {
const card = document.createElement('div');
card.className = `message ${item.role}`;
card.textContent = item.content;
return card;
}));
messages.scrollTop = messages.scrollHeight;
};
form.addEventListener('submit', async event => {
event.preventDefault();
const message = prompt.value.trim();
if (!message) return;
prompt.value = '';
send.disabled = true;
try {
const response = await fetch('/diagnostics/settings-and-logs/chat', {
method: 'POST',
headers: {'content-type': 'application/json'},
body: JSON.stringify({message})
});
const body = await response.json();
render(body.messages);
} finally {
send.disabled = false;
prompt.focus();
}
});
prompt.addEventListener('keydown', event => {
if (event.key === 'Enter' && !event.shiftKey) {
event.preventDefault();
form.requestSubmit();
}
});
</script>
</body>
</html>
""";
}
+42 -6
View File
@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PreserveCompilationContext>true</PreserveCompilationContext>
<MicrosoftSpeechVersion>1.51.2</MicrosoftSpeechVersion>
<MicrosoftSpeechVersion>1.51.1</MicrosoftSpeechVersion>
</PropertyGroup>
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
@@ -15,21 +15,36 @@
<UseWPF>true</UseWPF>
</PropertyGroup>
<PropertyGroup Condition="$([MSBuild]::IsOSPlatform('OSX')) and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
<MacOsNativeHelpersEnabled>true</MacOsNativeHelpersEnabled>
<MacOsAudioCaptureSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/main.swift</MacOsAudioCaptureSource>
<MacOsAudioCaptureOutputPath>Native/macos-meeting-audio-capture</MacOsAudioCaptureOutputPath>
<MacOsAudioCaptureRid Condition="'$(RuntimeIdentifier)' != ''">$(RuntimeIdentifier)</MacOsAudioCaptureRid>
<MacOsAudioCaptureRid Condition="'$(MacOsAudioCaptureRid)' == ''">$(NETCoreSdkRuntimeIdentifier)</MacOsAudioCaptureRid>
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureRid)' == 'osx-x64'">x86_64</MacOsAudioCaptureArchitecture>
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureArchitecture)' == ''">arm64</MacOsAudioCaptureArchitecture>
<MacOsDesktopControlsSource>$(MSBuildProjectDirectory)/Native/MacOsDesktopControls/main.swift</MacOsDesktopControlsSource>
<MacOsDesktopControlsOutputPath>Native/macos-desktop-controls</MacOsDesktopControlsOutputPath>
<MacOsMeetingIntegrationsSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/main.swift</MacOsMeetingIntegrationsSource>
<MacOsMeetingIntegrationsInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/Info.plist</MacOsMeetingIntegrationsInfoPlist>
<MacOsMeetingIntegrationsOutputPath>Native/macos-meeting-integrations</MacOsMeetingIntegrationsOutputPath>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.17.0" />
<PackageReference Include="DiffPlex" Version="1.9.0" />
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
<PackageReference Include="NAudio" Version="3.1.0" />
<PackageReference Include="NCalcSync" Version="7.2.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="NCalcSync" Version="7.0.2" />
<PackageReference Include="RazorLight" Version="2.3.1" />
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
<PackageReference Include="System.Drawing.Common" Version="10.0.10" />
<PackageReference Include="Whisper.net" Version="1.9.1" />
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
@@ -47,6 +62,7 @@
<None Include="..\docs\meeting-assistant-configuration.md" Link="docs\meeting-assistant-configuration.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
<None Include="Content\Project-AGENTS.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
<None Include="..\openspec\specs\**\*.md" Link="openspec\specs\%(RecursiveDir)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
<None Include="Native\MacOsMeetingIntegrations\calendar-events.js" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" TargetPath="Native\macos-calendar-events.js" />
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
@@ -61,6 +77,26 @@
<Copy SourceFiles="@(MeetingAssistantRootConfig)" DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true" />
</Target>
<Target
Name="BuildMacOsNativeHelpers"
AfterTargets="Build"
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
<MakeDir Directories="$(TargetDir)Native" />
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AVFoundation -framework CoreMedia -framework ScreenCaptureKit &quot;$(MacOsAudioCaptureSource)&quot; -o &quot;$(TargetDir)$(MacOsAudioCaptureOutputPath)&quot;" />
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework Carbon -framework WebKit &quot;$(MacOsDesktopControlsSource)&quot; -o &quot;$(TargetDir)$(MacOsDesktopControlsOutputPath)&quot;" />
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework CoreGraphics -framework EventKit -framework ImageIO -framework UniformTypeIdentifiers -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker &quot;$(MacOsMeetingIntegrationsInfoPlist)&quot; &quot;$(MacOsMeetingIntegrationsSource)&quot; -o &quot;$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)&quot;" />
</Target>
<Target
Name="CopyMacOsNativeHelpersToPublish"
AfterTargets="Publish"
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
<MakeDir Directories="$(PublishDir)Native" />
<Copy SourceFiles="$(TargetDir)$(MacOsAudioCaptureOutputPath)" DestinationFolder="$(PublishDir)Native" />
<Copy SourceFiles="$(TargetDir)$(MacOsDesktopControlsOutputPath)" DestinationFolder="$(PublishDir)Native" />
<Copy SourceFiles="$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)" DestinationFolder="$(PublishDir)Native" />
</Target>
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
<Compile Remove="Recording\NaudioCaptureSource.cs" />
+7 -52
View File
@@ -165,8 +165,6 @@ public sealed class RecordingInactivitySafeguardOptions
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
@@ -183,8 +181,10 @@ public sealed class WhisperLocalOptions
public PyannoteDiarizationOptions Diarization { get; set; } = new();
}
public class PyannoteRuntimeOptions
public sealed class PyannoteDiarizationOptions
{
public bool Enabled { get; set; } = true;
public string DockerCommand { get; set; } = "docker";
public string BaseImage { get; set; } = "python:3.11-slim";
@@ -208,11 +208,6 @@ public class PyannoteRuntimeOptions
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
}
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
{
public bool Enabled { get; set; } = true;
}
public sealed class AzureSpeechOptions
{
public string Endpoint { get; set; } = "";
@@ -345,9 +340,9 @@ public sealed class SpeakerIdentificationOptions
public int MaxSnippetsPerSpeaker { get; set; } = 3;
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
@@ -360,47 +355,6 @@ public sealed class SpeakerIdentificationOptions
public AzureSpeechOptions AzureSpeech { get; set; } = new();
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
}
public sealed class ResemblyzerSpeakerRecognitionOptions
{
public bool Enabled { get; set; }
public int RequiredVectorsPerSpeaker { get; set; } = 5;
public int MaxVectorsPerIdentity { get; set; } = 1000;
public int OutlierPruningMinimumVectors { get; set; } = 20;
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
public double MinimumClusterCohesion { get; set; } = 0.75;
public double MinimumIdentitySimilarity { get; set; } = 0.75;
public double MinimumSimilarityMargin { get; set; } = 0.05;
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
public string PackageVersion { get; set; } = "0.1.4";
public string PythonCommand { get; set; } = "python";
public string TorchVersion { get; set; } = "2.14.0+cpu";
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
public string WebRtcVadVersion { get; set; } = "2.0.14";
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
}
public sealed class SpeakerIdentityPyannoteValidationOptions
@@ -411,8 +365,9 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
public PyannoteRuntimeOptions Diarization { get; set; } = new()
public PyannoteDiarizationOptions Diarization { get; set; } = new()
{
Enabled = true,
CommandTimeout = TimeSpan.FromHours(1),
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
};
@@ -0,0 +1,117 @@
using MeetingAssistant.Calendar;
namespace MeetingAssistant.MeetingNotes;
internal static class CalendarAppointmentMetadata
{
public static MeetingMetadata Create(
string? subject,
IEnumerable<string> attendees,
string? body,
DateTimeOffset end)
{
return new MeetingMetadata(
ReadSubject(subject),
NormalizeAttendees(attendees),
ExtractAgenda(body ?? ""),
end);
}
public static string ReadSubject(string? subject)
{
var trimmed = subject?.Trim();
return string.IsNullOrWhiteSpace(trimmed) ? "Teams meeting" : trimmed;
}
public static bool IsTeamsAppointment(
string? subject,
string? location,
string? body,
string? url = null)
{
return TeamsMeetingMarkerDetector.IsTeamsAppointment(
subject ?? "",
location ?? "",
string.Join(Environment.NewLine, body, url));
}
public static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || TeamsMeetingMarkerDetector.ContainsTeamsMarker(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim()
.Replace("mailto:", "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
return emailStart > 0 && emailEnd > emailStart
? attendee[(emailStart + 1)..emailEnd].Trim()
: attendee.Trim();
}
}
@@ -0,0 +1,58 @@
using System.Diagnostics;
namespace MeetingAssistant.MeetingNotes;
internal static class MeetingMetadataDiagnosticRunner
{
public static async Task<MeetingMetadataDiagnosticResult> RunAsync(
string provider,
DateTimeOffset startedAt,
TimeSpan timeout,
Func<CancellationToken, Task<MeetingMetadata?>> lookupAsync,
string noMatchError,
CancellationToken cancellationToken)
{
var stopwatch = Stopwatch.StartNew();
try
{
var lookup = lookupAsync(cancellationToken);
var metadata = timeout > TimeSpan.Zero
? await lookup.WaitAsync(timeout, cancellationToken)
: await lookup.WaitAsync(cancellationToken);
return new MeetingMetadataDiagnosticResult(
provider,
startedAt,
metadata is not null,
false,
stopwatch.ElapsedMilliseconds,
metadata,
metadata is null ? noMatchError : null);
}
catch (TimeoutException exception)
{
return new MeetingMetadataDiagnosticResult(
provider,
startedAt,
false,
true,
stopwatch.ElapsedMilliseconds,
null,
exception.Message);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
return new MeetingMetadataDiagnosticResult(
provider,
startedAt,
false,
false,
stopwatch.ElapsedMilliseconds,
null,
exception.Message);
}
}
}
@@ -7,18 +7,17 @@ internal static class OutlookClassicAppointmentMetadata
{
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
{
var title = ReadSubject(appointment);
return new MeetingMetadata(
title,
return CalendarAppointmentMetadata.Create(
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")),
ReadAttendees(appointment),
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")),
OutlookClassicCom.ToLocalOffset(end));
}
public static string ReadSubject(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
return CalendarAppointmentMetadata.ReadSubject(
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")));
}
public static bool IsTeamsAppointment(object appointment)
@@ -26,29 +25,12 @@ internal static class OutlookClassicAppointmentMetadata
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
return CalendarAppointmentMetadata.IsTeamsAppointment(subject, location, body);
}
public static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
return CalendarAppointmentMetadata.ExtractAgenda(body);
}
public static IReadOnlyList<string> ReadAttendees(object appointment)
@@ -93,66 +75,7 @@ internal static class OutlookClassicAppointmentMetadata
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim();
var mailtoPrefix = "mailto:";
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
return attendee[(emailStart + 1)..emailEnd].Trim();
}
return attendee.Trim();
return CalendarAppointmentMetadata.NormalizeAttendees(attendees);
}
private static string FormatRecipient(object recipient)
@@ -1,4 +1,3 @@
using System.Diagnostics;
using MeetingAssistant.Calendar;
namespace MeetingAssistant.MeetingNotes;
@@ -36,54 +35,16 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
TimeSpan timeout,
CancellationToken cancellationToken)
{
var stopwatch = Stopwatch.StartNew();
try
{
var lookup = OutlookClassicCom.RunStaAsync(
return await MeetingMetadataDiagnosticRunner.RunAsync(
nameof(OutlookClassicMeetingMetadataProvider),
startedAt,
timeout,
token => OutlookClassicCom.RunStaAsync(
() => GetCurrentMeeting(startedAt),
cancellationToken,
"Meeting Assistant Outlook COM Lookup");
var metadata = timeout > TimeSpan.Zero
? await lookup.WaitAsync(timeout, cancellationToken)
: await lookup.WaitAsync(cancellationToken);
stopwatch.Stop();
return new MeetingMetadataDiagnosticResult(
nameof(OutlookClassicMeetingMetadataProvider),
startedAt,
metadata is not null,
false,
stopwatch.ElapsedMilliseconds,
metadata,
metadata is null ? "No matching Teams appointment was found." : null);
}
catch (TimeoutException exception)
{
stopwatch.Stop();
return new MeetingMetadataDiagnosticResult(
nameof(OutlookClassicMeetingMetadataProvider),
startedAt,
false,
true,
stopwatch.ElapsedMilliseconds,
null,
exception.Message);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
stopwatch.Stop();
return new MeetingMetadataDiagnosticResult(
nameof(OutlookClassicMeetingMetadataProvider),
startedAt,
false,
false,
stopwatch.ElapsedMilliseconds,
null,
exception.Message);
}
token,
"Meeting Assistant Outlook COM Lookup"),
"No matching Teams appointment was found.",
cancellationToken);
}
private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt)
@@ -0,0 +1,289 @@
import AppKit
import Carbon
import Foundation
import WebKit
struct Manifest: Decodable {
let baseUrl: String
let statusUrl: String
let openAgentUrl: String
let profiles: [Profile]
let hotkeys: [Hotkey]
}
struct Profile: Decodable {
let name: String
let toggleShortcut: String
}
struct Hotkey: Decodable {
let shortcut: String
let endpoint: String
}
struct RecordingStatus: Decodable {
let isRecording: Bool
let state: String
let launchProfile: String?
}
@main
final class DesktopControls: NSObject, NSApplicationDelegate, NSMenuDelegate {
private let manifest: Manifest
private var status = RecordingStatus(isRecording: false, state: "Idle", launchProfile: nil)
private var statusItem: NSStatusItem!
private var registeredHotkeys: [UInt32: Hotkey] = [:]
private var timer: Timer?
private var agentWindow: NSWindow?
override init() {
guard CommandLine.arguments.count == 2,
let data = try? Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[1])),
let decoded = try? JSONDecoder().decode(Manifest.self, from: data) else {
fputs("Usage: macos-desktop-controls <manifest.json>\n", stderr)
exit(2)
}
manifest = decoded
super.init()
}
static func main() {
let application = NSApplication.shared
let delegate = DesktopControls()
application.delegate = delegate
application.setActivationPolicy(.accessory)
application.run()
}
func applicationDidFinishLaunching(_ notification: Notification) {
statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.squareLength)
statusItem.button?.image = NSImage(
systemSymbolName: "waveform.circle",
accessibilityDescription: "Meeting Assistant")
let menu = NSMenu()
menu.delegate = self
statusItem.menu = menu
registerHotkeys()
refreshStatus()
timer = Timer.scheduledTimer(withTimeInterval: 2, repeats: true) { [weak self] _ in
self?.refreshStatus()
}
}
func applicationWillTerminate(_ notification: Notification) {
timer?.invalidate()
}
func menuWillOpen(_ menu: NSMenu) {
rebuildMenu(menu)
refreshStatus()
}
private func rebuildMenu(_ menu: NSMenu) {
menu.removeAllItems()
let stateItem = NSMenuItem(title: tooltip, action: nil, keyEquivalent: "")
stateItem.isEnabled = false
menu.addItem(stateItem)
menu.addItem(.separator())
menu.addItem(item("Open agent", #selector(openAgent)))
if status.isRecording {
menu.addItem(item("Stop meeting recording and transcribe", #selector(stopRecording)))
menu.addItem(item("Cancel meeting recording and discard", #selector(abortRecording)))
} else {
for (index, profile) in manifest.profiles.enumerated() {
let entry = item(
"Start meeting recording (\(profile.name))",
#selector(startProfile(_:)))
entry.tag = index
entry.keyEquivalent = menuKey(from: profile.toggleShortcut)
entry.keyEquivalentModifierMask = menuModifiers(from: profile.toggleShortcut)
menu.addItem(entry)
}
}
menu.addItem(.separator())
menu.addItem(item("Exit", #selector(exitApplication)))
}
private var tooltip: String {
if status.isRecording {
return status.launchProfile.map { "Meeting Assistant — recording (\($0))" }
?? "Meeting Assistant — recording"
}
return status.state.lowercased() == "summarizing"
? "Meeting Assistant — summarizing"
: "Meeting Assistant — idle"
}
private func item(_ title: String, _ action: Selector) -> NSMenuItem {
let result = NSMenuItem(title: title, action: action, keyEquivalent: "")
result.target = self
return result
}
@objc private func openAgent() {
if agentWindow == nil {
let webView = WKWebView(frame: .zero)
let window = NSWindow(
contentRect: NSRect(x: 0, y: 0, width: 840, height: 760),
styleMask: [.titled, .closable, .miniaturizable, .resizable],
backing: .buffered,
defer: false)
window.title = "Meeting Summary Agent"
window.contentView = webView
window.center()
window.isReleasedWhenClosed = false
webView.load(URLRequest(url: URL(string: manifest.openAgentUrl)!))
agentWindow = window
}
NSApplication.shared.activate(ignoringOtherApps: true)
agentWindow?.makeKeyAndOrderFront(nil)
}
@objc private func startProfile(_ sender: NSMenuItem) {
let profile = manifest.profiles[sender.tag]
post(path: "/profiles/\(escaped(profile.name))/recording/start")
}
@objc private func stopRecording() {
post(path: "/recording/stop")
}
@objc private func abortRecording() {
post(path: "/recording/abort")
}
@objc private func exitApplication() {
post(path: "/application/exit")
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
NSApplication.shared.terminate(nil)
}
}
private func refreshStatus() {
guard let url = URL(string: manifest.statusUrl) else { return }
URLSession.shared.dataTask(with: url) { [weak self] data, _, _ in
guard let data,
let next = try? JSONDecoder().decode(RecordingStatus.self, from: data) else { return }
DispatchQueue.main.async {
self?.status = next
self?.statusItem.button?.image = NSImage(
systemSymbolName: next.isRecording ? "record.circle.fill" : "waveform.circle",
accessibilityDescription: self?.tooltip)
}
}.resume()
}
private func post(path: String) {
post(url: manifest.baseUrl + path)
}
private func post(url value: String) {
guard let url = URL(string: value) else { return }
var request = URLRequest(url: url)
request.httpMethod = "POST"
URLSession.shared.dataTask(with: request) { [weak self] _, _, _ in
DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) {
self?.refreshStatus()
}
}.resume()
}
private func registerHotkeys() {
var eventType = EventTypeSpec(
eventClass: OSType(kEventClassKeyboard),
eventKind: OSType(kEventHotKeyPressed))
InstallEventHandler(
GetApplicationEventTarget(),
{ _, event, context in
guard let event, let context else { return noErr }
var identifier = EventHotKeyID()
GetEventParameter(
event,
EventParamName(kEventParamDirectObject),
EventParamType(typeEventHotKeyID),
nil,
MemoryLayout<EventHotKeyID>.size,
nil,
&identifier)
let owner = Unmanaged<DesktopControls>.fromOpaque(context).takeUnretainedValue()
if let hotkey = owner.registeredHotkeys[identifier.id] {
owner.post(path: hotkey.endpoint)
}
return noErr
},
1,
&eventType,
Unmanaged.passUnretained(self).toOpaque(),
nil)
for (index, hotkey) in manifest.hotkeys.enumerated() {
guard let keyCode = carbonKeyCode(from: hotkey.shortcut) else { continue }
let identifier = EventHotKeyID(signature: fourCC("MTGA"), id: UInt32(index + 1))
var reference: EventHotKeyRef?
let result = RegisterEventHotKey(
keyCode,
carbonModifiers(from: hotkey.shortcut),
identifier,
GetApplicationEventTarget(),
0,
&reference)
if result == noErr {
registeredHotkeys[identifier.id] = hotkey
}
}
}
private func escaped(_ value: String) -> String {
value.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? value
}
}
private func tokens(_ shortcut: String) -> [String] {
shortcut.uppercased().split(separator: "+").map(String.init)
}
private func carbonModifiers(from shortcut: String) -> UInt32 {
let parts = tokens(shortcut)
var result: UInt32 = 0
if parts.contains("CTRL") || parts.contains("CONTROL") { result |= UInt32(controlKey) }
if parts.contains("ALT") || parts.contains("OPTION") { result |= UInt32(optionKey) }
if parts.contains("SHIFT") { result |= UInt32(shiftKey) }
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
result |= UInt32(cmdKey)
}
return result
}
private func menuModifiers(from shortcut: String) -> NSEvent.ModifierFlags {
let parts = tokens(shortcut)
var result: NSEvent.ModifierFlags = []
if parts.contains("CTRL") || parts.contains("CONTROL") { result.insert(.control) }
if parts.contains("ALT") || parts.contains("OPTION") { result.insert(.option) }
if parts.contains("SHIFT") { result.insert(.shift) }
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
result.insert(.command)
}
return result
}
private func menuKey(from shortcut: String) -> String {
tokens(shortcut).last?.lowercased() ?? ""
}
private func carbonKeyCode(from shortcut: String) -> UInt32? {
guard let key = tokens(shortcut).last, key.count == 1, let scalar = key.unicodeScalars.first else {
return nil
}
let codes: [Character: UInt32] = [
"A": 0, "S": 1, "D": 2, "F": 3, "H": 4, "G": 5, "Z": 6, "X": 7,
"C": 8, "V": 9, "B": 11, "Q": 12, "W": 13, "E": 14, "R": 15,
"Y": 16, "T": 17, "M": 46, "L": 37, "N": 45, "P": 35, "O": 31,
"I": 34, "U": 32, "J": 38, "K": 40
]
return codes[Character(String(scalar))]
}
private func fourCC(_ value: String) -> OSType {
value.utf8.reduce(0) { ($0 << 8) + OSType($1) }
}
@@ -0,0 +1,328 @@
import AVFoundation
import CoreMedia
import Darwin
import Foundation
import ScreenCaptureKit
private enum CaptureFailure: Error, CustomStringConvertible {
case usage
case invalidFormat
case microphonePermission
case noDisplay
case audioConverter(String)
case systemAudio(String)
var description: String {
switch self {
case .usage:
return "Usage: macos-meeting-audio-capture <microphone|system> <sample-rate> <channels>"
case .invalidFormat:
return "Sample rate must be positive and channels must be 1 or 2."
case .microphonePermission:
return "Microphone permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Microphone."
case .noDisplay:
return "ScreenCaptureKit could not find a display for system-audio capture."
case .audioConverter(let detail):
return "Native PCM conversion failed: \(detail)"
case .systemAudio(let detail):
return "Screen Recording/System Audio permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Screen & System Audio Recording. \(detail)"
}
}
}
private final class ErrorReporter: @unchecked Sendable {
private let lock = NSLock()
func write(_ message: String) {
lock.lock()
defer { lock.unlock() }
let line = message.hasSuffix("\n") ? message : message + "\n"
if let data = line.data(using: .utf8) {
try? FileHandle.standardError.write(contentsOf: data)
}
}
func exit(with failure: CaptureFailure, code: Int32) -> Never {
write(failure.description)
Darwin.exit(code)
}
}
private final class PcmWriter: @unchecked Sendable {
private let targetFormat: AVAudioFormat
private let output = FileHandle.standardOutput
private let reporter: ErrorReporter
private let lock = NSLock()
private var converter: AVAudioConverter?
init(sampleRate: Int, channels: Int, reporter: ErrorReporter) throws {
guard let format = AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: Double(sampleRate),
channels: AVAudioChannelCount(channels),
interleaved: true
) else {
throw CaptureFailure.invalidFormat
}
targetFormat = format
self.reporter = reporter
}
func write(_ input: AVAudioPCMBuffer) {
lock.lock()
defer { lock.unlock() }
do {
let outputBuffer = try convert(input)
let buffers = UnsafeMutableAudioBufferListPointer(outputBuffer.mutableAudioBufferList)
guard let audioBuffer = buffers.first, let bytes = audioBuffer.mData else {
return
}
let data = Data(bytes: bytes, count: Int(audioBuffer.mDataByteSize))
try output.write(contentsOf: data)
} catch {
reporter.exit(with: .audioConverter(error.localizedDescription), code: 70)
}
}
private func convert(_ input: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer {
let sourceFormat = input.format
let activeConverter: AVAudioConverter
if let converter {
activeConverter = converter
} else {
guard let created = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
throw CaptureFailure.audioConverter("AVAudioConverter could not convert \(sourceFormat) to \(targetFormat).")
}
converter = created
activeConverter = created
}
let ratio = targetFormat.sampleRate / sourceFormat.sampleRate
let capacity = max(1, AVAudioFrameCount(ceil(Double(input.frameLength) * ratio)) + 32)
guard let converted = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
throw CaptureFailure.audioConverter("Could not allocate the target PCM buffer.")
}
var suppliedInput = false
var conversionError: NSError?
let status = activeConverter.convert(to: converted, error: &conversionError) { _, inputStatus in
if suppliedInput {
inputStatus.pointee = .noDataNow
return nil
}
suppliedInput = true
inputStatus.pointee = .haveData
return input
}
if status == .error {
throw conversionError ?? CaptureFailure.audioConverter("AVAudioConverter returned an unknown error.")
}
return converted
}
}
private final class MicrophoneCapture: @unchecked Sendable {
private let engine = AVAudioEngine()
private let writer: PcmWriter
init(writer: PcmWriter) {
self.writer = writer
}
func run() async throws -> Never {
let authorized = await withCheckedContinuation { continuation in
AVCaptureDevice.requestAccess(for: .audio) { granted in
continuation.resume(returning: granted)
}
}
guard authorized else {
throw CaptureFailure.microphonePermission
}
let input = engine.inputNode
let inputFormat = input.outputFormat(forBus: 0)
input.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [writer] buffer, _ in
writer.write(buffer)
}
engine.prepare()
try engine.start()
while true {
try await Task.sleep(for: .seconds(3_600))
}
}
}
private final class SystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
private let writer: PcmWriter
private let reporter: ErrorReporter
private let sampleQueue = DispatchQueue(label: "meeting-assistant.system-audio")
private var stream: SCStream?
init(writer: PcmWriter, reporter: ErrorReporter) {
self.writer = writer
self.reporter = reporter
}
func run(sampleRate: Int, channels: Int) async throws -> Never {
let content: SCShareableContent
do {
content = try await SCShareableContent.excludingDesktopWindows(
false,
onScreenWindowsOnly: false
)
} catch {
throw CaptureFailure.systemAudio(error.localizedDescription)
}
guard let display = content.displays.first else {
throw CaptureFailure.noDisplay
}
let filter = SCContentFilter(
display: display,
excludingApplications: [],
exceptingWindows: []
)
let configuration = SCStreamConfiguration()
configuration.capturesAudio = true
configuration.excludesCurrentProcessAudio = false
configuration.sampleRate = sampleRate
configuration.channelCount = channels
configuration.width = 2
configuration.height = 2
configuration.showsCursor = false
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
let captureStream = SCStream(filter: filter, configuration: configuration, delegate: self)
try captureStream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
do {
try await captureStream.startCapture()
} catch {
throw CaptureFailure.systemAudio(error.localizedDescription)
}
stream = captureStream
while true {
try await Task.sleep(for: .seconds(3_600))
}
}
func stream(
_ stream: SCStream,
didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
of outputType: SCStreamOutputType
) {
guard outputType == .audio,
sampleBuffer.isValid,
CMSampleBufferDataIsReady(sampleBuffer),
let description = CMSampleBufferGetFormatDescription(sampleBuffer),
let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description)
else {
return
}
let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
guard frameCount > 0,
let sourceFormat = AVAudioFormat(streamDescription: streamDescription)
else {
return
}
var bufferListSize = 0
var retainedBlockBuffer: CMBlockBuffer?
var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
sampleBuffer,
bufferListSizeNeededOut: &bufferListSize,
bufferListOut: nil,
bufferListSize: 0,
blockBufferAllocator: nil,
blockBufferMemoryAllocator: nil,
flags: 0,
blockBufferOut: &retainedBlockBuffer
)
guard status == noErr else {
reporter.exit(
with: .audioConverter("CoreMedia could not describe system audio (OSStatus \(status))."),
code: 71
)
}
let rawBufferList = UnsafeMutableRawPointer.allocate(
byteCount: bufferListSize,
alignment: MemoryLayout<AudioBufferList>.alignment
)
defer { rawBufferList.deallocate() }
let audioBufferList = rawBufferList.bindMemory(to: AudioBufferList.self, capacity: 1)
status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
sampleBuffer,
bufferListSizeNeededOut: nil,
bufferListOut: audioBufferList,
bufferListSize: bufferListSize,
blockBufferAllocator: nil,
blockBufferMemoryAllocator: nil,
flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment),
blockBufferOut: &retainedBlockBuffer
)
guard status == noErr,
let buffer = AVAudioPCMBuffer(
pcmFormat: sourceFormat,
bufferListNoCopy: audioBufferList,
deallocator: nil
)
else {
reporter.exit(
with: .audioConverter("CoreMedia could not expose system audio (OSStatus \(status))."),
code: 71
)
}
buffer.frameLength = frameCount
writer.write(buffer)
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
reporter.exit(with: .systemAudio(error.localizedDescription), code: 72)
}
}
@main
private struct MacOsMeetingAudioCapture {
static func main() async {
let reporter = ErrorReporter()
do {
let arguments = CommandLine.arguments
guard arguments.count == 4,
let sampleRate = Int(arguments[2]),
let channels = Int(arguments[3])
else {
throw CaptureFailure.usage
}
guard sampleRate > 0, channels == 1 || channels == 2 else {
throw CaptureFailure.invalidFormat
}
let writer = try PcmWriter(sampleRate: sampleRate, channels: channels, reporter: reporter)
switch arguments[1] {
case "microphone":
try await MicrophoneCapture(writer: writer).run()
case "system":
try await SystemAudioCapture(writer: writer, reporter: reporter)
.run(sampleRate: sampleRate, channels: channels)
default:
throw CaptureFailure.usage
}
} catch let failure as CaptureFailure {
reporter.exit(with: failure, code: 64)
} catch {
reporter.write(error.localizedDescription)
Darwin.exit(1)
}
}
}
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>cloud.schweigert.meeting-assistant.integrations</string>
<key>CFBundleName</key>
<string>Meeting Assistant Integrations</string>
<key>NSCalendarsUsageDescription</key>
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
<key>NSCalendarsFullAccessUsageDescription</key>
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
</dict>
</plist>
@@ -0,0 +1,68 @@
function safe(read, fallback) {
try {
const value = read();
return value === null || value === undefined ? fallback : value;
} catch (_) {
return fallback;
}
}
function participantText(participant) {
const name = String(safe(() => participant.displayName(), "")).trim();
const email = String(safe(() => participant.email(), "")).trim();
if (!name) {
return email;
}
if (!email || name.toLowerCase() === email.toLowerCase()) {
return name;
}
return `${name} <${email}>`;
}
function run(argv) {
if (argv.length !== 2) {
throw new Error("Usage: calendar-events.js <from-iso8601> <to-iso8601>");
}
const from = new Date(argv[0]);
const to = new Date(argv[1]);
const calendar = Application("Calendar");
const result = [];
for (const sourceCalendar of calendar.calendars()) {
const events = sourceCalendar.events.whose({
_and: [
{ startDate: { _lessThan: to } },
{ endDate: { _greaterThan: from } }
]
})();
for (const event of events) {
const start = event.startDate();
const end = event.endDate();
const attendees = safe(() => event.attendees(), [])
.map(participantText)
.filter(value => value.length > 0);
const organizer = String(safe(() => event.organizer(), "")).trim();
if (organizer && !attendees.includes(organizer)) {
attendees.unshift(organizer);
}
result.push({
id: String(safe(() => event.uid(), `${start.toISOString()}|${event.summary()}`)),
subject: String(safe(() => event.summary(), "")),
start: start.toISOString(),
end: end.toISOString(),
location: String(safe(() => event.location(), "")),
notes: String(safe(() => event.description(), "")),
url: String(safe(() => event.url(), "")),
attendees: attendees,
isCanceled: String(safe(() => event.status(), ""))
.toLowerCase()
.includes("cancel")
});
}
}
result.sort((left, right) => left.start.localeCompare(right.start));
return JSON.stringify(result);
}
@@ -0,0 +1,330 @@
import AppKit
import CoreGraphics
import EventKit
import Foundation
import ImageIO
import UniformTypeIdentifiers
private struct CalendarEventOutput: Encodable {
let id: String
let subject: String
let start: Date
let end: Date
let location: String
let notes: String
let url: String
let attendees: [String]
let isCanceled: Bool
}
private struct PromptRequest: Decodable {
let subject: String
let start: Date
}
private struct PromptOutput: Encodable {
let response: String
}
private struct ScreenshotOutput: Encodable {
let windowTitle: String?
}
private struct SelfTestOutput: Encodable {
let status: String
let features: [String]
}
private enum IntegrationError: LocalizedError {
case usage
case invalidDate(String)
case calendarPermission
case screenRecordingPermission
case noFrontmostApplication
case noActiveWindow
case screenshotCapture
case screenshotWrite
var errorDescription: String? {
switch self {
case .usage:
return "Usage: macos-meeting-integrations self-test | calendar <from-iso8601> <to-iso8601> | prompt <request.json> | screenshot <output.png> | crop <input.png> <output.png> <x> <y> <width> <height>"
case .invalidDate(let value):
return "Invalid ISO-8601 date: \(value)"
case .calendarPermission:
return "Calendar Full Access permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Calendars."
case .screenRecordingPermission:
return "Screen & System Audio Recording permission is required. Enable Meeting Assistant under System Settings > Privacy & Security."
case .noFrontmostApplication:
return "Could not determine the frontmost macOS application."
case .noActiveWindow:
return "Could not find an on-screen window for the frontmost macOS application."
case .screenshotCapture:
return "macOS could not capture the active application window."
case .screenshotWrite:
return "macOS could not write the active-window screenshot as PNG."
}
}
}
@main
private enum MacOsMeetingIntegrations {
@MainActor
static func main() async {
do {
let arguments = Array(CommandLine.arguments.dropFirst())
guard let command = arguments.first else {
throw IntegrationError.usage
}
switch command {
case "self-test":
try writeJson(SelfTestOutput(
status: "ok",
features: ["calendar", "prompt", "screenshot", "crop"]))
case "calendar":
guard arguments.count == 3 else {
throw IntegrationError.usage
}
let from = try parseDate(arguments[1])
let to = try parseDate(arguments[2])
try await writeCalendar(from: from, to: to)
case "prompt":
guard arguments.count == 2 else {
throw IntegrationError.usage
}
try showPrompt(requestPath: arguments[1])
case "screenshot":
guard arguments.count == 2 else {
throw IntegrationError.usage
}
try captureScreenshot(outputPath: arguments[1])
case "crop":
guard arguments.count == 7,
let x = Int(arguments[3]),
let y = Int(arguments[4]),
let width = Int(arguments[5]),
let height = Int(arguments[6]) else {
throw IntegrationError.usage
}
try cropScreenshot(
inputPath: arguments[1],
outputPath: arguments[2],
x: x,
y: y,
width: width,
height: height)
default:
throw IntegrationError.usage
}
} catch {
fputs("\(error.localizedDescription)\n", stderr)
if let integrationError = error as? IntegrationError,
case .calendarPermission = integrationError {
exit(77)
}
exit(1)
}
}
private static func parseDate(_ value: String) throws -> Date {
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
if let date = formatter.date(from: value) {
return date
}
formatter.formatOptions = [.withInternetDateTime]
guard let date = formatter.date(from: value) else {
throw IntegrationError.invalidDate(value)
}
return date
}
private static func writeCalendar(from: Date, to: Date) async throws {
let store = EKEventStore()
guard try await requestCalendarAccess(store) else {
throw IntegrationError.calendarPermission
}
let predicate = store.predicateForEvents(withStart: from, end: to, calendars: nil)
let events = store.events(matching: predicate)
.map { event in
CalendarEventOutput(
id: event.eventIdentifier ?? "\(event.startDate.timeIntervalSince1970)|\(event.title ?? "")",
subject: event.title ?? "",
start: event.startDate,
end: event.endDate,
location: event.location ?? "",
notes: event.notes ?? "",
url: event.url?.absoluteString ?? "",
attendees: attendees(from: event),
isCanceled: event.status == .canceled)
}
.sorted { $0.start < $1.start }
try writeJson(events)
}
private static func requestCalendarAccess(_ store: EKEventStore) async throws -> Bool {
if #available(macOS 14.0, *) {
return try await store.requestFullAccessToEvents()
}
return try await withCheckedThrowingContinuation { continuation in
store.requestAccess(to: .event) { granted, error in
if let error {
continuation.resume(throwing: error)
} else {
continuation.resume(returning: granted)
}
}
}
}
private static func attendees(from event: EKEvent) -> [String] {
var values: [String] = []
if let organizer = event.organizer,
let value = participantText(organizer) {
values.append(value)
}
for participant in event.attendees ?? [] {
if let value = participantText(participant) {
values.append(value)
}
}
return Array(NSOrderedSet(array: values)) as? [String] ?? values
}
private static func participantText(_ participant: EKParticipant) -> String? {
let name = participant.name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let email = participant.url
.absoluteString
.replacingOccurrences(of: "mailto:", with: "", options: .caseInsensitive)
if name.isEmpty {
return email.isEmpty ? nil : email
}
if email.isEmpty || name.caseInsensitiveCompare(email) == .orderedSame {
return name
}
return "\(name) <\(email)>"
}
@MainActor
private static func showPrompt(requestPath: String) throws {
let data = try Data(contentsOf: URL(fileURLWithPath: requestPath))
let decoder = JSONDecoder()
decoder.dateDecodingStrategy = .iso8601
let request = try decoder.decode(PromptRequest.self, from: data)
let application = NSApplication.shared
application.setActivationPolicy(.accessory)
application.activate(ignoringOtherApps: true)
let alert = NSAlert()
alert.messageText = "Record meeting?"
alert.informativeText = "\(request.subject) starts at \(request.start.formatted(date: .omitted, time: .shortened))."
alert.alertStyle = .informational
alert.addButton(withTitle: "Record")
alert.addButton(withTitle: "Skip")
let response = alert.runModal() == .alertFirstButtonReturn ? "record" : "skip"
try writeJson(PromptOutput(response: response))
}
private static func captureScreenshot(outputPath: String) throws {
guard CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() else {
throw IntegrationError.screenRecordingPermission
}
guard let frontmost = NSWorkspace.shared.frontmostApplication else {
throw IntegrationError.noFrontmostApplication
}
guard let windows = CGWindowListCopyWindowInfo(
[.optionOnScreenOnly, .excludeDesktopElements],
kCGNullWindowID) as? [[String: Any]] else {
throw IntegrationError.noActiveWindow
}
let candidate = windows.first { window in
let ownerPid = window[kCGWindowOwnerPID as String] as? Int32
let layer = window[kCGWindowLayer as String] as? Int
let alpha = window[kCGWindowAlpha as String] as? Double
guard ownerPid == frontmost.processIdentifier,
layer == 0,
(alpha ?? 1) > 0,
let bounds = window[kCGWindowBounds as String] as? [String: Any],
let width = bounds["Width"] as? Double,
let height = bounds["Height"] as? Double else {
return false
}
return width > 1 && height > 1
}
guard let candidate,
let windowNumber = candidate[kCGWindowNumber as String] as? UInt32 else {
throw IntegrationError.noActiveWindow
}
guard let image = CGWindowListCreateImage(
.null,
.optionIncludingWindow,
CGWindowID(windowNumber),
[.boundsIgnoreFraming, .bestResolution]) else {
throw IntegrationError.screenshotCapture
}
try writePng(image, to: outputPath)
try writeJson(ScreenshotOutput(
windowTitle: candidate[kCGWindowName as String] as? String))
}
private static func cropScreenshot(
inputPath: String,
outputPath: String,
x: Int,
y: Int,
width: Int,
height: Int
) throws {
guard x >= 0,
y >= 0,
width > 0,
height > 0,
let source = CGImageSourceCreateWithURL(
URL(fileURLWithPath: inputPath) as CFURL,
nil),
let image = CGImageSourceCreateImageAtIndex(source, 0, nil),
x + width <= image.width,
y + height <= image.height,
let cropped = image.cropping(to: CGRect(
x: x,
y: y,
width: width,
height: height)) else {
throw IntegrationError.screenshotCapture
}
try writePng(cropped, to: outputPath)
try writeJson(["status": "ok"])
}
private static func writePng(_ image: CGImage, to outputPath: String) throws {
let outputUrl = URL(fileURLWithPath: outputPath) as CFURL
guard let destination = CGImageDestinationCreateWithURL(
outputUrl,
UTType.png.identifier as CFString,
1,
nil) else {
throw IntegrationError.screenshotWrite
}
CGImageDestinationAddImage(destination, image, nil)
guard CGImageDestinationFinalize(destination) else {
throw IntegrationError.screenshotWrite
}
}
private static func writeJson<T: Encodable>(_ value: T) throws {
let encoder = JSONEncoder()
encoder.dateEncodingStrategy = .iso8601
encoder.outputFormatting = [.sortedKeys]
FileHandle.standardOutput.write(try encoder.encode(value))
FileHandle.standardOutput.write(Data([0x0A]))
}
}
@@ -1,56 +0,0 @@
namespace MeetingAssistant.Notifications;
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
public interface IAgentApprovalPromptService
{
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
}
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
{
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
}
public sealed class AgentApprovalService
{
private readonly IAgentApprovalPromptService prompts;
private readonly ILogger<AgentApprovalService> logger;
private readonly TimeSpan timeout;
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
{
this.prompts = prompts;
this.logger = logger;
this.timeout = timeout;
}
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
deadline.CancelAfter(timeout);
try
{
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
.WaitAsync(deadline.Token);
cancellationToken.ThrowIfCancellationRequested();
return accepted && !deadline.IsCancellationRequested;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
return false;
}
finally
{
await deadline.CancelAsync();
}
}
}
@@ -1,85 +0,0 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Notifications;
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
: IAgentApprovalPromptService, IDisposable
{
private const string Source = "summary-agent-approval";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly object gate = new();
private bool registered;
private bool disposed;
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (disposed) return false;
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source).AddArgument("requestId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText(prompt.Title)
.AddText(prompt.Message)
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
});
}
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("requestId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
disposed = true;
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
@@ -0,0 +1,52 @@
using MeetingAssistant.Calendar;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Workflow;
namespace MeetingAssistant;
public sealed record PlatformCapabilityReport(
string Host,
string MeetingMetadataProvider,
string CalendarMeetingProvider,
string MeetingStartPromptProvider,
string ScreenshotProvider,
string WorkflowEditorProvider,
bool WorkflowEditorAvailable,
bool CalendarRecordingPromptsEnabled,
bool SpeakerIdentificationEnabled,
bool ScreenshotOcrEnabled,
bool FunAsrBackendEnabled,
bool FunAsrDiarizationEnabled,
bool WhisperLocalDiarizationEnabled)
{
public static PlatformCapabilityReport Create(
HostOperatingSystem host,
MeetingAssistantOptions options,
IMeetingMetadataProvider meetingMetadataProvider,
ICalendarMeetingProvider calendarMeetingProvider,
IMeetingStartPromptService meetingStartPromptProvider,
IActiveWindowScreenshotCapture screenshotProvider,
IWorkflowRulesEditorWindowService workflowEditorProvider)
{
return new PlatformCapabilityReport(
host.Kind switch
{
HostOperatingSystemKind.Windows => "Windows",
HostOperatingSystemKind.MacOS => "macOS",
_ => "Unsupported"
},
meetingMetadataProvider.GetType().Name,
calendarMeetingProvider.GetType().Name,
meetingStartPromptProvider.GetType().Name,
screenshotProvider.GetType().Name,
workflowEditorProvider.GetType().Name,
workflowEditorProvider is not NoopWorkflowRulesEditorWindowService,
options.CalendarRecordingPrompts.Enabled,
options.SpeakerIdentification.Enabled,
options.Screenshots.Ocr.Enabled,
options.FunAsr.Backend.Enabled,
options.FunAsr.Diarization.Enabled,
options.WhisperLocal.Diarization.Enabled);
}
}
+46 -57
View File
@@ -3,8 +3,8 @@ using MeetingAssistant.Calendar;
using MeetingAssistant.Hotkeys;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Logging;
using MeetingAssistant.MacOs;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Recording;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers;
@@ -17,10 +17,8 @@ using Microsoft.Extensions.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
builder.Services.AddOptions<MeetingAssistantOptions>()
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
.ValidateOnStart();
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
builder.Services.AddSingleton(HostOperatingSystem.DetectCurrent());
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
#if WINDOWS
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
@@ -29,9 +27,6 @@ builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
@@ -42,10 +37,9 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
#else
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
PortableMeetingAudioRegistration.Add(builder.Services);
MacOsMeetingIntegrationRegistration.Add(builder.Services, OperatingSystem.IsMacOS());
builder.Services.AddHostedService<MacOsDesktopControlService>();
#endif
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
@@ -75,14 +69,8 @@ builder.Services.AddSingleton<IMeetingStartPromptService>(services =>
services.GetRequiredService<WindowsMeetingStartPromptService>());
builder.Services.AddHostedService(services =>
services.GetRequiredService<WindowsMeetingStartPromptService>());
#else
builder.Services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
builder.Services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
#endif
#if WINDOWS
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
#else
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
builder.Services.AddSingleton<IScreenshotImageCropper, WindowsScreenshotImageCropper>();
#endif
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
builder.Services.AddSingleton<MeetingScreenshotService>();
@@ -100,35 +88,8 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
builder.Services.AddSingleton<SpeakerIdentityService>();
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
builder.Services.AddSingleton(services =>
{
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer;
return resemblyzer.Enabled
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
: SpeakerSampleCollectionPolicy.ExistingBackend;
});
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
: services.GetRequiredService<SpeakerIdentityService>());
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
: services.GetRequiredService<SpeakerIdentityMergeService>());
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
@@ -138,11 +99,7 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
#if WINDOWS
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
#else
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
#endif
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
#if WINDOWS
@@ -150,11 +107,9 @@ builder.Services.AddSingleton(services => WorkflowRulesEditorMarkdownLinkResolve
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value));
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRulesEditorWindowService>();
#else
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
builder.Services.AddSingleton<MacOsWorkflowRulesEditorSession>();
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, MacOsWorkflowRulesEditorWindowService>();
#endif
builder.Services.AddSingleton<AgentApprovalService>();
builder.Services.AddSingleton<WorkflowChangeRequestService>();
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
builder.Services.AddSingleton<AsrDiagnosticService>();
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
@@ -177,7 +132,6 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
#if WINDOWS
builder.Services.AddHostedService<GlobalHotkeyService>();
builder.Services.AddHostedService<UnoTaskbarIconService>();
@@ -190,11 +144,32 @@ app.MapGet("/health", () => Results.Ok(new
service = "meeting-assistant",
status = "ok"
}));
app.MapGet("/diagnostics/platform-capabilities", (
HostOperatingSystem host,
IOptions<MeetingAssistantOptions> options,
IMeetingMetadataProvider meetingMetadataProvider,
ICalendarMeetingProvider calendarMeetingProvider,
IMeetingStartPromptService meetingStartPromptProvider,
IActiveWindowScreenshotCapture screenshotProvider,
IWorkflowRulesEditorWindowService workflowEditorProvider) =>
Results.Ok(PlatformCapabilityReport.Create(
host,
options.Value,
meetingMetadataProvider,
calendarMeetingProvider,
meetingStartPromptProvider,
screenshotProvider,
workflowEditorProvider)));
app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
app.MapGet("/profiles/{launchProfile}/recording/status", (
string launchProfile,
MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
app.MapPost("/application/exit", (IHostApplicationLifetime lifetime) =>
{
lifetime.StopApplication();
return Results.Accepted();
});
app.MapGet("/diagnostics/outlook/current-meeting", async (
IMeetingMetadataProvider metadataProvider,
IOptions<MeetingAssistantOptions> options,
@@ -324,6 +299,15 @@ app.MapPost("/diagnostics/settings-and-logs/show", (
rulesEditorWindow.Show();
return Results.Accepted();
});
#if !WINDOWS
app.MapGet("/diagnostics/settings-and-logs", () =>
Results.Content(MacOsWorkflowRulesEditorPage.Html, "text/html"));
app.MapPost("/diagnostics/settings-and-logs/chat", async (
MacOsWorkflowRulesEditorChatRequest request,
MacOsWorkflowRulesEditorSession session,
CancellationToken cancellationToken) =>
Results.Ok(await session.SendAsync(request.Message, cancellationToken)));
#endif
static WorkflowConfigurationReloadResponse ReloadWorkflowConfiguration(IConfiguration configuration)
{
@@ -366,6 +350,11 @@ app.MapPost("/profiles/{launchProfile}/recording/abort", async (
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
app.MapPost("/profiles/{launchProfile}/meetings/screenshot/capture", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.CaptureScreenshotAsync(launchProfile, cancellationToken)));
app.MapPost("/meetings/current/summary/run", async (
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
@@ -6,8 +6,6 @@ public interface IMeetingInactivityPromptService
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken);
Task DismissAllAsync(CancellationToken cancellationToken);
}
public sealed record MeetingInactivityPromptRequest(
@@ -20,32 +18,9 @@ public enum MeetingInactivityPromptResponse
{
Dismissed,
Continue,
Pause,
Stop
}
internal sealed record MeetingInactivityPromptAction(
string Content,
string ResponseArgument,
MeetingInactivityPromptResponse Response);
internal static class MeetingInactivityPromptActions
{
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
[
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
new("No", "continue", MeetingInactivityPromptResponse.Continue),
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
];
public static MeetingInactivityPromptResponse ParseResponse(string? value)
{
return All.FirstOrDefault(action =>
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
?.Response ?? MeetingInactivityPromptResponse.Continue;
}
}
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
{
public Task ShowStopPromptAsync(
@@ -55,9 +30,4 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
{
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
@@ -2,7 +2,5 @@ namespace MeetingAssistant.Recording;
public interface IMicrophoneCaptureSourceFactory
{
// The caller owns the returned source and disposes it when it implements IDisposable,
// including when capture is canceled before enumeration starts.
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
}
@@ -1,12 +0,0 @@
namespace MeetingAssistant.Recording;
public interface IMicrophoneSwitchPromptService
{
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
}
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
{
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
=> Task.FromResult(false);
}
@@ -33,17 +33,6 @@ public interface ITranscriptStore
string line,
CancellationToken cancellationToken);
async Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
foreach (var line in lines)
{
await AppendLineAsync(session, line, cancellationToken);
}
}
Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -0,0 +1,268 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
namespace MeetingAssistant.Recording;
internal enum MacOsAudioCaptureKind
{
Microphone,
System
}
internal sealed record MacOsAudioCaptureRequest(
MacOsAudioCaptureKind Kind,
int SampleRate,
int Channels);
internal interface IMacOsAudioCaptureProcessFactory
{
IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request);
}
internal interface IMacOsAudioCaptureProcess : IAsyncDisposable
{
Stream StandardOutput { get; }
int ExitCode { get; }
bool HasExited { get; }
Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken);
Task WaitForExitAsync(CancellationToken cancellationToken);
void Terminate();
}
internal abstract class MacOsMeetingAudioSource : IMeetingAudioSource
{
private const int ReadBufferSize = 8 * 1024;
private readonly MacOsAudioCaptureKind kind;
private readonly IMacOsAudioCaptureProcessFactory processFactory;
private readonly ILogger logger;
protected MacOsMeetingAudioSource(
MacOsAudioCaptureKind kind,
IMacOsAudioCaptureProcessFactory processFactory,
ILogger logger)
{
this.kind = kind;
this.processFactory = processFactory;
this.logger = logger;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
}
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
MeetingAssistantOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var request = new MacOsAudioCaptureRequest(
kind,
options.Recording.SampleRate,
options.Recording.Channels);
await using var process = processFactory.Start(request);
var standardErrorTask = process.ReadStandardErrorAsync(CancellationToken.None);
var buffer = new byte[ReadBufferSize];
byte? pendingByte = null;
var hasLoggedFirstChunk = false;
logger.LogInformation(
"Starting macOS {CaptureKind} audio capture at {SampleRate} Hz/{Channels} channel(s)",
kind,
request.SampleRate,
request.Channels);
try
{
while (true)
{
var bytesRead = await process.StandardOutput.ReadAsync(buffer, cancellationToken);
if (bytesRead == 0)
{
break;
}
var pcm = CopyCompleteSamples(buffer, bytesRead, ref pendingByte);
if (pcm.Length == 0)
{
continue;
}
if (!hasLoggedFirstChunk)
{
hasLoggedFirstChunk = true;
logger.LogInformation(
"macOS {CaptureKind} audio produced its first PCM chunk: {ByteCount} bytes",
kind,
pcm.Length);
}
yield return new AudioChunk(pcm, request.SampleRate, request.Channels);
}
await process.WaitForExitAsync(cancellationToken);
var standardError = await standardErrorTask;
if (process.ExitCode != 0)
{
throw CreateCaptureException(kind, process.ExitCode, standardError);
}
}
finally
{
if (!process.HasExited)
{
process.Terminate();
await process.WaitForExitAsync(CancellationToken.None);
}
}
}
private static byte[] CopyCompleteSamples(byte[] buffer, int bytesRead, ref byte? pendingByte)
{
var totalBytes = bytesRead + (pendingByte.HasValue ? 1 : 0);
var completeBytes = totalBytes - totalBytes % sizeof(short);
var pcm = new byte[completeBytes];
var targetOffset = 0;
if (pendingByte is { } firstByte)
{
pcm[0] = firstByte;
targetOffset = 1;
pendingByte = null;
}
var copyCount = completeBytes - targetOffset;
if (copyCount > 0)
{
Buffer.BlockCopy(buffer, 0, pcm, targetOffset, copyCount);
}
var consumedFromBuffer = copyCount;
if (consumedFromBuffer < bytesRead)
{
pendingByte = buffer[consumedFromBuffer];
}
return pcm;
}
private static InvalidOperationException CreateCaptureException(
MacOsAudioCaptureKind kind,
int exitCode,
string standardError)
{
var detail = string.IsNullOrWhiteSpace(standardError)
? "The native capture helper did not provide an error message."
: standardError.Trim();
return new InvalidOperationException(
$"macOS {kind.ToString().ToLowerInvariant()} audio capture exited with code {exitCode}. {detail}");
}
}
internal sealed class MacOsMicrophoneAudioSource : MacOsMeetingAudioSource
{
public MacOsMicrophoneAudioSource(
IMacOsAudioCaptureProcessFactory processFactory,
ILogger<MacOsMicrophoneAudioSource> logger)
: base(MacOsAudioCaptureKind.Microphone, processFactory, logger)
{
}
}
internal sealed class MacOsSystemAudioSource : MacOsMeetingAudioSource
{
public MacOsSystemAudioSource(
IMacOsAudioCaptureProcessFactory processFactory,
ILogger<MacOsSystemAudioSource> logger)
: base(MacOsAudioCaptureKind.System, processFactory, logger)
{
}
}
internal sealed class MacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
{
private readonly ILogger<MacOsAudioCaptureProcessFactory> logger;
public MacOsAudioCaptureProcessFactory(ILogger<MacOsAudioCaptureProcessFactory> logger)
{
this.logger = logger;
}
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
{
if (!OperatingSystem.IsMacOS())
{
throw new PlatformNotSupportedException("The macOS audio capture helper can only run on macOS.");
}
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-meeting-audio-capture");
if (!File.Exists(helperPath))
{
throw new FileNotFoundException(
$"The macOS audio capture helper was not found at '{helperPath}'. Build or publish the application on macOS so the Swift helper is compiled into the Native folder.",
helperPath);
}
var startInfo = new ProcessStartInfo
{
FileName = helperPath,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
startInfo.ArgumentList.Add(request.Kind == MacOsAudioCaptureKind.Microphone ? "microphone" : "system");
startInfo.ArgumentList.Add(request.SampleRate.ToString(System.Globalization.CultureInfo.InvariantCulture));
startInfo.ArgumentList.Add(request.Channels.ToString(System.Globalization.CultureInfo.InvariantCulture));
logger.LogDebug("Starting native macOS audio helper {HelperPath} for {CaptureKind}", helperPath, request.Kind);
var process = Process.Start(startInfo)
?? throw new InvalidOperationException($"Failed to start the macOS audio capture helper '{helperPath}'.");
return new MacOsAudioCaptureProcess(process);
}
}
internal sealed class MacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
{
private readonly Process process;
public MacOsAudioCaptureProcess(Process process)
{
this.process = process;
}
public Stream StandardOutput => process.StandardOutput.BaseStream;
public int ExitCode => process.ExitCode;
public bool HasExited => process.HasExited;
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
{
return process.StandardError.ReadToEndAsync(cancellationToken);
}
public Task WaitForExitAsync(CancellationToken cancellationToken)
{
return process.WaitForExitAsync(cancellationToken);
}
public void Terminate()
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
public ValueTask DisposeAsync()
{
process.Dispose();
return ValueTask.CompletedTask;
}
}
@@ -34,7 +34,6 @@ public sealed class MeetingRecordingCoordinator
private readonly IMeetingInactivityClock inactivityClock;
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
private readonly MeetingAssistantOptions options;
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private RecordingRun? currentRun;
@@ -64,8 +63,7 @@ public sealed class MeetingRecordingCoordinator
IMeetingRunArtifactCleaner? artifactCleaner = null,
IMeetingInactivityPromptService? inactivityPromptService = null,
IMeetingInactivityClock? inactivityClock = null,
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -88,8 +86,6 @@ public sealed class MeetingRecordingCoordinator
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
this.options = options.Value;
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
?? SpeakerSampleCollectionPolicy.ExistingBackend;
this.logger = logger;
}
@@ -100,8 +96,7 @@ public sealed class MeetingRecordingCoordinator
currentArtifacts?.AssistantContextPath,
currentArtifacts?.SummaryPath,
GetProcessState(currentRun),
currentRun?.LaunchProfileName,
IsRecording && currentRun?.IsTranscriptionPaused == true);
currentRun?.LaunchProfileName);
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
@@ -294,12 +289,10 @@ public sealed class MeetingRecordingCoordinator
startedAt,
launchProfile.Name,
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
logger);
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
if (runOptions.Recording.InactivitySafeguard.Enabled)
{
_ = Task.Run(
() => RunInactivitySafeguardAsync(run),
@@ -341,14 +334,11 @@ public sealed class MeetingRecordingCoordinator
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return await StopAsync(null, null, null, null, cancellationToken);
return await StopAsync(null, cancellationToken);
}
private async Task<RecordingStatus> StopAsync(
DateTimeOffset? inferredEndTime,
RecordingRun? expectedRun,
long? expectedActivityVersion,
DateTimeOffset? expectedPauseStartedAt,
CancellationToken cancellationToken)
{
RecordingRun run;
@@ -357,30 +347,13 @@ public sealed class MeetingRecordingCoordinator
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is null ||
currentRun.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
if (currentRun is null || currentRun.IsCaptureStopping)
{
return CurrentStatus;
}
run = currentRun;
var inactivityStopClaimed = expectedActivityVersion is not null;
if (expectedActivityVersion is not null &&
(inferredEndTime is null ||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
{
return CurrentStatus;
}
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
if (expectedPauseStartedAt is not null &&
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
{
return CurrentStatus;
}
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
if (inferredEndTime is not null)
{
run.SetInferredEndTime(inferredEndTime.Value);
}
@@ -390,7 +363,7 @@ public sealed class MeetingRecordingCoordinator
{
run.Abort();
}
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
else
{
run.StopCapture();
}
@@ -416,12 +389,6 @@ public sealed class MeetingRecordingCoordinator
}
catch (TimeoutException)
{
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
{
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
return CurrentStatus;
}
if (IsAzureSpeechRun(run))
{
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
@@ -442,65 +409,6 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
CancellationToken cancellationToken)
{
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
}
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
RecordingRun? expectedRun,
long? expectedActivityVersion,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
var run = currentRun;
if (run is null ||
run.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
{
return CurrentStatus;
}
var changed = false;
await run.PipelineGate.WaitAsync(cancellationToken);
try
{
var changedAt = inactivityClock.Now;
changed = run.TrySetTranscriptionPaused(
isPaused,
changedAt,
expectedActivityVersion);
}
finally
{
run.PipelineGate.Release();
}
if (changed && isPaused)
{
await DismissInactivityPromptsAsync(cancellationToken);
}
if (changed)
{
logger.LogInformation(
"Meeting transcription {PauseState}",
isPaused ? "paused" : "unpaused");
}
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
{
RecordingRun run;
@@ -753,7 +661,8 @@ public sealed class MeetingRecordingCoordinator
chunk.Channels);
}
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
}
logger.LogInformation(
@@ -792,6 +701,7 @@ public sealed class MeetingRecordingCoordinator
run.ResetSpeakerIdentification();
}
run.RecordTranscriptActivity(segment, inactivityClock.Now);
var sample = run.TryAddSpeakerSample(segment);
if (sample is not null)
{
@@ -808,56 +718,6 @@ public sealed class MeetingRecordingCoordinator
var relabeledSegment = run.Relabel(segment);
run.AddLiveSegment(relabeledSegment);
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
run.RecordTranscriptActivity(segment, inactivityClock.Now);
if (!string.IsNullOrWhiteSpace(segment.Text))
{
// Switching profiles holds the coordinator gate while draining this reader.
// Notification cleanup must not make transcript draining wait for that gate.
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
}
}
}
private async Task DismissInactivityPromptsForRunAsync(
RecordingRun run,
CancellationToken cancellationToken)
{
try
{
await gate.WaitAsync(cancellationToken);
try
{
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
{
return;
}
await DismissInactivityPromptsAsync(cancellationToken);
}
finally
{
gate.Release();
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Pending cleanup is no longer needed after this run stops capturing.
}
}
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
{
try
{
await inactivityPromptService.DismissAllAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
}
}
@@ -985,10 +845,7 @@ public sealed class MeetingRecordingCoordinator
private async Task RunInactivitySafeguardAsync(RecordingRun run)
{
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
var promptThresholds = safeguardOptions.Enabled
? GetInactivityPromptThresholds(safeguardOptions)
: [];
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
var promptedThresholds = new HashSet<TimeSpan>();
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
@@ -996,7 +853,7 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
string.Join(", ", promptThresholds),
safeguardOptions.AutoStopAfter,
checkInterval);
safeguardOptions.CheckInterval);
try
{
@@ -1012,38 +869,9 @@ public sealed class MeetingRecordingCoordinator
snapshot.LastTranscriptActivityAt);
}
var pauseStartedAt = snapshot.TranscriptionPausedAt;
if (pauseStartedAt is not null)
{
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
if (maximumPauseDuration > TimeSpan.Zero &&
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
{
logger.LogWarning(
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
maximumPauseDuration);
var stopStatus = await StopAsync(
null,
run,
null,
pauseStartedAt,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
}
await inactivityClock.DelayAsync(
checkInterval,
run.CaptureCancellation);
continue;
}
var now = inactivityClock.Now;
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
if (safeguardOptions.Enabled &&
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
inactivityDuration >= safeguardOptions.AutoStopAfter)
{
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
@@ -1051,16 +879,8 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
inactivityDuration,
inferredEndTime);
var stopStatus = await StopAsync(
inferredEndTime,
run,
snapshot.ActivityVersion,
null,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
await StopAsync(inferredEndTime, CancellationToken.None);
return;
}
foreach (var threshold in promptThresholds)
@@ -1077,7 +897,6 @@ public sealed class MeetingRecordingCoordinator
inactivityDuration,
threshold,
inferredEndTime);
var promptActivityVersion = snapshot.ActivityVersion;
await inactivityPromptService.ShowStopPromptAsync(
new MeetingInactivityPromptRequest(
inactivityDuration,
@@ -1093,36 +912,16 @@ public sealed class MeetingRecordingCoordinator
if (response == MeetingInactivityPromptResponse.Stop)
{
await StopAsync(
inferredEndTime,
run,
promptActivityVersion,
null,
CancellationToken.None);
}
else if (response == MeetingInactivityPromptResponse.Pause)
{
await SetTranscriptionPausedAsync(
true,
run,
promptActivityVersion,
CancellationToken.None);
await StopAsync(inferredEndTime, CancellationToken.None);
}
},
run.CaptureCancellation);
var currentActivity = run.GetTranscriptActivitySnapshot();
if (currentActivity.IsTranscriptionPaused ||
currentActivity.ActivityVersion != promptActivityVersion)
{
await DismissInactivityPromptsAsync(run.CaptureCancellation);
}
break;
}
await inactivityClock.DelayAsync(
checkInterval,
GetInactivitySafeguardCheckInterval(safeguardOptions),
run.CaptureCancellation);
}
}
@@ -1190,11 +989,11 @@ public sealed class MeetingRecordingCoordinator
try
{
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
if (checkpoint is null)
{
logger.LogInformation(
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
return;
}
@@ -2022,11 +1821,6 @@ public sealed class MeetingRecordingCoordinator
.ToList();
}
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
{
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
}
private static TimeSpan GetInactivitySafeguardCheckInterval(
RecordingInactivitySafeguardOptions options)
{
@@ -2083,8 +1877,6 @@ public sealed class MeetingRecordingCoordinator
private TimeSpan? lastTranscriptSegmentEnd;
private long transcriptActivityVersion;
private DateTimeOffset? inferredEndTime;
private bool isTranscriptionPaused;
private DateTimeOffset? transcriptionPausedAt;
public RecordingRun(
CancellationTokenSource captureCancellation,
@@ -2100,7 +1892,6 @@ public sealed class MeetingRecordingCoordinator
string launchProfileName,
TimeSpan liveSampleBufferDuration,
int maxSpeakerSamples,
bool requireNonOverlappingSpeakerSamples,
ILogger logger)
{
CaptureCancellationSource = captureCancellation;
@@ -2121,9 +1912,8 @@ public sealed class MeetingRecordingCoordinator
liveSampleBufferDuration,
maxSpeakerSamples,
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleDuration,
logger,
requireNonOverlappingSpeakerSamples);
options.SpeakerIdentification.MaximumSampleSegmentGap,
logger);
}
public CancellationTokenSource CaptureCancellationSource { get; }
@@ -2170,17 +1960,6 @@ public sealed class MeetingRecordingCoordinator
public bool HasAttachedPromptMetadata { get; private set; }
public bool IsTranscriptionPaused
{
get
{
lock (transcriptActivityGate)
{
return isTranscriptionPaused;
}
}
}
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public DateTimeOffset? InferredEndTime
@@ -2204,37 +1983,6 @@ public sealed class MeetingRecordingCoordinator
HasAttachedPromptMetadata = true;
}
public bool TrySetTranscriptionPaused(
bool isPaused,
DateTimeOffset changedAt,
long? expectedActivityVersion)
{
lock (transcriptActivityGate)
{
if (expectedActivityVersion is not null &&
transcriptActivityVersion != expectedActivityVersion.Value)
{
return false;
}
if (isTranscriptionPaused == isPaused)
{
return false;
}
isTranscriptionPaused = isPaused;
transcriptionPausedAt = isPaused ? changedAt : null;
if (!isPaused)
{
lastTranscriptActivityAt = changedAt;
}
transcriptActivityVersion++;
return true;
}
}
public void Abort()
{
IsAborted = true;
@@ -2263,17 +2011,13 @@ public sealed class MeetingRecordingCoordinator
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
}
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
await PipelineGate.WaitAsync(cancellationToken);
try
{
var routedChunk = IsTranscriptionPaused
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
: chunk;
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
AppendAudio(routedChunk);
await Pipeline.WriteAsync(routedChunk, cancellationToken);
AppendAudio(chunk);
await Pipeline.WriteAsync(chunk, cancellationToken);
}
finally
{
@@ -2457,9 +2201,7 @@ public sealed class MeetingRecordingCoordinator
return new TranscriptActivitySnapshot(
lastTranscriptActivityAt,
lastTranscriptSegmentEnd,
transcriptActivityVersion,
isTranscriptionPaused,
transcriptionPausedAt);
transcriptActivityVersion);
}
}
@@ -2479,35 +2221,6 @@ public sealed class MeetingRecordingCoordinator
}
}
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
{
lock (transcriptActivityGate)
{
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
{
return false;
}
inferredEndTime = endTime;
StopCapture();
return true;
}
}
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
{
lock (transcriptActivityGate)
{
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
{
return false;
}
StopCapture();
return true;
}
}
public void AppendAudio(AudioChunk chunk)
{
speakerSampleCollector.AppendAudio(chunk);
@@ -2531,11 +2244,9 @@ public sealed class MeetingRecordingCoordinator
return speakerSampleCollector.Snapshot();
}
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
MeetingNote meetingNote,
IReadOnlyList<SpeakerAudioSample> samples)
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
{
var speakers = GetUnmappedSampleSpeakers(samples);
var speakers = GetUnmappedSampleSpeakers();
if (speakers.Count == 0)
{
return null;
@@ -2543,11 +2254,7 @@ public sealed class MeetingRecordingCoordinator
var checkpoint = new LiveIdentificationCheckpoint(
speakers,
BuildAttendeeSignature(meetingNote),
Options.SpeakerIdentification.Resemblyzer.Enabled
? speakers.Select(speaker => samples.Count(sample =>
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
: []);
BuildAttendeeSignature(meetingNote));
lock (liveIdentificationGate)
{
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
@@ -2605,9 +2312,9 @@ public sealed class MeetingRecordingCoordinator
StringComparer.OrdinalIgnoreCase);
}
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
{
return samples
return GetSpeakerSamplesSnapshot()
.Select(sample => sample.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
@@ -2643,9 +2350,7 @@ public sealed class MeetingRecordingCoordinator
public sealed record TranscriptActivitySnapshot(
DateTimeOffset LastTranscriptActivityAt,
TimeSpan? LastTranscriptSegmentEnd,
long ActivityVersion,
bool IsTranscriptionPaused,
DateTimeOffset? TranscriptionPausedAt);
long ActivityVersion);
private static int StateRank(AssistantContextState state)
{
@@ -2663,13 +2368,11 @@ public sealed class MeetingRecordingCoordinator
public sealed record LiveIdentificationCheckpoint(
IReadOnlyList<string> Speakers,
string AttendeeSignature,
IReadOnlyList<int> SampleCounts)
string AttendeeSignature)
{
public bool Matches(LiveIdentificationCheckpoint other)
{
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
SampleCounts.SequenceEqual(other.SampleCounts) &&
Speakers.Count == other.Speakers.Count &&
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
}
@@ -2702,8 +2405,7 @@ public sealed record RecordingStatus(
string? AssistantContextPath,
string? SummaryPath,
RecordingProcessState State = RecordingProcessState.Idle,
string? LaunchProfile = null,
bool IsPaused = false);
string? LaunchProfile = null);
public enum RecordingProcessState
{
@@ -8,30 +8,22 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
private readonly IMicrophoneCaptureSourceFactory captureSources;
private readonly ILogger<MicrophoneAudioSource> logger;
private readonly TimeSpan recoveryDelay;
private readonly MicrophoneSwitchMonitor? switchMonitor;
private readonly MicrophoneDeviceSelection? selection;
public MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
ILogger<MicrophoneAudioSource> logger)
: this(captureSources, logger, DefaultRecoveryDelay)
{
}
internal MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
TimeSpan recoveryDelay,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
TimeSpan recoveryDelay)
{
this.captureSources = captureSources;
this.logger = logger;
this.recoveryDelay = recoveryDelay;
this.switchMonitor = switchMonitor;
this.selection = selection;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
@@ -42,44 +34,21 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
MeetingAssistantOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
using var session = new CaptureSession(selection, cancellationToken);
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
try
{
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
{
yield return chunk;
}
}
finally
{
session.Stop();
await monitoring;
}
}
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
MeetingAssistantOptions options,
CaptureSession session,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var failedAttempts = 0;
while (!cancellationToken.IsCancellationRequested)
{
var captureToken = session.BeginCapture();
IMeetingAudioSource? captureSource = null;
IAsyncEnumerator<AudioChunk>? capture = null;
Exception? failure = null;
try
{
captureSource = captureSources.CreateCapture(options);
capture = captureSource
.CaptureAsync(options, captureToken)
.GetAsyncEnumerator(captureToken);
capture = captureSources
.CreateCapture(options)
.CaptureAsync(options, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -87,7 +56,6 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
failure = exception;
}
using var captureOwner = captureSource as IDisposable;
if (cancellationToken.IsCancellationRequested)
{
yield break;
@@ -97,14 +65,14 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
try
{
while (!captureToken.IsCancellationRequested)
while (!cancellationToken.IsCancellationRequested)
{
var hasNext = false;
try
{
hasNext = await capture.MoveNextAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -112,7 +80,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
failure = exception;
}
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
if (cancellationToken.IsCancellationRequested || failure is not null || !hasNext)
{
break;
}
@@ -134,7 +102,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
await capture.DisposeAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
@@ -144,19 +112,11 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
}
}
var switchRequested = captureToken.IsCancellationRequested;
session.EndCapture();
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
if (switchRequested)
{
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
continue;
}
failedAttempts++;
logger.LogWarning(
failure,
@@ -182,53 +142,4 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
return false;
}
}
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
{
private readonly object gate = new();
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
private CancellationTokenSource? capture;
public CancellationToken Token => lifetime.Token;
public CancellationToken BeginCapture()
{
lock (gate)
{
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
return capture.Token;
}
}
public void EndCapture()
{
lock (gate)
{
capture?.Dispose();
capture = null;
}
}
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
{
lock (gate)
{
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
selection.Select(device.Id);
capture?.Cancel();
}
}
public void Stop()
{
lock (gate) lifetime.Cancel();
}
public void Dispose()
{
Stop();
EndCapture();
lifetime.Dispose();
}
}
}
@@ -1,114 +0,0 @@
namespace MeetingAssistant.Recording;
public sealed class MicrophoneSwitchMonitor
{
private readonly IMicrophoneDeviceProvider devices;
private readonly IMicrophoneSwitchPromptService prompts;
private readonly ILogger<MicrophoneSwitchMonitor> logger;
private readonly TimeSpan pollInterval;
private readonly TimeSpan promptLifetime;
public MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger)
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
{
}
internal MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger,
TimeSpan pollInterval,
TimeSpan promptLifetime)
{
this.devices = devices;
this.prompts = prompts;
this.logger = logger;
this.pollInterval = pollInterval;
this.promptLifetime = promptLifetime;
}
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
{
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
try
{
while (true)
{
await Task.Delay(pollInterval, cancellationToken);
var available = ReadDevices();
if (available is null) continue;
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var (id, prompt) in pending.ToArray())
{
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
if (prompt.Task.IsCompleted)
{
prompt.Cancellation.Dispose();
pending.Remove(id);
}
}
foreach (var device in available)
{
if (known.Add(device.Id))
{
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
promptCancellation.CancelAfter(promptLifetime);
pending.Add(device.Id, new PendingPrompt(promptCancellation,
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
}
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
finally
{
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
}
}
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
CancellationToken cancellationToken)
{
try
{
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
&& !cancellationToken.IsCancellationRequested
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
&& !cancellationToken.IsCancellationRequested)
{
switchMicrophone(device, cancellationToken);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
}
}
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
{
try
{
return devices.GetAvailableMicrophones();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not check for newly available microphones");
return null;
}
}
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
}
@@ -4,12 +4,11 @@ using NAudio.Wave;
namespace MeetingAssistant.Recording;
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource
{
private readonly IWaveIn capture;
private readonly string sourceName;
private readonly ILogger logger;
private int disposed;
public NaudioCaptureAudioSource(
IWaveIn capture,
@@ -23,15 +22,13 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposabl
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureWith(cancellationToken);
return CaptureWith(capture, sourceName, logger, cancellationToken);
}
public void Dispose()
{
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
}
private async IAsyncEnumerable<AudioChunk> CaptureWith(
private static async IAsyncEnumerable<AudioChunk> CaptureWith(
IWaveIn capture,
string sourceName,
ILogger logger,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded<AudioChunk>();
@@ -86,7 +83,7 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposabl
}
finally
{
Dispose();
capture.Dispose();
}
}
}
@@ -117,25 +117,8 @@ public sealed class OfflineTranscriptionBacklogProcessor
pipelineOptions,
cancellationToken);
if (!finishedSegments.Any(segment =>
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
{
throw new InvalidOperationException(
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
}
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
await transcriptStore.AppendLinesAsync(session,
[
"",
"## Offline transcription recovery",
"",
"> The following transcript was recovered by replaying the entire recording. "
+ "It may duplicate earlier transcript passages. "
+ "Treat repeated passages as the same discussion when summarizing.",
"",
.. lines
], cancellationToken);
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
@@ -0,0 +1,30 @@
using Microsoft.Extensions.DependencyInjection;
namespace MeetingAssistant.Recording;
internal static class PortableMeetingAudioRegistration
{
public static void Add(IServiceCollection services)
{
Add(services, OperatingSystem.IsMacOS());
}
internal static void Add(IServiceCollection services, bool isMacOs)
{
if (!isMacOs)
{
services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
return;
}
services.AddSingleton<IMacOsAudioCaptureProcessFactory, MacOsAudioCaptureProcessFactory>();
services.AddSingleton<MacOsMicrophoneAudioSource>();
services.AddSingleton<MacOsSystemAudioSource>();
services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
services.AddSingleton<IMeetingAudioSource>(serviceProvider => new CompositeMeetingAudioSource(
serviceProvider.GetRequiredService<MacOsMicrophoneAudioSource>(),
serviceProvider.GetRequiredService<MacOsSystemAudioSource>(),
serviceProvider.GetRequiredService<IAcousticEchoCancellerFactory>(),
serviceProvider.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
}
}
@@ -9,31 +9,37 @@ internal sealed class SpeakerAudioSampleCollector
private readonly object gate = new();
private readonly RollingAudioBuffer audioBuffer;
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly int maxSamplesPerSpeaker;
private readonly TimeSpan minimumSampleSpeechDuration;
private readonly TimeSpan maximumSampleDuration;
private readonly bool requireNonOverlappingSamples;
private readonly TimeSpan minimumUninterruptedSpeechDuration;
private readonly TimeSpan maximumSegmentGap;
private readonly ILogger? logger;
private SpeakerSampleSpan? pendingSpan;
private PendingSpeakerSpan? pendingSpan;
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
: this(
bufferDuration,
maxSamplesPerSpeaker,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(1),
logger: null)
{
}
public SpeakerAudioSampleCollector(
TimeSpan bufferDuration,
int maxSamplesPerSpeaker,
TimeSpan minimumSampleSpeechDuration,
TimeSpan maximumSampleDuration,
ILogger? logger = null,
bool requireNonOverlappingSamples = false)
TimeSpan minimumUninterruptedSpeechDuration,
TimeSpan maximumSegmentGap,
ILogger? logger = null)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
minimumSampleSpeechDuration,
maximumSampleDuration,
"Speaker sample collector configuration is invalid");
audioBuffer = new RollingAudioBuffer(bufferDuration);
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
this.maximumSampleDuration = maximumSampleDuration;
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
? minimumUninterruptedSpeechDuration
: TimeSpan.Zero;
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
? maximumSegmentGap
: TimeSpan.Zero;
this.logger = logger;
}
@@ -47,7 +53,6 @@ internal sealed class SpeakerAudioSampleCollector
lock (gate)
{
samplesBySpeaker.Clear();
lastAcceptedEndBySpeaker.Clear();
pendingSpan = null;
audioBuffer.Reset();
}
@@ -63,54 +68,34 @@ internal sealed class SpeakerAudioSampleCollector
return null;
}
SpeakerSampleSpan? sampleSpan;
SpeakerSampleSpan? previousSpan;
TranscriptionSegment sampleSegment;
PendingSpanReset? reset;
lock (gate)
{
if (requireNonOverlappingSamples &&
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
segment.Start < lastAcceptedEnd)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
segment.Speaker,
lastAcceptedEnd);
return null;
}
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
(sampleSegment, reset) = ExtendPendingSpan(segment);
}
if (sampleSpan is null)
if (reset is not null)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
segment.Speaker);
return null;
}
var sampleSegment = sampleSpan.ToSegment();
if (previousSpan is not null)
{
logger?.LogInformation(
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
previousSpan.Speaker,
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
reset.PreviousSpeaker,
segment.Speaker,
previousSpan.End,
segment.Start);
reset.PreviousEnd,
segment.Start,
reset.Gap,
maximumSegmentGap);
}
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
if (!score.Accepted)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
sampleSegment.Speaker,
score.Reason,
sampleSpan.SpeechDuration,
sampleSegment.End - sampleSegment.Start,
minimumSampleSpeechDuration,
minimumUninterruptedSpeechDuration,
score.WordCount);
return null;
}
@@ -130,13 +115,6 @@ internal sealed class SpeakerAudioSampleCollector
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
lock (gate)
{
if (requireNonOverlappingSamples &&
ReferenceEquals(pendingSpan, sampleSpan))
{
pendingSpan = null;
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
}
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
{
samples = [];
@@ -185,26 +163,35 @@ internal sealed class SpeakerAudioSampleCollector
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
}
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
{
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
if (pendingSpan is null ||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
{
pendingSpan = extended;
return (pendingSpan, null);
var reset = pendingSpan is null
? null
: new PendingSpanReset(
pendingSpan.Speaker,
pendingSpan.End,
segment.Start - pendingSpan.End);
pendingSpan = new PendingSpeakerSpan(
segment.Speaker,
segment.Start,
segment.End,
[segment.Text]);
return (pendingSpan.ToSegment(), reset);
}
var previousSpan = pendingSpan;
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
return (pendingSpan, previousSpan);
pendingSpan = pendingSpan.Extend(segment);
return (pendingSpan.ToSegment(), null);
}
private static SampleScore Score(
TranscriptionSegment segment,
TimeSpan speechDuration,
TimeSpan minimumSampleSpeechDuration)
TimeSpan minimumUninterruptedSpeechDuration)
{
var durationSeconds = speechDuration.TotalSeconds;
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
{
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
}
@@ -215,7 +202,7 @@ internal sealed class SpeakerAudioSampleCollector
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
}
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
var wordScore = Math.Min(words / 60.0, 1);
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
segment.Text.TrimEnd().EndsWith('?') ||
@@ -234,4 +221,30 @@ internal sealed class SpeakerAudioSampleCollector
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
private sealed record PendingSpeakerSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
IReadOnlyList<string> TextParts)
{
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
{
return this with
{
End = segment.End > End ? segment.End : End,
TextParts = TextParts.Append(segment.Text).ToList()
};
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
}
}
}
@@ -5,6 +5,6 @@ public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
throw new PlatformNotSupportedException(
"Meeting audio capture is only implemented for the Windows target in this version.");
"Meeting audio capture is supported on Windows and macOS, but is unavailable on this host.");
}
}
@@ -49,25 +49,6 @@ public sealed class VaultTranscriptStore : ITranscriptStore
cancellationToken);
}
public Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
if (lines.Count == 0)
{
return Task.CompletedTask;
}
return WithFileGateAsync(
session.TranscriptPath,
() => File.AppendAllTextAsync(
session.TranscriptPath,
string.Concat(lines.Select(line => line + Environment.NewLine)),
cancellationToken),
cancellationToken);
}
public async Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
private const string NotificationGroup = "meeting-inactivity";
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
private readonly object notificationGate = new();
private readonly object registrationGate = new();
private bool registered;
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
@@ -37,27 +37,24 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
try
{
lock (notificationGate)
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
catch (Exception exception)
{
@@ -79,36 +76,6 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
lock (notificationGate)
{
if (pendingPrompts.IsEmpty)
{
return Task.CompletedTask;
}
pendingPrompts.Clear();
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
{
return Task.CompletedTask;
}
try
{
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
logger.LogInformation("Dismissed all native Windows inactivity notifications");
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
}
return Task.CompletedTask;
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
@@ -116,28 +83,29 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
public void Dispose()
{
lock (notificationGate)
if (!registered)
{
if (!registered)
{
return;
}
return;
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
registered = false;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
}
}
private void EnsureRegistered()
{
lock (notificationGate)
if (registered)
{
return;
}
lock (registrationGate)
{
if (registered)
{
@@ -154,25 +122,29 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
string promptId,
MeetingInactivityPromptRequest request)
{
var builder = new ToastContentBuilder()
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "stop")
.SetBackgroundActivation();
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "continue")
.SetBackgroundActivation();
return new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("Stop meeting?")
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
foreach (var action in MeetingInactivityPromptActions.All)
{
builder.AddButton(new ToastButton()
.SetContent(action.Content)
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", action.ResponseArgument)
.SetBackgroundActivation());
}
return builder;
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
.AddButton(yesButton)
.AddButton(noButton);
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
@@ -205,9 +177,10 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue)
? MeetingInactivityPromptActions.ParseResponse(responseValue)
: MeetingInactivityPromptResponse.Continue;
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
? MeetingInactivityPromptResponse.Stop
: MeetingInactivityPromptResponse.Continue;
_ = Task.Run(async () =>
{
try
@@ -1,96 +0,0 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
namespace MeetingAssistant.Recording;
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
{
private const string Source = "meeting-microphone-switch";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
private readonly object gate = new();
private bool registered;
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
{
this.logger = logger;
}
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source)
.AddArgument("promptId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("New microphone available")
.AddText($"Switch this meeting to {microphone.Name}?")
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
});
}
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try
{
ToastNotificationManagerCompat.History.Remove(id, Source);
}
catch (Exception exception)
{
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
}
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("promptId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
@@ -1,7 +1,4 @@
using System.Collections.Concurrent;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
@@ -46,6 +43,25 @@ public sealed record ScreenshotOcrResult
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
public interface IScreenshotImageCropper
{
Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken);
}
internal sealed class UnavailableScreenshotImageCropper : IScreenshotImageCropper
{
public Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
return Task.FromResult<string?>(null);
}
}
public interface IMeetingScreenshotService
{
Task<MeetingScreenshotCaptureResult> CaptureAsync(
@@ -147,6 +163,7 @@ public sealed partial class MeetingScreenshotService :
IMeetingNoteImageOcrService
{
private readonly IActiveWindowScreenshotCapture screenshotCapture;
private readonly IScreenshotImageCropper screenshotImageCropper;
private readonly IMeetingArtifactStore artifactStore;
private readonly IMeetingNoteStore meetingNoteStore;
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
@@ -162,6 +179,7 @@ public sealed partial class MeetingScreenshotService :
IMeetingNoteStore meetingNoteStore,
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
IScreenshotOcrClient ocrClient,
IScreenshotImageCropper screenshotImageCropper,
ILogger<MeetingScreenshotService> logger,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{
@@ -171,6 +189,7 @@ public sealed partial class MeetingScreenshotService :
this.attendeeCanonicalizer = attendeeCanonicalizer;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.ocrClient = ocrClient;
this.screenshotImageCropper = screenshotImageCropper;
this.logger = logger;
}
@@ -639,17 +658,20 @@ public sealed partial class MeetingScreenshotService :
return "";
}
if (!OperatingSystem.IsWindows())
{
logger.LogWarning(
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
screenshotPath);
return "";
}
try
{
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
var croppedPath = await screenshotImageCropper.SaveCroppedScreenshotAsync(
screenshotPath,
crop,
cancellationToken);
if (croppedPath is null)
{
logger.LogWarning(
"Ignoring screenshot crop coordinates for {ScreenshotPath} because no image cropper is available on this host",
screenshotPath);
return "";
}
var relativePath = ToMarkdownPath(Path.GetRelativePath(
Path.GetDirectoryName(assistantContextPath)!,
croppedPath));
@@ -657,7 +679,7 @@ public sealed partial class MeetingScreenshotService :
Environment.NewLine +
Environment.NewLine;
}
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
catch (Exception exception) when (exception is InvalidDataException or ArgumentException)
{
logger.LogWarning(
exception,
@@ -667,39 +689,6 @@ public sealed partial class MeetingScreenshotService :
}
}
#pragma warning disable CA1416
private static Task<string> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
using var original = new Bitmap(screenshotPath);
ValidateCrop(crop, original.Width, original.Height);
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
var croppedPath = Path.Combine(
Path.GetDirectoryName(screenshotPath)!,
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
cropped.Save(croppedPath, ImageFormat.Png);
return Task.FromResult(croppedPath);
}
#pragma warning restore CA1416
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
{
if (crop.X < 0 ||
crop.Y < 0 ||
crop.Width <= 0 ||
crop.Height <= 0 ||
crop.X + crop.Width > imageWidth ||
crop.Y + crop.Height > imageHeight)
{
throw new InvalidDataException(
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
}
}
private void TrackOcr(
MeetingSessionArtifacts artifacts,
CancellationTokenSource cancellation,
@@ -0,0 +1,55 @@
#if WINDOWS
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
namespace MeetingAssistant.Screenshots;
internal sealed class WindowsScreenshotImageCropper : IScreenshotImageCropper
{
#pragma warning disable CA1416
public Task<string?> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using var original = new Bitmap(screenshotPath);
ValidateCrop(crop, original.Width, original.Height);
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
var croppedPath = Path.Combine(
Path.GetDirectoryName(screenshotPath)!,
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
cropped.Save(croppedPath, ImageFormat.Png);
return Task.FromResult<string?>(croppedPath);
}
catch (ExternalException exception)
{
throw new InvalidDataException(
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
exception);
}
}
#pragma warning restore CA1416
private static void ValidateCrop(
ScreenshotCropCoordinates crop,
int imageWidth,
int imageHeight)
{
if (crop.X < 0 ||
crop.Y < 0 ||
crop.Width <= 0 ||
crop.Height <= 0 ||
crop.X + crop.Width > imageWidth ||
crop.Y + crop.Height > imageHeight)
{
throw new InvalidDataException(
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
}
}
}
#endif
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
return [];
}
return await finalizer.FinalizeEnabledAsync(
return await finalizer.FinalizeAsync(
wavPath,
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
options.Diarization,
@@ -1,631 +0,0 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerSnippetExtractor snippetExtractor;
private readonly IResemblyzerVoiceEncoder encoder;
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
public ResemblyzerSpeakerIdentificationService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerSnippetExtractor snippetExtractor,
IResemblyzerVoiceEncoder encoder,
ResemblyzerVoiceClusterMatcher clusterMatcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentificationService> logger)
{
this.dbContextFactory = dbContextFactory;
this.snippetExtractor = snippetExtractor;
this.encoder = encoder;
this.clusterMatcher = clusterMatcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
}
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
}
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
}
public async Task ApplySpeakerOverrideAsync(
SpeakerIdentificationRequest request,
string sourceSpeaker,
string targetSpeaker,
CancellationToken cancellationToken)
{
if (!options.Enabled ||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
string.IsNullOrWhiteSpace(targetSpeaker) ||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
{
return;
}
var sourceLabel = sourceSpeaker.Trim();
var targetName = targetSpeaker.Trim();
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var sourceCandidate = identities
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => identity.Id)
.FirstOrDefault();
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
if (target is null && sourceCandidate is null && vectors.Count == 0)
{
logger.LogWarning(
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
sourceLabel,
targetName);
return;
}
var now = DateTimeOffset.UtcNow;
if (target is null)
{
target = sourceCandidate ?? new SpeakerIdentity
{
CreatedAt = now
};
if (target.Id == 0)
{
context.SpeakerIdentities.Add(target);
}
}
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
{
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
sourceCandidate,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
context.SpeakerIdentities.Remove(sourceCandidate);
}
target.CanonicalName = targetName;
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
outlierPruner.AddAndPrune(
target,
vectors,
now);
target.UpdatedAt = now;
await context.SaveChangesAsync(cancellationToken);
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
target.References,
sourceLabel,
targetName,
cancellationToken);
}
public async Task DeleteSpeakerIdentityAsync(
string identity,
CancellationToken cancellationToken)
{
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
{
return;
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities.FirstOrDefault(candidate =>
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
if (target is null)
{
return;
}
context.SpeakerIdentities.Remove(target);
await context.SaveChangesAsync(cancellationToken);
}
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
SpeakerIdentificationRequest request,
bool final,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
if (!options.Enabled || request.Segments.Count == 0)
{
return EmptyResult(request.Segments);
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var knownMappings = request.KnownSpeakerMappings ??
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (final && knownMappings.Count > 0)
{
await PersistMappedSpeakerEvidenceAsync(
context,
request,
knownMappings,
cancellationToken);
}
var identifiedNames = knownMappings.Values
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var segmentSpeaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
{
identifiedNames.Add(segmentSpeaker.Trim());
}
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
var pendingAudits = new List<PendingIdentificationAudit>();
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
{
continue;
}
var vectors = await ResolveAutomaticVectorsAsync(
request,
speaker,
allowAudioFallback,
cancellationToken);
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
{
logger.LogInformation(
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
speaker,
vectors.Count,
resemblyzerOptions.RequiredVectorsPerSpeaker);
continue;
}
var (identity, decision) = await FindMatchAsync(
context,
attendees,
identifiedNames,
vectors,
cancellationToken);
if (identity is null)
{
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
{
unmatchedSpeakers.Add((speaker, vectors));
}
continue;
}
var now = DateTimeOffset.UtcNow;
var previousCanonicalName = identity.CanonicalName;
var previousReferenceCount = identity.References.Count;
outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
if (final)
{
UpdateMatchedIdentity(identity, attendees);
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
}
var displayName = identity.GetDisplayName();
if (!string.IsNullOrWhiteSpace(displayName))
{
mappings[speaker] = displayName;
identifiedNames.Add(displayName);
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
identifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
displayName,
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
}
}
if (final)
{
LearnUnmatchedSpeakers(
context,
request.MeetingNote,
attendees,
matchedAcceptedNames,
unmatchedSpeakers,
pendingAudits);
}
await context.SaveChangesAsync(cancellationToken);
await AppendAuditsAsync(pendingAudits, cancellationToken);
var relabeled = request.Segments
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
? segment with { Speaker = name }
: segment)
.ToList();
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
}
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
SpeakerIdentityDbContext context,
IReadOnlyList<string> attendees,
IReadOnlySet<string> identifiedNames,
IReadOnlyList<float[]> queryVectors,
CancellationToken cancellationToken)
{
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
var identities = await LoadIdentities(context)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var candidates = identities
.Select(identity => new
{
Identity = identity,
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.OrderByDescending(candidate => candidate.IsAttendee)
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
.ThenBy(candidate => candidate.Identity.Id)
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
candidate.Identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
candidate.Identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
var match = clusterMatcher.Match(queryVectors, candidates);
return (
match.IdentityId is { } identityId
? identities.Single(identity => identity.Id == identityId)
: null,
match);
}
private void LearnUnmatchedSpeakers(
SpeakerIdentityDbContext context,
MeetingNote meetingNote,
IReadOnlyList<string> attendees,
IReadOnlySet<string> matchedAcceptedNames,
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
ICollection<PendingIdentificationAudit> pendingAudits)
{
var candidates = attendees
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (candidates.Count == 0)
{
return;
}
foreach (var (speaker, vectors) in unmatchedSpeakers)
{
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates
.Select(name => new SpeakerCandidateName { Name = name })
.ToList(),
References = [CreateReference(meetingNote, now)]
};
outlierPruner.AddAndPrune(
identity,
vectors,
now);
context.SpeakerIdentities.Add(identity);
logger.LogInformation(
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
speaker,
identity.VoiceVectors.Count,
string.Join(", ", candidates));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
}
}
private async Task AppendAuditsAsync(
IEnumerable<PendingIdentificationAudit> pendingAudits,
CancellationToken cancellationToken)
{
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
audit.Identity.References,
audit.Speaker,
audit.Name,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
audit.Identity.Id);
}
}
}
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback,
cancellationToken);
if (samples.Count < requiredCount)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task PersistMappedSpeakerEvidenceAsync(
SpeakerIdentityDbContext context,
SpeakerIdentificationRequest request,
IReadOnlyDictionary<string, string> knownMappings,
CancellationToken cancellationToken)
{
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
foreach (var (speaker, mappedName) in knownMappings)
{
var identity = identities
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
.OrderBy(candidate =>
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(candidate => candidate.Id)
.FirstOrDefault();
if (identity is null)
{
logger.LogWarning(
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
speaker,
mappedName);
continue;
}
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
if (vectors.Count == 0)
{
continue;
}
var now = DateTimeOffset.UtcNow;
var previousReferenceCount = identity.References.Count;
var vectorUpdate = outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
logger.LogInformation(
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
vectorUpdate.AddedCount,
speaker,
identity.Id);
}
}
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
CancellationToken cancellationToken)
{
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback: true,
cancellationToken);
if (samples.Count == 0)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
SpeakerIdentificationRequest request,
string speaker,
int maxSamples,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var suppliedSamples = request.Samples?
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
.Where(sample => sample.WavBytes.Length > 0)
.OrderByDescending(sample => sample.Score)
.Take(maxSamples)
.ToList()
?? [];
var wavSamples = suppliedSamples
.Select(sample => sample.WavBytes)
.ToList();
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
{
return wavSamples;
}
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
request.Segments,
speaker,
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
request.Segments.Count);
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
{
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
span,
cancellationToken);
if (wavBytes.Length > 0)
{
wavSamples.Add(wavBytes);
}
if (wavSamples.Count >= maxSamples)
{
break;
}
}
logger.LogInformation(
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
wavSamples.Count,
maxSamples,
speaker,
suppliedSamples.Count,
wavSamples.Count - suppliedSamples.Count);
return wavSamples;
}
private static bool OverlapsSuppliedSample(
IReadOnlyList<TranscriptionSegment> span,
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
{
if (span.Count == 0)
{
return false;
}
var start = span[0].Start;
var end = span[^1].End;
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
}
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
{
return context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References);
}
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return;
}
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
}
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
{
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
}
private sealed record PendingIdentificationAudit(
SpeakerIdentity Identity,
string Speaker,
string Name);
}
@@ -1,172 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ResemblyzerVoiceClusterMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
public ResemblyzerSpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ResemblyzerVoiceClusterMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
TimeSpan? recentIdentityAge,
CancellationToken cancellationToken)
{
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var recentIds = identities
.Where(identity => identity.CreatedAt >= cutoff)
.Select(identity => identity.Id)
.ToList();
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
var attempts = 0;
var mergedPairs = 0;
var pendingAudits = new List<PendingMergeAudit>();
foreach (var sourceId in recentIds)
{
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
if (source is null)
{
continue;
}
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
source,
resemblyzerOptions.ModelId,
logger)
.Take(required * 2)
.ToList();
if (sourceVectors.Count < required * 2)
{
logger.LogInformation(
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
source.Id,
sourceVectors.Count,
required * 2);
continue;
}
var candidates = identities
.Where(identity => identity.Id != source.Id)
.Where(identity => identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(identity => new ResemblyzerVoiceVectorCandidate(
identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
if (candidates.Count == 0)
{
continue;
}
attempts++;
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
if (!first.Accepted || first.IdentityId is not { } targetId)
{
continue;
}
attempts++;
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
if (!second.Accepted || second.IdentityId != targetId)
{
logger.LogInformation(
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
source.Id,
first.IdentityId,
second.IdentityId);
continue;
}
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
if (target is null)
{
continue;
}
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
source,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
pendingAudits.Add(new PendingMergeAudit(
target,
targetName,
sourceName));
context.SpeakerIdentities.Remove(source);
identities.Remove(source);
mergedPairs++;
}
await context.SaveChangesAsync(cancellationToken);
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
audit.Identity.References,
audit.TargetName,
audit.SourceName,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
audit.Identity.Id);
}
}
return new SpeakerIdentityMergeResult(
recentIds.Count,
identities.Count,
attempts,
mergedPairs);
}
private sealed record PendingMergeAudit(
SpeakerIdentity Identity,
string TargetName,
string SourceName);
}
@@ -1,204 +0,0 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorCandidate(
int IdentityId,
IReadOnlyList<float[]> Vectors);
public sealed record ResemblyzerVoiceClusterMatchResult(
int? IdentityId,
string Reason,
double Cohesion,
double? BestSimilarity,
double? RunnerUpSimilarity)
{
public bool Accepted => IdentityId.HasValue;
}
public sealed class ResemblyzerVoiceClusterMatcher
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
public ResemblyzerVoiceClusterMatcher(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceClusterMatcher> logger)
{
this.options = options;
this.logger = logger;
}
public ResemblyzerVoiceClusterMatchResult Match(
IReadOnlyList<float[]> queryVectors,
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
{
var requiredCount = options.RequiredVectorsPerSpeaker;
if (queryVectors.Count < requiredCount)
{
return Reject(
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
cohesion: 0,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var normalizedQuery = queryVectors
.Take(requiredCount)
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var cohesion = MeanPairwiseCosine(normalizedQuery);
if (cohesion < options.MinimumClusterCohesion)
{
return Reject(
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
cohesion,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var scored = candidates
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
.Where(candidate => candidate is not null)
.Select(candidate => candidate!.Value)
.OrderByDescending(candidate => candidate.Similarity)
.ThenBy(candidate => candidate.IdentityId)
.ToList();
if (scored.Count == 0)
{
return Reject("no compatible identity vectors were available", cohesion, null, null);
}
var best = scored[0];
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
if (best.Similarity < options.MinimumIdentitySimilarity)
{
return Reject(
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
cohesion,
best.Similarity,
runnerUp);
}
if (runnerUp is { } runnerUpSimilarity &&
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
{
return Reject(
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
cohesion,
best.Similarity,
runnerUpSimilarity);
}
logger.LogInformation(
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
best.IdentityId,
cohesion,
best.Similarity,
runnerUp);
return new ResemblyzerVoiceClusterMatchResult(
best.IdentityId,
"accepted",
cohesion,
best.Similarity,
runnerUp);
}
private ResemblyzerVoiceClusterMatchResult Reject(
string reason,
double cohesion,
double? bestSimilarity,
double? runnerUpSimilarity)
{
logger.LogInformation(
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
return new ResemblyzerVoiceClusterMatchResult(
null,
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
}
private (int IdentityId, double Similarity)? ScoreCandidate(
IReadOnlyList<float[]> normalizedQuery,
ResemblyzerVoiceVectorCandidate candidate)
{
if (candidate.Vectors.Count == 0)
{
return null;
}
try
{
var normalizedCandidate = candidate.Vectors
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
var similarities = normalizedQuery
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
.Order()
.ToList();
return (candidate.IdentityId, Median(similarities));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
candidate.IdentityId);
return null;
}
}
private static float[] Centroid(IReadOnlyList<float[]> vectors)
{
var centroid = new float[ResemblyzerVectorContract.Dimensions];
foreach (var vector in vectors)
{
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] += vector[index];
}
}
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] /= vectors.Count;
}
return centroid;
}
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
{
if (vectors.Count < 2)
{
return 1;
}
double total = 0;
var pairs = 0;
for (var first = 0; first < vectors.Count - 1; first++)
{
for (var second = first + 1; second < vectors.Count; second++)
{
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
pairs++;
}
}
return total / pairs;
}
private static double Median(IReadOnlyList<double> sortedValues)
{
var middle = sortedValues.Count / 2;
return sortedValues.Count % 2 == 0
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
: sortedValues[middle];
}
}
@@ -1,208 +0,0 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorUpdateResult(
int AddedCount,
int RemovedCount)
{
public bool Changed => AddedCount > 0 || RemovedCount > 0;
}
public sealed class ResemblyzerVoiceVectorOutlierPruner
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
public ResemblyzerVoiceVectorOutlierPruner(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
{
this.options = options;
this.logger = logger;
}
public int Prune(SpeakerIdentity identity)
{
var compatible = new List<DecodedSpeakerVoiceVector>();
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
identity,
options.ModelId,
logger))
{
try
{
compatible.Add(new DecodedSpeakerVoiceVector(
entry.Stored,
SpeakerVoiceVectors.Normalize(
entry.Vector,
$"Stored voice vector {entry.Stored.Id}")));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
entry.Stored.Id,
identity.Id);
}
}
var minimumVectors = options.OutlierPruningMinimumVectors;
if (compatible.Count < minimumVectors)
{
return 0;
}
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
var ranked = clusters
.GroupBy(cluster => cluster)
.Where(group => group.Key > 0)
.Select(group => new { Id = group.Key, Count = group.Count() })
.OrderByDescending(group => group.Count)
.ThenBy(group => group.Id)
.ToList();
if (ranked.Count == 0)
{
return Skip(identity, "no dense cluster was found");
}
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
{
return Skip(identity, "no uniquely largest dense cluster was found");
}
var dominant = ranked[0];
var ratio = (double)dominant.Count / compatible.Count;
if (ratio < options.OutlierPruningMinimumClusterRatio)
{
return Skip(
identity,
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
}
var removed = compatible
.Where((_, index) => clusters[index] != dominant.Id)
.Select(item => item.Stored)
.ToList();
foreach (var vector in removed)
{
identity.VoiceVectors.Remove(vector);
}
if (removed.Count > 0)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
}
logger.LogInformation(
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
removed.Count,
identity.Id,
dominant.Count,
compatible.Count);
return removed.Count;
}
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
SpeakerIdentity identity,
IReadOnlyList<float[]> vectors,
DateTimeOffset createdAt)
{
var removedBeforeAdding = Prune(identity);
var added = SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
options.ModelId,
options.MaxVectorsPerIdentity,
createdAt);
var removedAfterAdding = Prune(identity);
var result = new ResemblyzerVoiceVectorUpdateResult(
added,
removedBeforeAdding + removedAfterAdding);
if (result.Changed)
{
identity.UpdatedAt = createdAt;
}
return result;
}
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
{
var neighbors = Enumerable.Range(0, vectors.Count)
.Select(index => Enumerable.Range(0, vectors.Count)
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
.ToList())
.ToList();
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
var labels = new int[vectors.Count];
var clusterId = 0;
for (var point = 0; point < vectors.Count; point++)
{
if (labels[point] != 0)
{
continue;
}
if (neighbors[point].Count < minimumNeighbors)
{
labels[point] = -1;
continue;
}
clusterId++;
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
}
return labels;
}
private static void ExpandCluster(
int seed,
int clusterId,
IReadOnlyList<List<int>> neighbors,
int[] labels,
int minimumNeighbors)
{
labels[seed] = clusterId;
var pending = new Queue<int>(neighbors[seed]);
var queued = neighbors[seed].ToHashSet();
while (pending.TryDequeue(out var point))
{
if (labels[point] == -1)
{
labels[point] = clusterId;
}
if (labels[point] != 0)
{
continue;
}
labels[point] = clusterId;
if (neighbors[point].Count < minimumNeighbors)
{
continue;
}
foreach (var neighbor in neighbors[point])
{
if (queued.Add(neighbor))
{
pending.Enqueue(neighbor);
}
}
}
}
private int Skip(
SpeakerIdentity identity,
string reason)
{
logger.LogInformation(
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
identity.Id,
reason);
return 0;
}
}
@@ -1,54 +0,0 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerWarmupHostedService : BackgroundService
{
private readonly IResemblyzerVoiceEncoder encoder;
private readonly bool speakerIdentificationEnabled;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
public ResemblyzerWarmupHostedService(
IResemblyzerVoiceEncoder encoder,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerWarmupHostedService> logger)
{
this.encoder = encoder;
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!speakerIdentificationEnabled || !options.Enabled)
{
return;
}
await Task.Yield();
try
{
logger.LogInformation(
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
await encoder.WarmUpAsync(stoppingToken);
logger.LogInformation(
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
}
catch (Exception exception)
{
logger.LogError(
exception,
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
VaultPath.Resolve(options.RuntimeFolder));
}
}
}
+1 -36
View File
@@ -21,8 +21,6 @@ public sealed class SpeakerIdentity
public List<SpeakerSnippet> Snippets { get; set; } = [];
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
public List<SpeakerIdentityReference> References { get; set; } = [];
[NotMapped]
@@ -95,7 +93,7 @@ public static class SpeakerIdentityReferences
string.IsNullOrWhiteSpace(reference.TranscriptPath);
}
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
@@ -126,25 +124,6 @@ public sealed class SpeakerSnippet
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerVoiceVector
{
public int Id { get; set; }
public int SpeakerIdentityId { get; set; }
public SpeakerIdentity? SpeakerIdentity { get; set; }
public string ModelId { get; set; } = "";
public int Dimensions { get; set; }
public byte[] VectorBytes { get; set; } = [];
public string Fingerprint { get; set; } = "";
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerAlias
{
public int Id { get; set; }
@@ -171,8 +150,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -198,11 +175,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.VoiceVectors)
.WithOne(vector => vector.SpeakerIdentity)
.HasForeignKey(vector => vector.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.References)
.WithOne(reference => reference.SpeakerIdentity)
.HasForeignKey(reference => reference.SpeakerIdentityId)
@@ -217,13 +189,6 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
modelBuilder.Entity<SpeakerIdentityReference>()
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
.IsUnique();
@@ -20,20 +20,17 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerIdentityMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ILogger<SpeakerIdentityMergeService> logger;
public SpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerIdentityMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<SpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
this.logger = logger;
}
@@ -47,11 +44,9 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
@@ -159,12 +154,10 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeIntoAndPrune(
SpeakerIdentityMerger.MergeInto(
target,
source,
options.MaxSnippetsPerSpeaker,
options.Resemblyzer.MaxVectorsPerIdentity,
outlierPruner);
options.MaxSnippetsPerSpeaker);
logger.LogInformation(
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
source.Id,
@@ -2,24 +2,10 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityMerger
{
public static void MergeIntoAndPrune(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets,
int maxVoiceVectors,
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
{
outlierPruner.Prune(target);
outlierPruner.Prune(source);
MergeInto(target, source, maxSnippets, maxVoiceVectors);
outlierPruner.Prune(target);
}
public static void MergeInto(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets,
int maxVoiceVectors = int.MaxValue)
int maxSnippets)
{
AddAlias(target, source.CanonicalName);
foreach (var alias in source.Aliases)
@@ -49,26 +35,6 @@ internal static class SpeakerIdentityMerger
.ToList();
target.Snippets.Clear();
target.Snippets.AddRange(retainedSnippets);
var retainedVectors = target.VoiceVectors
.Concat(source.VoiceVectors)
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
.OrderByDescending(vector => vector.CreatedAt)
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Take(Math.Max(1, maxVoiceVectors))
.Select(vector => new SpeakerVoiceVector
{
SpeakerIdentity = target,
ModelId = vector.ModelId,
Dimensions = vector.Dimensions,
VectorBytes = vector.VectorBytes.ToArray(),
Fingerprint = vector.Fingerprint,
CreatedAt = vector.CreatedAt
})
.ToList();
target.VoiceVectors.Clear();
target.VoiceVectors.AddRange(retainedVectors);
target.UpdatedAt = DateTimeOffset.UtcNow;
}
@@ -1,103 +0,0 @@
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityNaming
{
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[] { identity.CanonicalName }
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
public static bool MatchesAnyAcceptedName(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return GetAcceptedNames(identity).Any(names.Contains);
}
public static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(MeetingAttendeeNames.NormalizeDisplayName)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
public static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
public static bool UpdateCandidateNames(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return false;
}
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
var resetToAttendees = intersection.Count == 0;
SetCandidates(identity, resetToAttendees ? attendees : intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
}
return resetToAttendees;
}
public static void SetCandidates(
SpeakerIdentity identity,
IEnumerable<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
}
@@ -46,33 +46,6 @@ internal static class SpeakerIdentitySchema
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
"SpeakerIdentityId" INTEGER NOT NULL,
"ModelId" TEXT NOT NULL,
"Dimensions" INTEGER NOT NULL,
"VectorBytes" BLOB NOT NULL,
"Fingerprint" TEXT NOT NULL,
"CreatedAt" TEXT NOT NULL,
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
);
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
""",
cancellationToken);
}
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
target.CanonicalName = targetName;
target.UpdatedAt = now;
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
ResetCandidates(target, [targetName]);
AddMeetingReference(target, meetingReference);
var snippetAdded = AddSnippetIfNeeded(target, snippet);
await context.SaveChangesAsync(cancellationToken);
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
{
alreadyIdentifiedNames.Add(speaker.Trim());
}
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
cancellationToken);
}
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
foreach (var acceptedName in GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
{
speakerMappings[speaker] = speakerName;
alreadyIdentifiedNames.Add(speakerName);
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
foreach (var acceptedName in GetAcceptedNames(identity))
{
alreadyIdentifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
speakerName,
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
GetAcceptedNames(identity).ToList()));
}
}
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Select(identity => new
{
Identity = identity,
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsAttendee = MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
string operation,
CancellationToken cancellationToken)
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
request.Segments,
speaker,
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration);
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration);
logger.LogInformation(
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
operation,
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Include(identity => identity.References)
.ToListAsync(cancellationToken);
return identities
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
.Where(identity => GetAcceptedNames(identity).Contains(name))
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.ToListAsync(cancellationToken);
return candidates
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
.Where(identity => identity.CandidateNames.Any(candidate =>
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
identity.Aliases.Any(alias =>
@@ -646,11 +646,27 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
}
private static bool IsSameReference(
SpeakerIdentityReference first,
SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
}
private static bool MatchesAcceptedNames(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
return GetAcceptedNames(identity).Any(names.Contains);
}
private static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
}
private static Task<SpeakerIdentity?> LoadIdentityAsync(
@@ -678,25 +694,48 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
if (resetToAttendees)
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (intersection.Count == 0)
{
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
identity.Id,
FormatNames(attendees));
ResetCandidates(identity, attendees);
ReplaceOldestSnippet(identity, snippet);
return;
}
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
identity.Id,
FormatNames(currentCandidates),
FormatNames(attendees),
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
FormatNames(intersection));
ResetCandidates(identity, intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
logger.LogInformation(
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
identity.Id,
@@ -833,6 +872,51 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
}
}
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[]
{
identity.CanonicalName
}
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static string NormalizeAttendee(string attendee)
{
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(
@@ -1,154 +0,0 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed record SpeakerSampleCollectionPolicy(
int MinimumRetainedSamples,
bool RequireNonOverlappingSamples)
{
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
{
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
}
public int ResolveRetainedSampleLimit(int configuredLimit)
{
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
}
}
internal static class SpeakerSampleDurationConfiguration
{
public static void ValidateOrThrow(
SpeakerIdentificationOptions options,
string context)
{
ValidateOrThrow(
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
context);
}
public static void ValidateOrThrow(
TimeSpan minimumDuration,
TimeSpan maximumDuration,
string context)
{
var error = GetError(minimumDuration, maximumDuration);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(
TimeSpan minimumDuration,
TimeSpan maximumDuration)
{
if (minimumDuration < TimeSpan.Zero)
{
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
}
if (maximumDuration <= TimeSpan.Zero)
{
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
}
return minimumDuration > maximumDuration
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
: null;
}
}
internal static class ResemblyzerSpeakerRecognitionConfiguration
{
public static void ValidateOrThrow(
ResemblyzerSpeakerRecognitionOptions options,
string context)
{
var error = GetError(options);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
{
if (options.RequiredVectorsPerSpeaker < 1)
{
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
}
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
}
if (options.OutlierPruningMinimumNeighbors < 1 ||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
}
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
}
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
}
if (!IsCosine(options.MinimumClusterCohesion))
{
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
}
if (!IsCosine(options.MinimumIdentitySimilarity))
{
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
}
return !double.IsFinite(options.MinimumSimilarityMargin) ||
options.MinimumSimilarityMargin is < 0 or > 2
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
: null;
}
private static bool IsCosine(double value)
{
return double.IsFinite(value) && value is >= -1 and <= 1;
}
}
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
: IValidateOptions<MeetingAssistantOptions>
{
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
{
var error = SpeakerSampleDurationConfiguration.GetError(
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleDuration) ??
ResemblyzerSpeakerRecognitionConfiguration.GetError(
options.SpeakerIdentification.Resemblyzer);
return error is null
? ValidateOptionsResult.Success
: ValidateOptionsResult.Fail(error);
}
}
@@ -1,82 +0,0 @@
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Speakers;
internal sealed record SpeakerSampleSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
TimeSpan SpeechDuration,
TimeSpan MaximumDuration,
IReadOnlyList<TranscriptionSegment> Segments)
{
public static SpeakerSampleSpan? Create(
TranscriptionSegment segment,
TimeSpan maximumDuration)
{
if (maximumDuration <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(maximumDuration),
maximumDuration,
"Speaker sample maximum duration must be greater than zero.");
}
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
return boundedEnd > segment.Start
? new SpeakerSampleSpan(
segment.Speaker,
segment.Start,
boundedEnd,
boundedEnd - segment.Start,
maximumDuration,
[segment with { End = boundedEnd }])
: null;
}
public bool TryExtend(
TranscriptionSegment segment,
out SpeakerSampleSpan extended)
{
extended = this;
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
segment.Start >= Start + MaximumDuration)
{
return false;
}
var boundedEnd = Min(segment.End, Start + MaximumDuration);
if (boundedEnd <= segment.Start)
{
return false;
}
var uncoveredStart = segment.Start > End ? segment.Start : End;
var additionalSpeechDuration = boundedEnd > uncoveredStart
? boundedEnd - uncoveredStart
: TimeSpan.Zero;
extended = this with
{
End = boundedEnd > End ? boundedEnd : End,
SpeechDuration = SpeechDuration + additionalSpeechDuration,
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
};
return true;
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', Segments
.Select(segment => segment.Text)
.Where(text => !string.IsNullOrWhiteSpace(text))));
}
private static TimeSpan Min(TimeSpan left, TimeSpan right)
{
return left < right ? left : right;
}
}
@@ -4,82 +4,54 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerSampleSpanSelector
{
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration)
public static bool CanExtend(
string currentSpeaker,
TimeSpan currentEnd,
TranscriptionSegment nextSegment,
TimeSpan maximumSegmentGap)
{
SpeakerSampleSpan? best = null;
SpeakerSampleSpan? current = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (!IsSpeaker(segment, speaker))
{
current = null;
continue;
}
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is not null &&
(best is null || current.SpeechDuration > best.SpeechDuration))
{
best = current;
}
}
return best is not null && best.SpeechDuration >= minimumDuration
? best.Segments
: [];
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
nextSegment.Start - currentEnd <= maximumSegmentGap;
}
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration,
int maxSpans)
TimeSpan maximumSegmentGap,
TimeSpan minimumDuration)
{
if (maxSpans <= 0)
{
return [];
}
var best = new List<TranscriptionSegment>();
var current = new List<TranscriptionSegment>();
var completed = new List<SpeakerSampleSpan>();
SpeakerSampleSpan? current = null;
TimeSpan? lastCompletedEnd = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (!IsSpeaker(segment, speaker))
if (current.Count == 0)
{
current = null;
if (IsSpeaker(segment, speaker))
{
current.Add(segment);
best = LongerSpan(current, best);
}
continue;
}
if (lastCompletedEnd is { } end && segment.Start < end)
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
{
current = null;
continue;
current.Clear();
if (!IsSpeaker(segment, speaker))
{
continue;
}
}
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is null || current.SpeechDuration < minimumDuration)
{
continue;
}
completed.Add(current);
lastCompletedEnd = current.End;
current = null;
current.Add(segment);
best = LongerSpan(current, best);
}
return completed
.OrderByDescending(span => span.SpeechDuration)
.ThenBy(span => span.Start)
.Take(maxSpans)
.OrderBy(span => span.Start)
.Select(span => span.Segments)
.ToList();
return SpanDuration(best) >= minimumDuration
? best
: [];
}
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
@@ -89,20 +61,19 @@ internal static class SpeakerSampleSpanSelector
: segments[^1].End - segments[0].Start;
}
private static SpeakerSampleSpan? ExtendOrStart(
SpeakerSampleSpan? current,
TranscriptionSegment segment,
TimeSpan maximumSampleDuration)
{
return current is not null && current.TryExtend(segment, out var extended)
? extended
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
}
private static bool IsSpeaker(
TranscriptionSegment segment,
string speaker)
{
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
}
private static List<TranscriptionSegment> LongerSpan(
List<TranscriptionSegment> current,
List<TranscriptionSegment> best)
{
return SpanDuration(current) > SpanDuration(best)
? current.ToList()
: best;
}
}
@@ -1,191 +0,0 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
namespace MeetingAssistant.Speakers;
internal sealed record DecodedSpeakerVoiceVector(
SpeakerVoiceVector Stored,
float[] Vector);
internal static class ResemblyzerVectorContract
{
public const int Dimensions = 256;
}
internal static class SpeakerVoiceVectors
{
public static float[] Decode(SpeakerVoiceVector stored)
{
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
{
throw new InvalidDataException(
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
}
var vector = new float[stored.Dimensions];
for (var index = 0; index < vector.Length; index++)
{
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
}
}
return vector;
}
public static int AddDistinct(
SpeakerIdentity identity,
IEnumerable<float[]> vectors,
string modelId,
int maximumCount,
DateTimeOffset createdAt)
{
var limit = Math.Max(1, maximumCount);
var fingerprints = identity.VoiceVectors
.Select(vector => vector.Fingerprint)
.ToHashSet(StringComparer.Ordinal);
var added = 0;
foreach (var vector in vectors)
{
if (identity.VoiceVectors.Count >= limit)
{
break;
}
var bytes = Encode(vector);
var fingerprint = Fingerprint(modelId, bytes);
if (!fingerprints.Add(fingerprint))
{
continue;
}
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = modelId,
Dimensions = vector.Length,
VectorBytes = bytes,
Fingerprint = fingerprint,
CreatedAt = createdAt
});
added++;
}
return added;
}
public static IReadOnlyList<float[]> DecodeCompatible(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
return DecodeCompatibleEntries(identity, modelId, logger)
.Select(entry => entry.Vector)
.ToList();
}
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
var vectors = new List<DecodedSpeakerVoiceVector>();
foreach (var stored in identity.VoiceVectors
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
.OrderBy(vector => vector.CreatedAt)
.ThenBy(vector => vector.Id))
{
try
{
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
stored.Id,
identity.Id);
}
}
return vectors;
}
public static double Cosine(float[] first, float[] second)
{
double dot = 0;
for (var index = 0; index < first.Length; index++)
{
dot += first[index] * second[index];
}
return dot;
}
public static float[] Normalize(float[]? vector, string description)
{
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
double magnitudeSquared = 0;
foreach (var value in vector)
{
if (!float.IsFinite(value))
{
throw new InvalidDataException($"{description} contains a non-finite value.");
}
magnitudeSquared += value * value;
}
var magnitude = Math.Sqrt(magnitudeSquared);
if (magnitude <= double.Epsilon)
{
throw new InvalidDataException($"{description} has zero magnitude.");
}
return vector.Select(value => (float)(value / magnitude)).ToArray();
}
private static byte[] Encode(float[] vector)
{
if (vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
var bytes = new byte[vector.Length * sizeof(float)];
for (var index = 0; index < vector.Length; index++)
{
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException("Voice vector contains a non-finite value.");
}
BinaryPrimitives.WriteSingleLittleEndian(
bytes.AsSpan(index * sizeof(float), sizeof(float)),
vector[index]);
}
return bytes;
}
private static string Fingerprint(string modelId, byte[] vectorBytes)
{
var modelBytes = Encoding.UTF8.GetBytes(modelId);
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
modelBytes.CopyTo(input, 0);
input[modelBytes.Length] = 0;
vectorBytes.CopyTo(input, modelBytes.Length + 1);
return Convert.ToHexString(SHA256.HashData(input));
}
}
@@ -1,391 +0,0 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public interface IResemblyzerVoiceEncoder
{
Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken);
Task WarmUpAsync(CancellationToken cancellationToken);
}
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
{
private const string EnvironmentSchemaVersion = "venv-v2";
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
private const string LibrosaRequirement = "librosa>=0.9.1";
private const string ScipyRequirement = "scipy>=1.2.1";
private readonly SemaphoreSlim commandLock = new(1, 1);
private readonly ICommandRunner commandRunner;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
private string? verifiedEnvironmentPythonPath;
public VenvResemblyzerVoiceEncoder(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<VenvResemblyzerVoiceEncoder> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
public async Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
if (wavSamples.Count == 0)
{
return [];
}
if (wavSamples.Any(sample => sample.Length == 0))
{
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
}
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(inputFolder);
try
{
for (var index = 0; index < wavSamples.Count; index++)
{
await File.WriteAllBytesAsync(
Path.Combine(inputFolder, $"{index:D4}.wav"),
wavSamples[index],
cancellationToken);
}
await commandLock.WaitAsync(cancellationToken);
try
{
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
}
finally
{
commandLock.Release();
}
}
finally
{
try
{
Directory.Delete(inputFolder, recursive: true);
}
catch (DirectoryNotFoundException)
{
}
catch (IOException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
catch (UnauthorizedAccessException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
}
}
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
string inputFolder,
int expectedCount,
CancellationToken cancellationToken)
{
return await RunWithTimeoutAsync(async token =>
{
var pythonPath = await EnsureEnvironmentAsync(token);
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
var result = await RunRequiredAsync(
pythonPath,
[scriptPath, Path.GetFullPath(inputFolder)],
"encoding",
token);
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
}, cancellationToken);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
await commandLock.WaitAsync(cancellationToken);
try
{
await RunWithTimeoutAsync(async token =>
{
await EnsureEnvironmentAsync(token);
return true;
}, cancellationToken);
}
finally
{
commandLock.Release();
}
}
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
{
ValidateDependencySettings();
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
var readyPath = Path.Combine(environmentFolder, ".ready");
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
&& File.Exists(pythonPath)
&& File.Exists(readyPath))
{
return pythonPath;
}
if (File.Exists(pythonPath) && File.Exists(readyPath))
{
var verification = await commandRunner.RunAsync(
pythonPath,
WarmupArguments,
cancellationToken);
if (verification.ExitCode == 0)
{
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
logger.LogWarning(
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
verification.ExitCode,
verification.StandardError);
File.Delete(readyPath);
}
Directory.CreateDirectory(environmentFolder);
await RunRequiredAsync(
options.PythonCommand,
["-m", "venv", "--clear", environmentFolder],
"virtual environment creation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
["-m", "pip", "install", "--upgrade", "pip"],
"pip upgrade",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
"--index-url", options.TorchIndexUrl,
TorchRequirement
],
"CPU PyTorch installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
.. RuntimeDependencyRequirements
],
"Resemblyzer dependency installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install", "--no-deps",
ResemblyzerRequirement
],
"Resemblyzer installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
WarmupArguments,
"environment verification",
cancellationToken);
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
private async Task<CommandResult> RunRequiredAsync(
string fileName,
IReadOnlyList<string> arguments,
string operation,
CancellationToken cancellationToken)
{
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
ThrowIfFailed(result, operation);
return result;
}
private static string BuildEncodingScript()
{
return
"import json\n"
+ "import sys\n"
+ "import numpy as np\n"
+ "from pathlib import Path\n"
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
+ "vectors = []\n"
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
+ " wav = preprocess_wav(path)\n"
+ " vector = encoder.embed_utterance(wav)\n"
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(vectors, allow_nan=False))\n"
+ $"print('{JsonEnd}')\n";
}
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
{
var json = ExtractJson(output);
float[][]? vectors;
try
{
vectors = JsonSerializer.Deserialize<float[][]>(json);
}
catch (JsonException exception)
{
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
}
if (vectors is null || vectors.Length != expectedCount)
{
throw new InvalidDataException(
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
}
return vectors
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
.ToList();
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private string BuildEnvironmentKey()
{
var settings = string.Join('\n', EnvironmentManifest);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
}
private string TorchRequirement => $"torch=={options.TorchVersion}";
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
private static string[] WarmupArguments =>
[
"-c",
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
];
private string[] RuntimeDependencyRequirements =>
[
NumpyBeforePython313Requirement,
NumpyFromPython313Requirement,
LibrosaRequirement,
ScipyRequirement,
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
];
private IEnumerable<string> EnvironmentManifest =>
[
EnvironmentSchemaVersion,
options.PythonCommand,
options.TorchIndexUrl,
TorchRequirement,
.. RuntimeDependencyRequirements,
ResemblyzerRequirement
];
private static string GetEnvironmentPythonPath(string environmentFolder)
{
return OperatingSystem.IsWindows()
? Path.Combine(environmentFolder, "Scripts", "python.exe")
: Path.Combine(environmentFolder, "bin", "python");
}
private void ValidateDependencySettings()
{
if (string.IsNullOrWhiteSpace(options.PythonCommand))
{
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
}
ValidateVersion(options.PackageVersion, "Resemblyzer package");
ValidateVersion(options.TorchVersion, "PyTorch");
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|| indexUri.Scheme != Uri.UriSchemeHttps)
{
throw new InvalidOperationException(
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
}
}
private static void ValidateVersion(string version, string dependency)
{
if (!PackageVersionPattern().IsMatch(version))
{
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
}
}
private async Task<T> RunWithTimeoutAsync<T>(
Func<CancellationToken, Task<T>> operation,
CancellationToken cancellationToken)
{
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(options.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
return await operation(linkedSource?.Token ?? cancellationToken);
}
catch (OperationCanceledException) when (
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
}
}
private static void ThrowIfFailed(CommandResult result, string operation)
{
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
private static partial Regex PackageVersionPattern();
}
@@ -18,12 +18,6 @@ public sealed class BoundMeetingProjectResolver
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
public List<BoundMeetingProject> GetAllProjects() => !Directory.Exists(ProjectsRoot) ? [] :
Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken = default)
@@ -39,8 +33,10 @@ public sealed class BoundMeetingProjectResolver
return [];
}
return GetAllProjects()
return Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
@@ -7,6 +7,7 @@ using System.Text.Json.Nodes;
using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Logging;
using OpenAI;
using OpenAI.Responses;
namespace MeetingAssistant.Summary;
@@ -797,7 +798,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
string apiKey,
AsyncLocal<string?> requestInitiator)
{
var options = new ResponsesClientOptions
var options = new OpenAIClientOptions
{
Endpoint = httpClient.BaseAddress
?? throw new InvalidOperationException("LiteLLM HTTP client requires a base address."),
@@ -20,20 +20,12 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
Use override_speaker only when you are very certain that a transcript speaker label belongs to a named person, for example from user notes, OCR evidence with a matching timestamp, or very clear context cues. Provide the exact speaker label from the transcript and the replacement speaker name. If the replacement speaker already exists in the transcript, use merge=true only when you are certain both speaker labels are the same identity; otherwise do not merge them.
Use delete_identity only when you are certain that an existing speaker identity was wrongfully matched. Provide the exact identity name currently used in the transcript.
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
Use list_projects first to see which projects are bound to this meeting. Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. When embedding them, encode spaces in image-link targets as `%20`; never leave literal spaces in the link target. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
Keep the output grounded in the source material and explicitly say when a section has no known items.
""";
private const string ProjectAndRequestInstructions = """
Project discovery and approval:
list_projects returns every configured project's id, displayname, and description; only projects in the current meeting note's projects frontmatter are accessible for reads, writes, search, and historical summaries. Catalog visibility does not grant file access.
Before finalizing the summary, use the catalog when meeting evidence suggests a missing project association. Call request_project_association with its exact id and a short evidence-based reason (one line, at most 100 characters). It waits for user approval for up to ten minutes. A denial or timeout grants no access; continue with the existing scope. Approval adds the project to the meeting and returns its current project instructions; follow them immediately.
For each associated project, maintain a root PROJECT.md with YAML frontmatter name (display name) and description (at most 256 characters). Describe the project's purpose and distinguishing context so future summarizers can select it. Read existing content first and use write_projectfile to create missing metadata or update it from grounded knowledge, preserving other fields and body notes. Do not invent facts or write unassociated projects.
If a workflow improvement would help, use request_workflow_change with a short one-line intention (at most 100 characters) and a detailed self-contained change prompt, preferably a scenario/specification with expected behavior. This returns requested immediately and does not mean approved or implemented. Continue summarizing; only user approval starts the separate settings agent. You cannot change workflows directly.
""";
private readonly MeetingAssistantOptions options;
private readonly BoundMeetingProjectResolver projectResolver;
@@ -58,7 +50,6 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt)
? DefaultInitialPrompt
: options.Agent.InitialPrompt.Trim();
instructions = instructions.TrimEnd() + "\n\n" + ProjectAndRequestInstructions;
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, options, cancellationToken);
return string.IsNullOrWhiteSpace(projectInstructions)
? instructions
@@ -69,15 +60,44 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, options, cancellationToken);
if (projects.Count == 0)
{
return "";
}
var blocks = projects.Select(project =>
$"# {project.Name}\n\n{project.Instructions.Trim()}");
return "---\nprojects:\n\n" + string.Join("\n\n", blocks);
}
private async Task<List<ProjectInstructions>> GetBoundProjectsWithInstructionsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var resolver = ReferenceEquals(options, this.options)
? projectResolver
: new BoundMeetingProjectResolver(options);
var blocks = new List<string>();
var projects = new List<ProjectInstructions>();
foreach (var project in await resolver.GetBoundProjectsAsync(artifacts, cancellationToken))
{
blocks.Add(await ProjectMetadata.BuildContextAsync(project, cancellationToken));
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
if (!File.Exists(agentsPath))
{
continue;
}
var content = await File.ReadAllTextAsync(agentsPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(content))
{
projects.Add(new ProjectInstructions(project.Name, content));
}
}
return blocks.Count == 0 ? "" : "---\nprojects:\n\n" + string.Join("\n\n", blocks);
return projects;
}
private sealed record ProjectInstructions(string Name, string Instructions);
}
+17 -89
View File
@@ -2,9 +2,6 @@ using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using System.Text.Json;
using MeetingAssistant.Notifications;
using YamlDotNet.RepresentationModel;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
@@ -19,8 +16,6 @@ public sealed class MeetingSummaryTools
private readonly SummaryAgentWriteAudit? writeAudit;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly BoundMeetingProjectResolver projectResolver;
private readonly AgentApprovalService? approvals;
private readonly WorkflowChangeRequestService? workflowRequests;
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
: this(artifacts, new MeetingAssistantOptions(), null)
@@ -32,9 +27,7 @@ public sealed class MeetingSummaryTools
MeetingAssistantOptions options,
IDictationWordStore? dictationWordStore = null,
SummaryAgentWriteAudit? writeAudit = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
AgentApprovalService? approvals = null,
WorkflowChangeRequestService? workflowRequests = null)
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{
this.artifacts = artifacts;
this.options = options;
@@ -42,8 +35,6 @@ public sealed class MeetingSummaryTools
this.writeAudit = writeAudit;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
projectResolver = new BoundMeetingProjectResolver(options);
this.approvals = approvals;
this.workflowRequests = workflowRequests;
}
public Task<string> ReadTranscript(int? @from = null, int? to = null, int? tail = null)
@@ -300,71 +291,8 @@ public sealed class MeetingSummaryTools
public async Task<string> ListProjects()
{
var projects = await Task.WhenAll(projectResolver.GetAllProjects().Select(project => ProjectMetadata.ReadAsync(project)));
return JsonSerializer.Serialize(projects.Select(project => new
{
id = project.Id, displayname = project.DisplayName, description = project.Description
}));
}
public async Task<string> RequestProjectAssociation(string project_id, string reason, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(reason) || reason.Length > 100 || reason.Contains('\n') || reason.Contains('\r'))
return "Refused: reason must be a nonempty single line of at most 100 characters.";
var project = projectResolver.GetAllProjects()
.FirstOrDefault(candidate => string.Equals(candidate.Name, project_id, StringComparison.OrdinalIgnoreCase));
if (project is null) return "Refused: unknown project ID.";
var associated = await projectResolver.ReadMeetingProjectNamesAsync(artifacts, cancellationToken);
if (!associated.Contains(project.Name))
{
var metadata = await ProjectMetadata.ReadAsync(project, cancellationToken);
if (approvals is null || !await approvals.ConfirmAsync("Project association requested",
$"Associate {metadata.DisplayName} ({metadata.Id}) with {Path.GetFileName(artifacts.MeetingNotePath)}?\n{reason}",
cancellationToken))
return "Denied: project association was not approved.";
cancellationToken.ThrowIfCancellationRequested();
if (!Directory.Exists(project.Path) || !File.Exists(artifacts.MeetingNotePath))
return "Refused: the project or meeting note no longer exists.";
await AddProjectAssociationAsync(project.Name, cancellationToken);
}
return "Approved: project is associated with this meeting.\n\n"
+ await ProjectMetadata.BuildContextAsync(project, cancellationToken);
}
public string RequestWorkflowChange(string intention, string detailed_prompt)
=> workflowRequests?.Request(intention, detailed_prompt, artifacts.MeetingNotePath)
?? "Refused: workflow change requests are unavailable.";
private async Task AddProjectAssociationAsync(string projectId, CancellationToken cancellationToken)
{
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath, cancellationToken);
var document = MarkdownDocumentParser.SplitRequired(content,
"Meeting note requires frontmatter.", "Meeting note frontmatter is not closed.");
var yaml = new YamlStream();
yaml.Load(new StringReader(document.Frontmatter));
var mapping = (YamlMappingNode)yaml.Documents.Single().RootNode;
mapping.Children.TryGetValue(new YamlScalarNode("projects"), out var existing);
var projects = existing as YamlSequenceNode ?? new YamlSequenceNode();
if (existing is YamlScalarNode scalar && !string.IsNullOrWhiteSpace(scalar.Value))
projects.Add(new YamlScalarNode(scalar.Value));
if (!projects.Children.OfType<YamlScalarNode>().Any(value => string.Equals(value.Value, projectId, StringComparison.OrdinalIgnoreCase)))
projects.Add(new YamlScalarNode(projectId));
mapping.Children[new YamlScalarNode("projects")] = projects;
using var writer = new StringWriter();
yaml.Save(writer, assignAnchors: false);
var pendingPath = artifacts.MeetingNotePath + "." + Guid.NewGuid().ToString("N") + ".tmp";
try
{
await File.WriteAllTextAsync(pendingPath, "---\n" + writer + "---\n\n" + document.Body, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
File.Replace(pendingPath, artifacts.MeetingNotePath, destinationBackupFileName: null);
}
finally
{
File.Delete(pendingPath);
}
var projects = await GetBoundProjectsAsync();
return string.Join('\n', projects.Select(project => project.Name));
}
public async Task<string> ListProjectFiles(string project)
@@ -447,18 +375,10 @@ public sealed class MeetingSummaryTools
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for append; set replace_file=true only for whole-file replacement.";
}
var target = await ResolveBoundProjectFileTargetAsync(project, path);
var target = ResolveExistingProjectFilePath(project, path);
if (target is null)
{
return "Refused: project is not assigned to this meeting, does not exist, or the path escapes the project folder.";
}
var existingContent = File.Exists(target.Path) ? await File.ReadAllTextAsync(target.Path) : "";
var updatedContent = AgentFileToolContent.ApplyLineEdit(existingContent, content, editMode);
if (string.Equals(Path.GetRelativePath(target.Project.Path, target.Path), "PROJECT.md", StringComparison.OrdinalIgnoreCase)
&& !ProjectMetadata.IsValidContent(updatedContent))
{
return "Refused: PROJECT.md requires YAML frontmatter name and description; description must be at most 256 characters.";
return "Refused: project does not exist or the path escapes the project folder.";
}
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
@@ -466,11 +386,11 @@ public sealed class MeetingSummaryTools
{
await writeAudit.CaptureFileWriteAsync(
target.Path,
() => File.WriteAllTextAsync(target.Path, updatedContent));
() => AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode));
}
else
{
await File.WriteAllTextAsync(target.Path, updatedContent);
await AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode);
}
return $"{target.Project.Name}/{AgentFileToolContent.ToToolPath(path)}";
@@ -669,7 +589,7 @@ public sealed class MeetingSummaryTools
return AgentFileToolContent.IsWithinDirectory(projectRoot, fullPath) ? fullPath : null;
}
private async Task<ProjectFileTarget?> ResolveBoundProjectFileTargetAsync(string project, string path)
private ProjectFileTarget? ResolveExistingProjectFilePath(string project, string path)
{
if (string.IsNullOrWhiteSpace(project) ||
string.IsNullOrWhiteSpace(path) ||
@@ -678,7 +598,15 @@ public sealed class MeetingSummaryTools
return null;
}
var projectFolder = await ResolveBoundProjectAsync(project);
var projectsRoot = GetProjectsRoot();
if (!Directory.Exists(projectsRoot))
{
return null;
}
var projectFolder = Directory.EnumerateDirectories(projectsRoot)
.Select(candidate => new BoundMeetingProject(Path.GetFileName(candidate), candidate))
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
if (projectFolder is null)
{
return null;
@@ -1,5 +1,4 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.Agents.AI;
@@ -20,8 +19,6 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
private readonly IMeetingSummaryInstructionBuilder instructionBuilder;
private readonly IDictationWordStore dictationWordStore;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly AgentApprovalService approvals;
private readonly WorkflowChangeRequestService workflowRequests;
public OpenAiMeetingSummaryAgentPipeline(
IOptions<MeetingAssistantOptions> options,
@@ -31,9 +28,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
IMeetingSummaryFailureWriter failureWriter,
IMeetingSummaryInstructionBuilder instructionBuilder,
IDictationWordStore dictationWordStore,
IMeetingWorkflowEngine meetingWorkflowEngine,
AgentApprovalService approvals,
WorkflowChangeRequestService workflowRequests)
IMeetingWorkflowEngine meetingWorkflowEngine)
{
this.options = options.Value;
this.loggerFactory = loggerFactory;
@@ -43,8 +38,6 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
this.instructionBuilder = instructionBuilder;
this.dictationWordStore = dictationWordStore;
this.meetingWorkflowEngine = meetingWorkflowEngine;
this.approvals = approvals;
this.workflowRequests = workflowRequests;
}
public async Task<MeetingSummaryRunResult> RunAsync(
@@ -67,9 +60,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
options,
dictationWordStore,
writeAudit,
meetingWorkflowEngine,
approvals,
workflowRequests);
meetingWorkflowEngine);
var tools = CreateTools(meetingTools);
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
using var compactionSummaryClient = LiteLlmResponsesChatClient.Create(
@@ -89,11 +80,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
.AsBuilder()
.UseFunctionInvocation(
loggerFactory,
client =>
{
client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync;
client.AllowConcurrentInvocation = false;
})
client => client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync)
.Build()
.AsAIAgent(
instructions,
@@ -189,15 +176,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
AIFunctionFactory.Create(
tools.ListProjects,
"list_projects",
"List all configured projects with id, displayname and description. Catalog visibility does not grant access; project files require current meeting association."),
AIFunctionFactory.Create(
tools.RequestProjectAssociation,
"request_project_association",
"Request user approval to associate a project_id with this meeting. reason must be one line and at most 100 characters. Waits up to ten minutes; approval adds the project and returns its AGENTS.md if present."),
AIFunctionFactory.Create(
tools.RequestWorkflowChange,
"request_workflow_change",
"Request a workflow change with a one-line intention (maximum 100 characters) and a detailed_prompt describing the desired scenario/specification. Returns requested immediately; user approval separately opens and starts the settings agent."),
"List the configured project folders bound to the current meeting note frontmatter."),
AIFunctionFactory.Create(
tools.ListProjectFiles,
"list_projectfiles",
@@ -209,7 +188,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
AIFunctionFactory.Create(
tools.WriteProjectFile,
"write_projectfile",
"Write a file inside a project associated with the current meeting. Root PROJECT.md requires YAML name and description (maximum 256 characters). With no line arguments, append or create. Set replace_file=true only for intentional whole-file replacement. With from and to, replace that inclusive 1-based line range; with insert, insert at that line."),
"Write a file inside an existing project folder. With no line arguments, append or create the file. Set replace_file=true only when intentionally replacing the whole file. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."),
AIFunctionFactory.Create(
tools.ListPastProjectMeetings,
"list_past_project_meetings",
@@ -1,55 +0,0 @@
using MeetingAssistant.MeetingNotes;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary;
public sealed record ProjectMetadata(string Id, string DisplayName, string Description)
{
public static async Task<string> BuildContextAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
{
var metadata = await ReadAsync(project, cancellationToken);
var context = $"# {metadata.Id}\nDisplay name: {metadata.DisplayName}\nDescription: {metadata.Description}";
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
if (File.Exists(agentsPath))
{
var instructions = await File.ReadAllTextAsync(agentsPath, cancellationToken);
if (!string.IsNullOrWhiteSpace(instructions)) context += "\n\n" + instructions.Trim();
}
return context;
}
public static bool IsValidContent(string content) => Parse(content) is not null;
public static async Task<ProjectMetadata> ReadAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
{
var path = Path.Combine(project.Path, "PROJECT.md");
var metadata = File.Exists(path) ? Parse(await File.ReadAllTextAsync(path, cancellationToken)) : null;
return new(project.Name,
string.IsNullOrWhiteSpace(metadata?.Name) ? project.Name : metadata.Name.Trim(),
metadata?.Description?.Trim() ?? "");
}
private static Frontmatter? Parse(string content)
{
try
{
var document = MarkdownDocumentParser.SplitOptional(content);
if (!document.HasFrontmatter) return null;
var metadata = FrontmatterYamlDeserializer.Create().Deserialize<Frontmatter>(document.Frontmatter);
return string.IsNullOrWhiteSpace(metadata?.Name) || metadata.Description is null || metadata.Description.Length > 256
? null
: metadata;
}
catch (YamlException) { return null; }
catch (InvalidDataException) { return null; }
}
private sealed class Frontmatter
{
[YamlMember(Alias = "name")]
public string? Name { get; set; }
[YamlMember(Alias = "description")]
public string? Description { get; set; }
}
}
@@ -1,70 +0,0 @@
using MeetingAssistant.Notifications;
using MeetingAssistant.Workflow;
using System.Threading.Channels;
namespace MeetingAssistant.Summary;
public sealed class WorkflowChangeRequestService(
AgentApprovalService approvals,
IWorkflowRulesEditorWindowService editor,
ILogger<WorkflowChangeRequestService> logger) : BackgroundService
{
private readonly Channel<RequestDetails> requests = Channel.CreateUnbounded<RequestDetails>(
new UnboundedChannelOptions { SingleReader = true });
public string Request(string intention, string detailedPrompt, string meetingNotePath)
{
if (string.IsNullOrWhiteSpace(intention) || intention.Length > 100 || intention.Contains('\r') || intention.Contains('\n'))
return "Refused: intention must be a nonempty single line of at most 100 characters.";
if (string.IsNullOrWhiteSpace(detailedPrompt))
return "Refused: supply a detailed change prompt, preferably a scenario or specification.";
var request = new RequestDetails(Guid.NewGuid().ToString("N"), intention, detailedPrompt, meetingNotePath);
return requests.Writer.TryWrite(request) ? $"Requested: {request.Id}. User approval is pending."
: "Refused: application is stopping.";
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var pending = new List<Task>();
try
{
await foreach (var request in requests.Reader.ReadAllAsync(stoppingToken))
{
pending.RemoveAll(task => task.IsCompleted);
pending.Add(HandleAsync(request, stoppingToken));
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
finally
{
await Task.WhenAll(pending);
}
}
private async Task HandleAsync(RequestDetails request, CancellationToken cancellationToken)
{
try
{
if (await approvals.ConfirmAsync($"Workflow change requested: {request.Intention}",
$"The summarizer for {Path.GetFileName(request.MeetingNotePath)} requests a workflow change. Open the settings agent to apply it?",
cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
editor.ShowWithPrompt(request.DetailedPrompt);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { }
catch (Exception exception)
{
logger.LogWarning(exception, "Workflow request {RequestId} could not be handled", request.Id);
}
}
public override Task StopAsync(CancellationToken cancellationToken)
{
requests.Writer.TryComplete();
return base.StopAsync(cancellationToken);
}
private sealed record RequestDetails(string Id, string Intention, string DetailedPrompt, string MeetingNotePath);
}
@@ -9,8 +9,6 @@ public enum MeetingTaskbarAction
OpenSubmenu,
StartRecording,
StopRecording,
PauseTranscription,
UnpauseTranscription,
AbortRecording,
SwitchProfile,
SelectMicrophone,
@@ -53,15 +51,6 @@ public static class MeetingTaskbarMenuBuilder
}
var secondaryControls = new List<MeetingTaskbarMenuItem>();
if (status.IsRecording)
{
secondaryControls.Add(new MeetingTaskbarMenuItem(
status.IsPaused ? "Unpause transcription" : "Pause transcription",
status.IsPaused
? MeetingTaskbarAction.UnpauseTranscription
: MeetingTaskbarAction.PauseTranscription));
}
if (microphones is { Count: > 0 })
{
secondaryControls.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
@@ -249,12 +249,6 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
case MeetingTaskbarAction.StopRecording:
await coordinator.StopAsync(CancellationToken.None);
break;
case MeetingTaskbarAction.PauseTranscription:
await coordinator.SetTranscriptionPausedAsync(true, CancellationToken.None);
break;
case MeetingTaskbarAction.UnpauseTranscription:
await coordinator.SetTranscriptionPausedAsync(false, CancellationToken.None);
break;
case MeetingTaskbarAction.AbortRecording:
await coordinator.AbortAsync(CancellationToken.None);
break;

Some files were not shown because too many files have changed in this diff Show More