Public Access
Compare commits
109
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9a0bf1f83 | ||
|
|
7b2bcd3631 | ||
|
|
e011e36ef7 | ||
|
|
e17fb8b225 | ||
|
|
c01213278e | ||
|
|
7814f1c7c4 | ||
|
|
88c3b21b2a | ||
|
|
b249e5d977 | ||
|
|
3e4bf88931 | ||
|
|
5e7c9f64d0 | ||
|
|
fbd5a0852f | ||
|
|
568bedb77a | ||
|
|
f21c56b1c7 | ||
|
|
c56de2b7ab | ||
|
|
40d10d634a | ||
|
|
264a3a823f | ||
|
|
072ebf38a3 | ||
|
|
4d66de943e | ||
|
|
150f11b6a7 | ||
|
|
d7db64bec7 | ||
|
|
cdeef1967e | ||
|
|
aeb0cfe42f | ||
|
|
ef0c0faba8 | ||
|
|
93eb3d4e98 | ||
|
|
d93d758a8f | ||
|
|
48ad4fb930 | ||
|
|
cb5a300b44 | ||
|
|
b6acba8402 | ||
|
|
a82e2fff03 | ||
|
|
336d940acd | ||
|
|
cbf854fb2a | ||
|
|
bacef060fe | ||
|
|
a928382e51 | ||
|
|
e23d04a38c | ||
|
|
c6a50d307e | ||
|
|
436fa6c149 | ||
|
|
5dce5fd350 | ||
|
|
c37654a7d9 | ||
|
|
cc9044e091 | ||
|
|
f86af983e8 | ||
|
|
43fc8aaec0 | ||
|
|
8b543a2161 | ||
|
|
58896073c0 | ||
|
|
945543c078 | ||
|
|
3e1f447b20 | ||
|
|
497390c2ec | ||
|
|
b1365b202d | ||
|
|
70ffaa7357 | ||
|
|
714da456df | ||
|
|
fa67f30c49 | ||
|
|
dc37719d46 | ||
|
|
ee453e170e | ||
|
|
d51463321b | ||
|
|
41ae47fbfe | ||
|
|
f7ad2ce41a | ||
|
|
bc76151926 | ||
|
|
af9f5b34dc | ||
|
|
f3489f8e95 | ||
|
|
39591eef12 | ||
|
|
b3aeef24eb | ||
|
|
c91d956beb | ||
|
|
4c858170d9 | ||
|
|
0b6643d87f | ||
|
|
44ce419c49 | ||
|
|
a087ff0cbf | ||
|
|
8dc1b70f8a | ||
|
|
aaee123b9a | ||
|
|
1a341f1eaa | ||
|
|
e886fac311 | ||
|
|
e35ccf661f | ||
|
|
aa42e8edda | ||
|
|
2f12a96688 | ||
|
|
20eed38477 | ||
|
|
ada4a40f62 | ||
|
|
5d0ae84426 | ||
|
|
b9547ae4c4 | ||
|
|
75250f6041 | ||
|
|
cd2446f620 | ||
|
|
e192ae7cd8 | ||
|
|
3cadf08fa2 | ||
|
|
407db80413 | ||
|
|
f0aac40dfb | ||
|
|
0e0feedad2 | ||
|
|
89d81fa4c6 | ||
|
|
dc49bc3330 | ||
|
|
aff3528406 | ||
|
|
d92da18d08 | ||
|
|
ec59464340 | ||
|
|
5667102272 | ||
|
|
2f62ed467a | ||
|
|
0439a819b2 | ||
|
|
2cfe4e4ef0 | ||
|
|
b5ccf2125c | ||
|
|
0a7fc240d9 | ||
|
|
d81c2731da | ||
|
|
aa8ac26bbb | ||
|
|
6ee2d0ea08 | ||
|
|
e8742f544d | ||
|
|
18bed0532a | ||
|
|
8e2f266e66 | ||
|
|
5c67738939 | ||
|
|
92e359646b | ||
|
|
c6b2add4dd | ||
|
|
4787bf8cec | ||
|
|
aecef30627 | ||
|
|
7d16b0cad7 | ||
|
|
91ce5f5aa0 | ||
|
|
19d7c28dcc | ||
|
|
9729e644af |
@@ -15,10 +15,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v5
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: "10.0.x"
|
||||
|
||||
|
||||
@@ -102,6 +102,23 @@ public sealed class AudioMixingTests
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompositeAudioSourceKeepsSystemAudioWhileMicrophoneIsRecovering()
|
||||
{
|
||||
var microphone = new WaitingAudioSource();
|
||||
var system = new FixedAudioSource(Pcm16(10_000));
|
||||
var source = CreateSource(microphone, system);
|
||||
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source
|
||||
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(10_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
|
||||
await cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal()
|
||||
{
|
||||
@@ -221,6 +238,16 @@ public sealed class AudioMixingTests
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class WaitingAudioSource : IMeetingAudioSource
|
||||
{
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class OptionsCapturingAudioSource : IMeetingAudioSource
|
||||
{
|
||||
private readonly AudioChunk chunk;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -22,6 +23,94 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
Assert.Equal(0, harness.Recorder.StopCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedPromptStartsRecordingWithPromptedMeetingMetadata()
|
||||
{
|
||||
var harness = CreateHarness(isRecording: false);
|
||||
var selectedMetadata = new MeetingMetadata(
|
||||
"Selected planning",
|
||||
["Ada"],
|
||||
"Selected agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||
var otherMetadata = new MeetingMetadata(
|
||||
"Other planning",
|
||||
["Grace"],
|
||||
"Other agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:40:00+00:00"));
|
||||
var selectedMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-selected",
|
||||
subject: "Selected planning",
|
||||
metadata: selectedMetadata);
|
||||
var otherMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-other",
|
||||
subject: "Other planning",
|
||||
startOffset: TimeSpan.FromMinutes(40),
|
||||
metadata: otherMetadata);
|
||||
harness.Provider.Meetings = [selectedMeeting, otherMeeting];
|
||||
|
||||
await SyncThenPromptAsync(harness, selectedMeeting);
|
||||
|
||||
Assert.Equal(1, harness.Recorder.StartCount);
|
||||
Assert.Same(selectedMetadata, harness.Recorder.StartMetadata.Single());
|
||||
Assert.Equal(1, harness.Recorder.PromptStartCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedPromptWithoutMetadataStillUsesPromptStartPath()
|
||||
{
|
||||
var harness = CreateHarness(isRecording: false);
|
||||
var meeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-without-metadata",
|
||||
subject: "No metadata",
|
||||
metadata: null);
|
||||
harness.Provider.Meetings = [meeting];
|
||||
|
||||
await SyncThenPromptAsync(harness, meeting);
|
||||
|
||||
Assert.Equal(1, harness.Recorder.StartCount);
|
||||
Assert.Null(harness.Recorder.StartMetadata.Single());
|
||||
Assert.Equal(1, harness.Recorder.PromptStartCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptingDisplayedPromptsOutOfOrderUsesAcceptedPromptMetadata()
|
||||
{
|
||||
var harness = CreateHarness(isRecording: false, autoAcceptPrompts: false);
|
||||
var firstMetadata = new MeetingMetadata(
|
||||
"First planning",
|
||||
["Ada"],
|
||||
"First agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||
var secondMetadata = new MeetingMetadata(
|
||||
"Second planning",
|
||||
["Grace"],
|
||||
"Second agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||
var firstMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-first",
|
||||
subject: "First planning",
|
||||
metadata: firstMetadata);
|
||||
var secondMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-second",
|
||||
subject: "Second planning",
|
||||
metadata: secondMetadata);
|
||||
harness.Provider.Meetings = [firstMeeting, secondMeeting];
|
||||
|
||||
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
|
||||
harness.Clock.Now = firstMeeting.Start;
|
||||
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
|
||||
await harness.PromptService.RespondAsync(secondMeeting, MeetingStartPromptResponse.Record);
|
||||
|
||||
Assert.Equal([firstMeeting, secondMeeting], harness.PromptService.PromptedMeetings);
|
||||
Assert.Equal(1, harness.Recorder.StartCount);
|
||||
Assert.Same(secondMetadata, harness.Recorder.StartMetadata.Single());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedPromptStopsActiveRecordingBeforeStartingNewRecording()
|
||||
{
|
||||
@@ -34,6 +123,68 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
Assert.Equal(["stop", "start"], harness.Recorder.Commands);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ActiveRecordingPromptAttachesTheSelectedAppointmentMetadataWithoutRestarting()
|
||||
{
|
||||
var harness = CreateHarness(isRecording: true, autoAcceptPrompts: false);
|
||||
var firstMetadata = new MeetingMetadata(
|
||||
"First planning",
|
||||
["Ada"],
|
||||
"First agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||
var secondMetadata = new MeetingMetadata(
|
||||
"Second planning",
|
||||
["Grace"],
|
||||
"Second agenda",
|
||||
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||
var firstMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-first",
|
||||
subject: "First planning",
|
||||
metadata: firstMetadata);
|
||||
var secondMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-second",
|
||||
subject: "Second planning",
|
||||
metadata: secondMetadata);
|
||||
harness.Provider.Meetings = [firstMeeting, secondMeeting];
|
||||
|
||||
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
|
||||
harness.Clock.Now = firstMeeting.Start;
|
||||
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
|
||||
await harness.PromptService.RespondAsync(
|
||||
secondMeeting,
|
||||
MeetingStartPromptResponse.AttachMetadataToCurrentMeeting);
|
||||
|
||||
Assert.All(
|
||||
harness.PromptService.PromptRequests,
|
||||
request => Assert.True(request.CanAttachToCurrentMeeting));
|
||||
Assert.Equal(["attach-metadata"], harness.Recorder.Commands);
|
||||
Assert.Same(secondMetadata, harness.Recorder.AttachedMetadata.Single());
|
||||
Assert.Equal(0, harness.Recorder.StopCount);
|
||||
Assert.Equal(0, harness.Recorder.StartCount);
|
||||
Assert.True(harness.Recorder.CurrentStatus.IsRecording);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanceledCachedMeetingDoesNotPromptRecording()
|
||||
{
|
||||
var harness = CreateHarness(isRecording: false);
|
||||
var canceledMeeting = CreateMeeting(
|
||||
harness.Clock,
|
||||
id: "teams-canceled",
|
||||
subject: "Canceled project sync",
|
||||
isCanceled: true);
|
||||
harness.Provider.Meetings = [canceledMeeting];
|
||||
|
||||
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
|
||||
harness.Clock.Now = canceledMeeting.Start;
|
||||
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(harness.PromptService.PromptedMeetings);
|
||||
Assert.Equal(0, harness.Recorder.StartCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisabledCalendarPromptsDoNotQueryCalendar()
|
||||
{
|
||||
@@ -115,23 +266,28 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
string id = "teams-1",
|
||||
string subject = "Project sync",
|
||||
TimeSpan? startOffset = null,
|
||||
TimeSpan? duration = null)
|
||||
TimeSpan? duration = null,
|
||||
MeetingMetadata? metadata = null,
|
||||
bool isCanceled = false)
|
||||
{
|
||||
var start = clock.Now.Add(startOffset ?? TimeSpan.FromMinutes(30));
|
||||
return new CalendarMeeting(
|
||||
id,
|
||||
subject,
|
||||
start,
|
||||
start.Add(duration ?? TimeSpan.FromMinutes(30)));
|
||||
start.Add(duration ?? TimeSpan.FromMinutes(30)),
|
||||
metadata,
|
||||
isCanceled);
|
||||
}
|
||||
|
||||
private static SchedulerHarness CreateHarness(
|
||||
bool isRecording = false,
|
||||
bool enabled = true)
|
||||
bool enabled = true,
|
||||
bool autoAcceptPrompts = true)
|
||||
{
|
||||
var clock = new ManualCalendarPromptClock(new DateTimeOffset(2026, 6, 3, 9, 30, 0, TimeSpan.Zero));
|
||||
var provider = new RecordingCalendarMeetingProvider([]);
|
||||
var promptService = new AcceptingMeetingStartPromptService();
|
||||
var promptService = new CapturingMeetingStartPromptService(autoAcceptPrompts);
|
||||
var recorder = new RecordingPromptRecorder(isRecording);
|
||||
var scheduler = new CalendarRecordingPromptScheduler(
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
@@ -155,7 +311,7 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
private sealed record SchedulerHarness(
|
||||
CalendarRecordingPromptScheduler Scheduler,
|
||||
RecordingCalendarMeetingProvider Provider,
|
||||
AcceptingMeetingStartPromptService PromptService,
|
||||
CapturingMeetingStartPromptService PromptService,
|
||||
RecordingPromptRecorder Recorder,
|
||||
ManualCalendarPromptClock Clock);
|
||||
|
||||
@@ -179,18 +335,45 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class AcceptingMeetingStartPromptService : IMeetingStartPromptService
|
||||
private sealed class CapturingMeetingStartPromptService : IMeetingStartPromptService
|
||||
{
|
||||
public List<CalendarMeeting> PromptedMeetings { get; } = [];
|
||||
private readonly bool autoAccept;
|
||||
private readonly List<PendingPrompt> pendingPrompts = [];
|
||||
|
||||
public CapturingMeetingStartPromptService(bool autoAccept)
|
||||
{
|
||||
this.autoAccept = autoAccept;
|
||||
}
|
||||
|
||||
public List<MeetingStartPromptRequest> PromptRequests { get; } = [];
|
||||
|
||||
public IReadOnlyList<CalendarMeeting> PromptedMeetings =>
|
||||
PromptRequests.Select(request => request.Meeting).ToList();
|
||||
|
||||
public async Task ShowPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
PromptedMeetings.Add(request.Meeting);
|
||||
await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken);
|
||||
PromptRequests.Add(request);
|
||||
pendingPrompts.Add(new PendingPrompt(request.Meeting, handleResponseAsync));
|
||||
if (autoAccept)
|
||||
{
|
||||
await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public Task RespondAsync(
|
||||
CalendarMeeting meeting,
|
||||
MeetingStartPromptResponse response)
|
||||
{
|
||||
var prompt = pendingPrompts.Single(pending => ReferenceEquals(pending.Meeting, meeting));
|
||||
return prompt.HandleResponseAsync(response, CancellationToken.None);
|
||||
}
|
||||
|
||||
private sealed record PendingPrompt(
|
||||
CalendarMeeting Meeting,
|
||||
Func<MeetingStartPromptResponse, CancellationToken, Task> HandleResponseAsync);
|
||||
}
|
||||
|
||||
private sealed class RecordingPromptRecorder : IMeetingPromptRecordingController
|
||||
@@ -206,16 +389,37 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
|
||||
public int StopCount { get; private set; }
|
||||
|
||||
public int PromptStartCount { get; private set; }
|
||||
|
||||
public List<string> Commands { get; } = [];
|
||||
|
||||
public List<MeetingMetadata?> StartMetadata { get; } = [];
|
||||
|
||||
public List<MeetingMetadata> AttachedMetadata { get; } = [];
|
||||
|
||||
public Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return StartRecordingAsync(null);
|
||||
}
|
||||
|
||||
private Task<RecordingStatus> StartRecordingAsync(
|
||||
MeetingMetadata? metadata)
|
||||
{
|
||||
StartCount++;
|
||||
Commands.Add("start");
|
||||
StartMetadata.Add(metadata);
|
||||
CurrentStatus = Status(isRecording: true);
|
||||
return Task.FromResult(CurrentStatus);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> StartFromPromptAsync(
|
||||
MeetingMetadata? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
PromptStartCount++;
|
||||
return StartRecordingAsync(metadata);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
StopCount++;
|
||||
@@ -224,6 +428,15 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
||||
return Task.FromResult(CurrentStatus);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||
MeetingMetadata metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Commands.Add("attach-metadata");
|
||||
AttachedMetadata.Add(metadata);
|
||||
return Task.FromResult(CurrentStatus);
|
||||
}
|
||||
|
||||
private static RecordingStatus Status(bool isRecording)
|
||||
{
|
||||
return new RecordingStatus(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
@@ -41,6 +42,17 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
|
||||
Assert.Equal("ok", body?.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
|
||||
{
|
||||
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
|
||||
var result = await playback.QueueAsync(1, [1, 2]);
|
||||
|
||||
Assert.Contains("Windows build", result);
|
||||
Assert.DoesNotContain("Queued", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
|
||||
{
|
||||
|
||||
@@ -58,6 +58,64 @@ public sealed class LaunchProfileOptionsProviderTests
|
||||
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
|
||||
{
|
||||
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
|
||||
|
||||
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
|
||||
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
|
||||
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
|
||||
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
|
||||
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
|
||||
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
|
||||
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContradictorySpeakerSampleDurationsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
|
||||
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
|
||||
Assert.Contains("MaximumSampleDuration", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DuplicateProfileHotkeysAreRejected()
|
||||
{
|
||||
|
||||
@@ -1,103 +1,253 @@
|
||||
using MeetingAssistant.Summary;
|
||||
using Microsoft.Extensions.AI;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class LiteLlmResponsesChatClientTests
|
||||
{
|
||||
[Fact]
|
||||
public void ParserIgnoresReasoningItemsWithNullStatusAndReadsText()
|
||||
public async Task ClientAssemblesStreamedTextResponseWithMetadataAndUsage()
|
||||
{
|
||||
const string json = """
|
||||
var handler = new SequencedHttpMessageHandler(
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
"id": "resp_test",
|
||||
"created_at": 1779147100,
|
||||
"model": "gpt-5.5-2026-04-23",
|
||||
"output": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [],
|
||||
"status": null
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "OK"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
Content = new StringContent(
|
||||
"""
|
||||
data: {"type":"response.created","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"in_progress","store":false},"sequence_number":0}
|
||||
|
||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
||||
data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_stream","type":"message","status":"in_progress","content":[],"role":"assistant"},"sequence_number":1}
|
||||
|
||||
Assert.Equal("OK", response.Text);
|
||||
Assert.Equal("resp_test", response.ResponseId);
|
||||
Assert.Equal("gpt-5.5-2026-04-23", response.ModelId);
|
||||
data: {"type":"response.content_part.added","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":""},"sequence_number":2}
|
||||
|
||||
data: {"type":"response.output_text.delta","item_id":"msg_stream","output_index":0,"content_index":0,"delta":"Streamed OK","sequence_number":3}
|
||||
|
||||
data: {"type":"response.output_text.done","item_id":"msg_stream","output_index":0,"content_index":0,"text":"Streamed OK","sequence_number":4}
|
||||
|
||||
data: {"type":"response.content_part.done","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":"Streamed OK"},"sequence_number":5}
|
||||
|
||||
data: {"type":"response.output_item.done","output_index":0,"item":{"id":"msg_stream","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Streamed OK"}],"role":"assistant"},"sequence_number":6}
|
||||
|
||||
data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}},"sequence_number":7}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
""",
|
||||
Encoding.UTF8,
|
||||
"text/event-stream")
|
||||
});
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||
|
||||
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
|
||||
|
||||
Assert.Equal("Streamed OK", response.Text);
|
||||
Assert.Equal("resp_stream", response.ResponseId);
|
||||
Assert.Equal("gpt-5.5", response.ModelId);
|
||||
Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1779147100), response.CreatedAt);
|
||||
Assert.NotNull(response.Usage);
|
||||
Assert.Equal(12, response.Usage.InputTokenCount);
|
||||
Assert.Equal(3, response.Usage.OutputTokenCount);
|
||||
Assert.Equal(15, response.Usage.TotalTokenCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParserReadsFunctionCalls()
|
||||
public async Task ClientAssemblesStreamedFunctionCall()
|
||||
{
|
||||
const string json = """
|
||||
var handler = new SequencedHttpMessageHandler(
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "write_summary",
|
||||
"arguments": "{\"markdown\":\"# Summary\\nDone\"}",
|
||||
"status": "completed"
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
Content = new StringContent(
|
||||
"""
|
||||
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_stream","name":"write_summary","arguments":"{\"markdown\":\"# Summary\\nDone\"}","status":"completed"}}
|
||||
|
||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
||||
data: {"type":"response.completed","response":{"id":"resp_stream","model":"gpt-5.5","output":[]}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
""",
|
||||
Encoding.UTF8,
|
||||
"text/event-stream")
|
||||
});
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||
|
||||
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
|
||||
var call = Assert.IsType<FunctionCallContent>(Assert.Single(response.Messages[0].Contents));
|
||||
|
||||
Assert.Equal("call_1", call.CallId);
|
||||
Assert.Equal("call_stream", call.CallId);
|
||||
Assert.Equal("write_summary", call.Name);
|
||||
Assert.NotNull(call.Arguments);
|
||||
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]);
|
||||
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]?.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParserReadsUsage()
|
||||
public async Task ClientUsesNonStreamingResponsesWhenConfigured()
|
||||
{
|
||||
const string json = """
|
||||
{
|
||||
"usage": {
|
||||
"input_tokens": 123,
|
||||
"output_tokens": 45,
|
||||
"total_tokens": 168
|
||||
},
|
||||
"output": [
|
||||
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
"""
|
||||
{
|
||||
"type": "message",
|
||||
"content": [
|
||||
"id": "resp_nonstream",
|
||||
"created_at": 1779147100,
|
||||
"model": "gpt-5.5",
|
||||
"object": "response",
|
||||
"output": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "OK"
|
||||
"id": "msg_nonstream",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"annotations": [],
|
||||
"text": "Non-streamed OK"
|
||||
}
|
||||
],
|
||||
"role": "assistant"
|
||||
}
|
||||
]
|
||||
],
|
||||
"parallel_tool_calls": true,
|
||||
"status": "completed",
|
||||
"store": false,
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 16
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
""",
|
||||
Encoding.UTF8,
|
||||
"application/json")
|
||||
});
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 0, useStreaming: false);
|
||||
|
||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
||||
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
|
||||
|
||||
Assert.NotNull(response.Usage);
|
||||
Assert.Equal(123, response.Usage.InputTokenCount);
|
||||
Assert.Equal(45, response.Usage.OutputTokenCount);
|
||||
Assert.Equal(168, response.Usage.TotalTokenCount);
|
||||
Assert.Equal("Non-streamed OK", response.Text);
|
||||
Assert.Contains("\"stream\":false", Assert.Single(handler.RequestBodies));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClientSendsChatMessagesAsResponsesMessageItems()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||
|
||||
await client.GetResponseAsync(
|
||||
[
|
||||
new ChatMessage(ChatRole.User, "write summary"),
|
||||
new ChatMessage(ChatRole.Assistant, "I will inspect the meeting.")
|
||||
]);
|
||||
|
||||
using var request = JsonDocument.Parse(Assert.Single(handler.RequestBodies));
|
||||
var inputItems = request.RootElement.GetProperty("input").EnumerateArray().ToArray();
|
||||
Assert.Equal(2, inputItems.Length);
|
||||
Assert.All(inputItems, item => Assert.Equal("message", item.GetProperty("type").GetString()));
|
||||
Assert.Equal("user", inputItems[0].GetProperty("role").GetString());
|
||||
Assert.Equal(
|
||||
"input_text",
|
||||
Assert.Single(inputItems[0].GetProperty("content").EnumerateArray())
|
||||
.GetProperty("type")
|
||||
.GetString());
|
||||
Assert.Equal("assistant", inputItems[1].GetProperty("role").GetString());
|
||||
Assert.Equal(
|
||||
"output_text",
|
||||
Assert.Single(inputItems[1].GetProperty("content").EnumerateArray())
|
||||
.GetProperty("type")
|
||||
.GetString());
|
||||
Assert.DoesNotContain("\"type\":\"unknown\"", handler.RequestBodies[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AgentLoopReturnsMalformedFunctionArgumentsWithoutInvokingTool()
|
||||
{
|
||||
var responses = new Queue<HttpResponseMessage>(
|
||||
[
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
"""
|
||||
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_invalid","name":"write_summary","arguments":"{not-json","status":"completed"}}
|
||||
|
||||
data: {"type":"response.completed","response":{"id":"resp_invalid","model":"gpt-5.5","output":[]}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
""",
|
||||
Encoding.UTF8,
|
||||
"text/event-stream")
|
||||
},
|
||||
CreateStreamedTextResponse("Recovered.")
|
||||
]);
|
||||
var handler = new RecordingHttpMessageHandler(_ => responses.Dequeue());
|
||||
var toolInvoked = false;
|
||||
var tool = AIFunctionFactory.Create(
|
||||
(string markdown) =>
|
||||
{
|
||||
toolInvoked = true;
|
||||
return markdown;
|
||||
},
|
||||
"write_summary",
|
||||
"Writes a summary.");
|
||||
using var innerClient = CreateClient(handler, reconnectionAttempts: 0);
|
||||
using var functionClient = innerClient
|
||||
.AsBuilder()
|
||||
.UseFunctionInvocation(
|
||||
loggerFactory: null,
|
||||
client => client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync)
|
||||
.Build();
|
||||
|
||||
var response = await functionClient.GetResponseAsync(
|
||||
[new ChatMessage(ChatRole.User, "write summary")],
|
||||
new ChatOptions { Tools = [tool] });
|
||||
|
||||
Assert.False(toolInvoked);
|
||||
Assert.Equal("Recovered.", response.Text);
|
||||
Assert.Equal(2, handler.RequestBodies.Count);
|
||||
Assert.Contains("invalid_tool_arguments", handler.RequestBodies[1]);
|
||||
Assert.Contains("call_invalid", handler.RequestBodies[1]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClientReportsVisibleReasoningSummariesSeparatelyFromResponseText()
|
||||
{
|
||||
var handler = new SequencedHttpMessageHandler(
|
||||
CreateStreamedTextResponse(
|
||||
"Done.",
|
||||
"Checked the configured rules file.",
|
||||
"Prepared a targeted update."));
|
||||
var reasoningSummaries = new List<string>();
|
||||
using var client = CreateClient(
|
||||
handler,
|
||||
reconnectionAttempts: 0,
|
||||
reasoningSummaryChanged: reasoningSummaries.Add);
|
||||
|
||||
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
|
||||
|
||||
Assert.Equal("Done.", response.Text);
|
||||
Assert.Equal(
|
||||
[
|
||||
"Checked the configured rules file.",
|
||||
"Prepared a targeted update."
|
||||
],
|
||||
reasoningSummaries);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClientDoesNotFailResponseWhenReasoningSummaryCallbackFails()
|
||||
{
|
||||
var handler = new SequencedHttpMessageHandler(
|
||||
CreateStreamedTextResponse("Done.", "Checked the configured rules file."));
|
||||
using var client = CreateClient(
|
||||
handler,
|
||||
reconnectionAttempts: 0,
|
||||
reasoningSummaryChanged: _ => throw new InvalidOperationException("UI failed"));
|
||||
|
||||
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
|
||||
|
||||
Assert.Equal("Done.", response.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -108,24 +258,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
{
|
||||
Content = new StringContent("Internal Server Error")
|
||||
},
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Done."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""")
|
||||
});
|
||||
CreateStreamedTextResponse("Done."));
|
||||
var retryCount = 0;
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 1, retrying: () => retryCount++);
|
||||
|
||||
@@ -156,24 +289,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
[Fact]
|
||||
public async Task ClientSendsUserInitiatorOnceThenAgentInitiator()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Done."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""")
|
||||
});
|
||||
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
|
||||
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||
|
||||
await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
|
||||
@@ -205,24 +321,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
};
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Done."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""")
|
||||
};
|
||||
return CreateStreamedTextResponse("Done.");
|
||||
});
|
||||
using var client = CreateClient(
|
||||
handler,
|
||||
@@ -257,24 +356,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
};
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Done."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""")
|
||||
};
|
||||
return CreateStreamedTextResponse("Done.");
|
||||
});
|
||||
using var client = CreateClient(
|
||||
handler,
|
||||
@@ -306,7 +388,9 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
HttpMessageHandler handler,
|
||||
int reconnectionAttempts,
|
||||
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
||||
Action? retrying = null)
|
||||
Action? retrying = null,
|
||||
Action<string>? reasoningSummaryChanged = null,
|
||||
bool useStreaming = true)
|
||||
{
|
||||
return new LiteLlmResponsesChatClient(
|
||||
new HttpClient(handler)
|
||||
@@ -320,7 +404,50 @@ public sealed class LiteLlmResponsesChatClientTests
|
||||
reconnectionAttempts,
|
||||
TimeSpan.Zero,
|
||||
compactionOptions,
|
||||
retrying: retrying);
|
||||
retrying: retrying,
|
||||
reasoningSummaryChanged: reasoningSummaryChanged,
|
||||
useStreaming: useStreaming);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage CreateStreamedTextResponse(
|
||||
string text,
|
||||
params string[] reasoningSummaries)
|
||||
{
|
||||
var events = new List<string>();
|
||||
for (var index = 0; index < reasoningSummaries.Length; index++)
|
||||
{
|
||||
events.Add("data: " + JsonSerializer.Serialize(new
|
||||
{
|
||||
type = "response.reasoning_summary_text.delta",
|
||||
item_id = "reasoning_stream",
|
||||
output_index = 0,
|
||||
summary_index = index,
|
||||
delta = reasoningSummaries[index]
|
||||
}));
|
||||
}
|
||||
|
||||
var outputIndex = reasoningSummaries.Length > 0 ? 1 : 0;
|
||||
events.Add("data: " + JsonSerializer.Serialize(new
|
||||
{
|
||||
type = "response.output_text.delta",
|
||||
item_id = "msg_stream",
|
||||
output_index = outputIndex,
|
||||
content_index = 0,
|
||||
delta = text
|
||||
}));
|
||||
events.Add(
|
||||
"""data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}}}""");
|
||||
events.Add("data: [DONE]");
|
||||
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
string.Join($"{Environment.NewLine}{Environment.NewLine}", events)
|
||||
+ Environment.NewLine
|
||||
+ Environment.NewLine,
|
||||
Encoding.UTF8,
|
||||
"text/event-stream")
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class SequencedHttpMessageHandler : HttpMessageHandler
|
||||
|
||||
@@ -13,23 +13,50 @@ namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class LiteLlmScreenshotOcrClientTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExtractUsesAgentStreamingTransportByDefault()
|
||||
{
|
||||
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||
var handler = new RecordingHandler(
|
||||
CreateStreamedTextResponse("Streamed OCR text"),
|
||||
"text/event-stream");
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Endpoint = "https://summary.local",
|
||||
Model = "vision-model",
|
||||
Key = "agent-key",
|
||||
UseStreaming = true
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Streamed OCR text", result.Text);
|
||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||
Assert.True(payload.RootElement.GetProperty("stream").GetBoolean());
|
||||
var message = Assert.Single(payload.RootElement.GetProperty("input").EnumerateArray());
|
||||
Assert.Equal("message", message.GetProperty("type").GetString());
|
||||
var content = message.GetProperty("content").EnumerateArray().ToArray();
|
||||
Assert.Equal("input_text", content[0].GetProperty("type").GetString());
|
||||
Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
|
||||
Assert.Equal("input_image", content[1].GetProperty("type").GetString());
|
||||
Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]);
|
||||
var handler = new RecordingHandler("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"content": [
|
||||
{ "type": "output_text", "text": "Visible slide text" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""");
|
||||
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||
var handler = new RecordingHandler(CreateNonStreamingTextResponse("Visible slide text"));
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
@@ -39,7 +66,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
{
|
||||
Endpoint = "https://summary.local",
|
||||
Model = "summary-model",
|
||||
Key = "agent-key"
|
||||
Key = "agent-key",
|
||||
UseStreaming = false
|
||||
},
|
||||
Screenshots =
|
||||
{
|
||||
@@ -64,6 +92,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
Assert.Equal("Bearer", handler.Authorization?.Scheme);
|
||||
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
|
||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||
Assert.False(payload.RootElement.GetProperty("stream").GetBoolean());
|
||||
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
|
||||
var content = payload.RootElement
|
||||
.GetProperty("input")[0]
|
||||
@@ -75,10 +104,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
[Fact]
|
||||
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]);
|
||||
var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
|
||||
var screenshotPath = await CreateScreenshotAsync([4, 5, 6]);
|
||||
var handler = new RecordingHandler(CreateNonStreamingTextResponse("OCR result"));
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
@@ -88,7 +115,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
{
|
||||
Endpoint = "https://summary.local",
|
||||
Model = "summary-model",
|
||||
Key = "agent-key"
|
||||
Key = "agent-key",
|
||||
UseStreaming = false
|
||||
},
|
||||
Screenshots =
|
||||
{
|
||||
@@ -116,14 +144,15 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
[Fact]
|
||||
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6));
|
||||
var handler = new RecordingHandler("""
|
||||
{
|
||||
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
|
||||
}
|
||||
""");
|
||||
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
||||
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||
"""
|
||||
Slide text
|
||||
|
||||
```json
|
||||
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 } }
|
||||
```
|
||||
"""));
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
@@ -131,7 +160,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Key = "agent-key"
|
||||
Key = "agent-key",
|
||||
UseStreaming = false
|
||||
}
|
||||
};
|
||||
|
||||
@@ -150,13 +180,84 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText()
|
||||
{
|
||||
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||
"""
|
||||
Visible participant tiles: Ada and Grace.
|
||||
|
||||
```json
|
||||
{ "crop": null, "attendees": ["Ada Lovelace", "Grace Hopper"] }
|
||||
```
|
||||
"""));
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Key = "agent-key",
|
||||
UseStreaming = false
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Visible participant tiles: Ada and Grace.", result.Text);
|
||||
Assert.Equal(["Ada Lovelace", "Grace Hopper"], result.Attendees);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop()
|
||||
{
|
||||
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
||||
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||
"""
|
||||
Slide text
|
||||
|
||||
```json
|
||||
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 }, "attendees": "Ada" }
|
||||
```
|
||||
"""));
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Key = "agent-key",
|
||||
UseStreaming = false
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Slide text", result.Text);
|
||||
Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop);
|
||||
Assert.Empty(result.Attendees);
|
||||
}
|
||||
|
||||
private sealed class RecordingHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly string responseBody;
|
||||
private readonly string mediaType;
|
||||
|
||||
public RecordingHandler(string responseBody)
|
||||
public RecordingHandler(string responseBody, string mediaType = "application/json")
|
||||
{
|
||||
this.responseBody = responseBody;
|
||||
this.mediaType = mediaType;
|
||||
}
|
||||
|
||||
public Uri? RequestUri { get; private set; }
|
||||
@@ -176,7 +277,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
: await request.Content.ReadAsStringAsync(cancellationToken);
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
|
||||
Content = new StringContent(responseBody, Encoding.UTF8, mediaType)
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -193,6 +294,63 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
}
|
||||
|
||||
private static string CreateNonStreamingTextResponse(string text)
|
||||
{
|
||||
return JsonSerializer.Serialize(new
|
||||
{
|
||||
id = "resp_ocr",
|
||||
created_at = 1779147100,
|
||||
model = "vision-model",
|
||||
@object = "response",
|
||||
output = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
id = "msg_ocr",
|
||||
type = "message",
|
||||
status = "completed",
|
||||
content = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "output_text",
|
||||
annotations = Array.Empty<object>(),
|
||||
text
|
||||
}
|
||||
},
|
||||
role = "assistant"
|
||||
}
|
||||
},
|
||||
parallel_tool_calls = true,
|
||||
status = "completed",
|
||||
store = false
|
||||
});
|
||||
}
|
||||
|
||||
private static string CreateStreamedTextResponse(string text)
|
||||
{
|
||||
var delta = JsonSerializer.Serialize(new
|
||||
{
|
||||
type = "response.output_text.delta",
|
||||
item_id = "msg_ocr",
|
||||
output_index = 0,
|
||||
content_index = 0,
|
||||
delta = text
|
||||
});
|
||||
return
|
||||
$"data: {delta}{Environment.NewLine}{Environment.NewLine}" +
|
||||
"""data: {"type":"response.completed","response":{"id":"resp_ocr","created_at":1779147100,"model":"vision-model","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false}}""" +
|
||||
$"{Environment.NewLine}{Environment.NewLine}data: [DONE]{Environment.NewLine}{Environment.NewLine}";
|
||||
}
|
||||
|
||||
private static async Task<string> CreateScreenshotAsync(byte[] bytes)
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, bytes);
|
||||
return screenshotPath;
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning restore CA1416
|
||||
|
||||
@@ -9,10 +9,10 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.8" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.7.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -135,6 +135,72 @@ public sealed class MeetingNoteStoreTests
|
||||
Assert.Equal("User notes.", loaded.UserNotes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreEscapesApostrophePrefixedAttendeesBeforeWritingFrontmatter()
|
||||
{
|
||||
var (store, saved) = await SaveNoteAsync(
|
||||
title: "Escaped Attendees",
|
||||
attendees: ["'Ada Lovelace"],
|
||||
projects: [],
|
||||
userNotes: "Discuss attendee import.");
|
||||
var content = await File.ReadAllTextAsync(saved.Path);
|
||||
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||
|
||||
Assert.Contains("- \"'Ada Lovelace\"", content);
|
||||
Assert.Equal(["'Ada Lovelace"], loaded.Frontmatter.Attendees);
|
||||
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Ada # platform lead")]
|
||||
[InlineData("- Ada Lovelace")]
|
||||
[InlineData("? Ada Lovelace")]
|
||||
[InlineData("{Ada: Platform}")]
|
||||
[InlineData("*Ada")]
|
||||
[InlineData("&Ada")]
|
||||
[InlineData("!Ada")]
|
||||
[InlineData("| Ada")]
|
||||
[InlineData("> Ada")]
|
||||
[InlineData("@Ada")]
|
||||
[InlineData("`Ada")]
|
||||
[InlineData("true")]
|
||||
[InlineData("null")]
|
||||
[InlineData("2026-07-08")]
|
||||
[InlineData("Ada \"The Architect\" Lovelace")]
|
||||
[InlineData("C:\\People\\Ada")]
|
||||
public async Task StoreEscapesYamlSensitiveAttendeesBeforeWritingFrontmatter(string attendee)
|
||||
{
|
||||
var (store, saved) = await SaveNoteAsync(
|
||||
title: "Escaped Attendees",
|
||||
attendees: [attendee],
|
||||
projects: [],
|
||||
userNotes: "Discuss attendee import.");
|
||||
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||
|
||||
Assert.Equal([attendee], loaded.Frontmatter.Attendees);
|
||||
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Planning # Q3")]
|
||||
[InlineData("- Planning")]
|
||||
[InlineData("{Planning: Q3}")]
|
||||
[InlineData("true")]
|
||||
[InlineData("2026-07-08")]
|
||||
public async Task StoreEscapesYamlSensitiveScalarAndProjectFrontmatterValues(string value)
|
||||
{
|
||||
var (store, saved) = await SaveNoteAsync(
|
||||
title: value,
|
||||
attendees: [],
|
||||
projects: [value],
|
||||
userNotes: "Discuss YAML escaping.");
|
||||
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||
|
||||
Assert.Equal(value, loaded.Frontmatter.Title);
|
||||
Assert.Equal([value], loaded.Frontmatter.Projects);
|
||||
Assert.Equal("Discuss YAML escaping.", loaded.UserNotes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActionLinkEscapesSummaryFileName()
|
||||
{
|
||||
@@ -147,6 +213,26 @@ public sealed class MeetingNoteStoreTests
|
||||
link);
|
||||
}
|
||||
|
||||
private static async Task<(MarkdownMeetingNoteStore Store, MeetingNote Saved)> SaveNoteAsync(
|
||||
string title,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlyList<string> projects,
|
||||
string userNotes)
|
||||
{
|
||||
var (vaultRoot, store) = CreateStore();
|
||||
var note = MeetingNoteTemplate.Create(
|
||||
title: title,
|
||||
attendees: attendees,
|
||||
projects: projects,
|
||||
transcriptPath: Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"),
|
||||
assistantContextPath: Path.Combine(vaultRoot, "Meetings", "Assistant Context", "20260519-context.md"),
|
||||
summaryPath: Path.Combine(vaultRoot, "Meetings", "Summaries", "20260519-summary.md"),
|
||||
userNotes: userNotes);
|
||||
|
||||
var saved = await store.SaveAsync(note, CancellationToken.None);
|
||||
return (store, saved);
|
||||
}
|
||||
|
||||
private static (string VaultRoot, MarkdownMeetingNoteStore Store) CreateStore()
|
||||
{
|
||||
var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
|
||||
@@ -130,6 +133,194 @@ public sealed class MeetingScreenshotServiceTests
|
||||
Assert.Contains("Shared screen text", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(
|
||||
options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
},
|
||||
attendees: ["Ada Lovelace"]);
|
||||
var ocr = new CapturingScreenshotOcrClient(
|
||||
"Visible participant tiles: Ada, Grace, and Ada again.",
|
||||
attendees: ["Ada L.", "Grace Hopper", "Ada Lovelace"]);
|
||||
var canonicalizer = new MappingAttendeeCanonicalizer(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["Ada L."] = "Ada Lovelace"
|
||||
});
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr,
|
||||
canonicalizer);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
|
||||
Assert.Equal(["Ada Lovelace", "Grace Hopper"], meeting.Frontmatter.Attendees);
|
||||
Assert.Contains(canonicalizer.Requests, request =>
|
||||
request.SequenceEqual(["Ada Lovelace", "Ada L.", "Grace Hopper", "Ada Lovelace"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureTransformsOcrAttendeesBeforeWritingMeetingNote()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(
|
||||
options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
new CapturingScreenshotOcrClient(
|
||||
"Visible participant tile: Ada.",
|
||||
attendees: ["Ada Lovelace (Contoso)"]),
|
||||
meetingWorkflowEngine: workflowEngine);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
|
||||
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
|
||||
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureWritesRetryLinkWhenOcrFails()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Api.PublicBaseUrl = "http://localhost:5090";
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
new ThrowingScreenshotOcrClient("vision offline"));
|
||||
|
||||
var result = await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var screenshotId = ExtractScreenshotOcrId(context);
|
||||
Assert.Contains("_OCR failed: vision offline_", context);
|
||||
Assert.Contains("<!-- screenshot-ocr:", context);
|
||||
Assert.Contains("<!-- /screenshot-ocr:", context);
|
||||
Assert.Contains("[Retry screenshot OCR](http://localhost:5090/meetings/screenshot-ocr/retry?", context);
|
||||
Assert.Contains($"screenshotId={screenshotId}", context);
|
||||
Assert.Contains($"screenshotPath={Uri.EscapeDataString(result.ScreenshotPath)}", context);
|
||||
Assert.Contains($"assistantContextPath={Uri.EscapeDataString(fixture.Artifacts.AssistantContextPath)}", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RetryOcrReplacesFailureForSameScreenshot()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new SequencedScreenshotOcrClient(
|
||||
new InvalidOperationException("vision offline"),
|
||||
new ScreenshotOcrResult("Retried OCR text", null, ["Grace Hopper"]));
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
|
||||
var result = await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
var failedContext = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var screenshotId = ExtractScreenshotOcrId(failedContext);
|
||||
|
||||
var retry = await service.TriggerOcrRetryAsync(
|
||||
fixture.Artifacts,
|
||||
result.ScreenshotPath,
|
||||
screenshotId,
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
Assert.NotNull(retry);
|
||||
Assert.Equal(result.ScreenshotPath, retry.ScreenshotPath);
|
||||
Assert.Equal(screenshotId, retry.ScreenshotId);
|
||||
Assert.Equal([result.ScreenshotPath, result.ScreenshotPath], ocr.ScreenshotPaths);
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("### OCR", context);
|
||||
Assert.Contains("Retried OCR text", context);
|
||||
Assert.DoesNotContain("vision offline", context);
|
||||
Assert.DoesNotContain("Retry screenshot OCR", context);
|
||||
Assert.DoesNotContain("<!-- screenshot-ocr:", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProcessMeetingNoteImageEmbedsAppendsContextAndRunsOcrWithoutCropOrAttendees()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(
|
||||
options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
},
|
||||
attendees: ["Ada Lovelace"],
|
||||
userNotes: "Discussed ![[whiteboard.png]] and .");
|
||||
var noteFolder = Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!;
|
||||
var attachmentsFolder = Path.Combine(noteFolder, "attachments");
|
||||
Directory.CreateDirectory(attachmentsFolder);
|
||||
var whiteboardPath = Path.Combine(noteFolder, "whiteboard.png");
|
||||
var diagramPath = Path.Combine(attachmentsFolder, "diagram.png");
|
||||
await File.WriteAllBytesAsync(whiteboardPath, [1, 2, 3]);
|
||||
await File.WriteAllBytesAsync(diagramPath, [4, 5, 6]);
|
||||
var ocr = new CapturingScreenshotOcrClient(
|
||||
"Manual image OCR",
|
||||
new ScreenshotCropCoordinates(1, 1, 2, 2),
|
||||
["Grace Hopper"]);
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
var originalMeetingNote = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
|
||||
var result = await service.ProcessMeetingNoteImageEmbedsAsync(
|
||||
fixture.Artifacts,
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.QueuedCount);
|
||||
Assert.Equal(2, ocr.CallCount);
|
||||
Assert.Equal([whiteboardPath, diagramPath], ocr.ScreenshotPaths);
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("## Meeting Note Image", context);
|
||||
Assert.Contains("Image from meeting note.", context);
|
||||
Assert.Contains("Original embed: `![[whiteboard.png]]`", context);
|
||||
Assert.Contains("Original embed: ``", context);
|
||||
Assert.Contains("whiteboard.png", context);
|
||||
Assert.Contains("diagram.png", context);
|
||||
Assert.Contains("Manual image OCR", context);
|
||||
Assert.DoesNotContain("Cropped screenshot", context);
|
||||
Assert.Equal(originalMeetingNote, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
|
||||
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WaitForPendingOcrWaitsForRunningScreenshotOcr()
|
||||
{
|
||||
@@ -192,11 +383,13 @@ public sealed class MeetingScreenshotServiceTests
|
||||
private ScreenshotFixture(
|
||||
MeetingAssistantOptions options,
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MarkdownMeetingArtifactStore artifactStore)
|
||||
MarkdownMeetingArtifactStore artifactStore,
|
||||
MarkdownMeetingNoteStore noteStore)
|
||||
{
|
||||
Options = options;
|
||||
Artifacts = artifacts;
|
||||
ArtifactStore = artifactStore;
|
||||
NoteStore = noteStore;
|
||||
}
|
||||
|
||||
public MeetingAssistantOptions Options { get; }
|
||||
@@ -205,8 +398,12 @@ public sealed class MeetingScreenshotServiceTests
|
||||
|
||||
public MarkdownMeetingArtifactStore ArtifactStore { get; }
|
||||
|
||||
public MarkdownMeetingNoteStore NoteStore { get; }
|
||||
|
||||
public static async Task<ScreenshotFixture> CreateAsync(
|
||||
Action<MeetingAssistantOptions>? configure = null)
|
||||
Action<MeetingAssistantOptions>? configure = null,
|
||||
IReadOnlyList<string>? attendees = null,
|
||||
string userNotes = "")
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var options = new MeetingAssistantOptions
|
||||
@@ -218,6 +415,9 @@ public sealed class MeetingScreenshotServiceTests
|
||||
}
|
||||
};
|
||||
configure?.Invoke(options);
|
||||
var noteStore = new MarkdownMeetingNoteStore(
|
||||
Microsoft.Extensions.Options.Options.Create(options),
|
||||
NullLogger<MarkdownMeetingNoteStore>.Instance);
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
Path.Combine(root, "Notes", "meeting.md"),
|
||||
Path.Combine(root, "Transcripts", "transcript.md"),
|
||||
@@ -225,6 +425,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
Path.Combine(root, "Summaries", "summary.md"));
|
||||
var artifactStore = new MarkdownMeetingArtifactStore(
|
||||
NullLogger<MarkdownMeetingArtifactStore>.Instance);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
var meeting = MeetingNoteTemplate.Create(
|
||||
"Planning",
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
@@ -232,26 +433,37 @@ public sealed class MeetingScreenshotServiceTests
|
||||
assistantContextPath: artifacts.AssistantContextPath,
|
||||
summaryPath: artifacts.SummaryPath) with
|
||||
{
|
||||
Path = artifacts.MeetingNotePath
|
||||
Path = artifacts.MeetingNotePath,
|
||||
UserNotes = userNotes
|
||||
};
|
||||
meeting.Frontmatter.Attendees = attendees?.ToList() ?? [];
|
||||
var savedMeeting = await noteStore.SaveAsync(
|
||||
meeting,
|
||||
options,
|
||||
CancellationToken.None);
|
||||
await artifactStore.CreateAssistantContextAsync(
|
||||
artifacts,
|
||||
meeting,
|
||||
savedMeeting,
|
||||
"",
|
||||
null,
|
||||
CancellationToken.None);
|
||||
return new ScreenshotFixture(options, artifacts, artifactStore);
|
||||
return new ScreenshotFixture(options, artifacts, artifactStore, noteStore);
|
||||
}
|
||||
|
||||
public MeetingScreenshotService CreateService(
|
||||
IActiveWindowScreenshotCapture capture,
|
||||
IScreenshotOcrClient ocrClient)
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
{
|
||||
return new MeetingScreenshotService(
|
||||
capture,
|
||||
ArtifactStore,
|
||||
NoteStore,
|
||||
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
|
||||
ocrClient,
|
||||
NullLogger<MeetingScreenshotService>.Instance);
|
||||
NullLogger<MeetingScreenshotService>.Instance,
|
||||
meetingWorkflowEngine);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,15 +488,18 @@ public sealed class MeetingScreenshotServiceTests
|
||||
|
||||
public CapturingScreenshotOcrClient(
|
||||
string text = "",
|
||||
ScreenshotCropCoordinates? crop = null)
|
||||
ScreenshotCropCoordinates? crop = null,
|
||||
IReadOnlyList<string>? attendees = null)
|
||||
{
|
||||
result = new ScreenshotOcrResult(text, crop);
|
||||
result = new ScreenshotOcrResult(text, crop, attendees ?? []);
|
||||
}
|
||||
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public string? Prompt { get; private set; }
|
||||
|
||||
public List<string> ScreenshotPaths { get; } = [];
|
||||
|
||||
public Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
@@ -293,6 +508,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
{
|
||||
CallCount++;
|
||||
Prompt = prompt;
|
||||
ScreenshotPaths.Add(screenshotPath);
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
}
|
||||
@@ -320,7 +536,79 @@ public sealed class MeetingScreenshotServiceTests
|
||||
|
||||
public void Release(string value)
|
||||
{
|
||||
result.TrySetResult(new ScreenshotOcrResult(value, null));
|
||||
result.TrySetResult(new ScreenshotOcrResult(value, null, []));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ThrowingScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private readonly string message;
|
||||
|
||||
public ThrowingScreenshotOcrClient(string message)
|
||||
{
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new InvalidOperationException(message);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SequencedScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private readonly Queue<object> results;
|
||||
|
||||
public SequencedScreenshotOcrClient(params object[] results)
|
||||
{
|
||||
this.results = new Queue<object>(results);
|
||||
}
|
||||
|
||||
public List<string> ScreenshotPaths { get; } = [];
|
||||
|
||||
public Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ScreenshotPaths.Add(screenshotPath);
|
||||
var result = results.Dequeue();
|
||||
if (result is Exception exception)
|
||||
{
|
||||
throw exception;
|
||||
}
|
||||
|
||||
return Task.FromResult((ScreenshotOcrResult)result);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class MappingAttendeeCanonicalizer : ISpeakerIdentityAttendeeCanonicalizer
|
||||
{
|
||||
private readonly IReadOnlyDictionary<string, string> aliases;
|
||||
|
||||
public MappingAttendeeCanonicalizer(IReadOnlyDictionary<string, string> aliases)
|
||||
{
|
||||
this.aliases = aliases;
|
||||
}
|
||||
|
||||
public List<IReadOnlyList<string>> Requests { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<string>> CanonicalizeAsync(
|
||||
IReadOnlyList<string> attendees,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(attendees.ToList());
|
||||
var result = attendees
|
||||
.Select(attendee => aliases.TryGetValue(attendee, out var canonical) ? canonical : attendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return Task.FromResult<IReadOnlyList<string>>(result);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,6 +624,17 @@ public sealed class MeetingScreenshotServiceTests
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
}
|
||||
|
||||
private static string ExtractScreenshotOcrId(string context)
|
||||
{
|
||||
const string prefix = "<!-- screenshot-ocr:";
|
||||
var start = context.IndexOf(prefix, StringComparison.Ordinal);
|
||||
Assert.True(start >= 0);
|
||||
start += prefix.Length;
|
||||
var end = context.IndexOf(" -->", start, StringComparison.Ordinal);
|
||||
Assert.True(end > start);
|
||||
return context[start..end];
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning restore CA1416
|
||||
|
||||
@@ -7,6 +7,37 @@ namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MeetingSummaryInstructionBuilderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
var artifacts = CreateArtifacts(root);
|
||||
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
|
||||
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
|
||||
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
Agent = new AgentOptions { InitialPrompt = "Custom base." },
|
||||
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
|
||||
}));
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.StartsWith("Custom base.", instructions);
|
||||
Assert.Contains("# Alpha", instructions);
|
||||
Assert.Contains("Alpha Platform", instructions);
|
||||
Assert.Contains("Builds meeting automation", instructions);
|
||||
Assert.Contains("PROJECT.md", instructions);
|
||||
Assert.Contains("256", instructions);
|
||||
Assert.Contains("request_project_association", instructions);
|
||||
Assert.Contains("request_workflow_change", instructions);
|
||||
Assert.DoesNotContain("Internal notes", instructions);
|
||||
Assert.DoesNotContain("Do not expose", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
|
||||
{
|
||||
@@ -24,6 +55,7 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
Assert.Contains("You are the Meeting Assistant summary agent.", instructions);
|
||||
Assert.Contains("include only the most relevant cropped screenshots", instructions);
|
||||
Assert.Contains("Do not include every cropped screenshot", instructions);
|
||||
Assert.Contains("encode spaces in image-link targets as `%20`", instructions);
|
||||
Assert.Contains("Use add_attendee and remove_attendee", instructions);
|
||||
Assert.Contains("partial screenshot", instructions);
|
||||
Assert.Contains("override_speaker", instructions);
|
||||
@@ -39,6 +71,27 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
Assert.Contains("title parameter", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DefaultPromptTreatsAssistantContextAsMeetingMemoryForUncertainty()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var artifacts = CreateArtifacts(root);
|
||||
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, []);
|
||||
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
Agent = new AgentOptions { InitialPrompt = " " },
|
||||
Vault = new VaultOptions { ProjectsFolder = Path.Combine(root, "Projects") }
|
||||
}));
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.Contains("meeting-specific memory", instructions);
|
||||
Assert.Contains("unexpected problems", instructions);
|
||||
Assert.Contains("missing information", instructions);
|
||||
Assert.Contains("assumptions", instructions);
|
||||
Assert.Contains("write_context", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuilderUsesConfiguredPrompt()
|
||||
{
|
||||
@@ -53,7 +106,8 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.Equal("Custom summarizer instructions.", instructions);
|
||||
Assert.StartsWith("Custom summarizer instructions.", instructions);
|
||||
Assert.Contains("request_project_association", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -74,21 +128,12 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
var expected = """
|
||||
Base.
|
||||
|
||||
---
|
||||
projects:
|
||||
|
||||
# Alpha
|
||||
|
||||
Alpha instructions.
|
||||
|
||||
# Beta
|
||||
|
||||
Beta instructions.
|
||||
""";
|
||||
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
|
||||
Assert.StartsWith("Base.", instructions);
|
||||
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
|
||||
Assert.Contains("# Alpha", instructions);
|
||||
Assert.Contains("Alpha instructions.", instructions);
|
||||
Assert.Contains("# Beta", instructions);
|
||||
Assert.Contains("Beta instructions.", instructions);
|
||||
}
|
||||
|
||||
private static MeetingSessionArtifacts CreateArtifacts(string root)
|
||||
|
||||
@@ -189,7 +189,9 @@ public sealed class MeetingSummaryRetryRunnerTests
|
||||
return;
|
||||
}
|
||||
|
||||
await stateChanged.Task.WaitAsync(TimeSpan.FromMilliseconds(100));
|
||||
await Task.WhenAny(
|
||||
stateChanged.Task,
|
||||
Task.Delay(TimeSpan.FromMilliseconds(100)));
|
||||
}
|
||||
|
||||
throw new TimeoutException($"Expected {count} state changes.");
|
||||
|
||||
@@ -293,6 +293,44 @@ public sealed class MeetingSummaryToolTests
|
||||
Assert.Contains("transcript: \"[[../Transcripts/transcript|Transcript]]\"", meetingNote);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToolsTransformAddedAttendeeBeforeWritingMeetingNote()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"),
|
||||
TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"),
|
||||
AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"),
|
||||
SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(
|
||||
artifacts.MeetingNotePath,
|
||||
"""
|
||||
---
|
||||
title: Meeting
|
||||
attendees: []
|
||||
projects: []
|
||||
transcript: "[[../Transcripts/transcript|Transcript]]"
|
||||
assistant_context: "[[../Assistant Context/context|Assistant Context]]"
|
||||
summary: "[[../Summaries/summary|Summary]]"
|
||||
---
|
||||
|
||||
User note line.
|
||||
""");
|
||||
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
|
||||
var tools = new MeetingSummaryTools(
|
||||
artifacts,
|
||||
new MeetingAssistantOptions(),
|
||||
meetingWorkflowEngine: workflowEngine);
|
||||
|
||||
Assert.Equal("Added attendee Ada Lovelace.", await tools.AddAttendee("Ada Lovelace (Contoso)"));
|
||||
|
||||
var meetingNote = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
|
||||
Assert.Contains("- Ada Lovelace", meetingNote);
|
||||
Assert.DoesNotContain("Ada Lovelace (Contoso)", meetingNote);
|
||||
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToolsOverrideSpeakerInTranscriptAndRecordOverride()
|
||||
{
|
||||
@@ -799,4 +837,5 @@ public sealed class MeetingSummaryToolTests
|
||||
Assert.False(tools.SummaryWasWritten);
|
||||
Assert.False(File.Exists(artifacts.SummaryPath));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -310,6 +310,10 @@ public sealed class MeetingWorkflowEngineTests
|
||||
[InlineData("speaker_identified:\n name: ADA", "speaker", null, null, "ada", true)]
|
||||
[InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Grace", false)]
|
||||
[InlineData("speaker_identified: {}", "speaker", null, null, "Grace", true)]
|
||||
[InlineData("attendee_added:\n equals: Ada Lovelace", "attendee", null, null, "ada lovelace", true)]
|
||||
[InlineData("attendee_added:\n contains: contoso", "attendee", null, null, "Ada (Contoso)", true)]
|
||||
[InlineData("attendee_added:\n regex: '^Ada .+Contoso\\)$'", "attendee", null, null, "Ada (Contoso)", true)]
|
||||
[InlineData("attendee_added:\n regex: '^Grace'", "attendee", null, null, "Ada (Contoso)", false)]
|
||||
public async Task TriggerMatchingScenarios(
|
||||
string triggerYaml,
|
||||
string eventKind,
|
||||
@@ -318,6 +322,16 @@ public sealed class MeetingWorkflowEngineTests
|
||||
string? speaker,
|
||||
bool shouldRun)
|
||||
{
|
||||
var stepYaml = eventKind == "attendee"
|
||||
? """
|
||||
- uses: set_property
|
||||
property: attendee.name
|
||||
value: Triggered
|
||||
"""
|
||||
: """
|
||||
- uses: add_project
|
||||
value: Triggered
|
||||
""";
|
||||
var fixture = await WorkflowFixture.CreateAsync(
|
||||
$$"""
|
||||
rules:
|
||||
@@ -325,14 +339,21 @@ public sealed class MeetingWorkflowEngineTests
|
||||
on:
|
||||
- {{triggerYaml}}
|
||||
steps:
|
||||
- uses: add_project
|
||||
value: Triggered
|
||||
{{stepYaml}}
|
||||
""");
|
||||
|
||||
await fixture.Engine.RunAsync(
|
||||
CreateEvent(eventKind, fixture.Artifacts, from, to, speaker),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
var workflowEvent = CreateEvent(eventKind, fixture.Artifacts, from, to, speaker);
|
||||
if (eventKind == "attendee")
|
||||
{
|
||||
var transformed = await fixture.Engine.TransformAttendeeAsync(
|
||||
workflowEvent,
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
Assert.Equal(shouldRun ? "Triggered" : speaker, transformed);
|
||||
return;
|
||||
}
|
||||
|
||||
await fixture.Engine.RunAsync(workflowEvent, fixture.Options, CancellationToken.None);
|
||||
|
||||
var meeting = await fixture.ReadMeetingAsync();
|
||||
Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("Triggered"));
|
||||
@@ -603,6 +624,34 @@ public sealed class MeetingWorkflowEngineTests
|
||||
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AttendeeAddedRuleCanTransformWorkflowAddedAttendee()
|
||||
{
|
||||
var fixture = await WorkflowFixture.CreateAsync(
|
||||
"""
|
||||
rules:
|
||||
- name: add-raw-attendee
|
||||
on:
|
||||
- created: {}
|
||||
steps:
|
||||
- uses: add_attendee
|
||||
value: 'Ada Lovelace (Contoso)'
|
||||
- name: clean-contoso-attendee
|
||||
on:
|
||||
- attendee_added:
|
||||
contains: 'Contoso'
|
||||
steps:
|
||||
- uses: set_property
|
||||
property: attendee.name
|
||||
value: '@Model.Attendee.Name.Replace(" (Contoso)", "")'
|
||||
""");
|
||||
|
||||
await RunCreatedAsync(fixture);
|
||||
|
||||
var meeting = await fixture.ReadMeetingAsync();
|
||||
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddAttendeeDoesNotDuplicateExistingDisplayNameFromEmailAddress()
|
||||
{
|
||||
@@ -792,6 +841,7 @@ public sealed class MeetingWorkflowEngineTests
|
||||
{
|
||||
"created" => MeetingWorkflowEvent.Created(artifacts),
|
||||
"speaker" => MeetingWorkflowEvent.SpeakerIdentified(artifacts, speaker ?? "Ada"),
|
||||
"attendee" => MeetingWorkflowEvent.AttendeeAdded(artifacts, speaker ?? "Ada"),
|
||||
"state" => MeetingWorkflowEvent.StateTransition(
|
||||
artifacts,
|
||||
ParseState(from ?? "collecting metadata"),
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MicrophoneAudioSourceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CaptureMovesToNewlyResolvedMicrophoneWhenCurrentCaptureFails()
|
||||
{
|
||||
var captureSources = new SequenceMicrophoneCaptureSourceFactory(
|
||||
new FailingAfterChunkAudioSource(Pcm16(1_000)),
|
||||
new ActiveAudioSource(Pcm16(2_000)));
|
||||
var source = new MicrophoneAudioSource(
|
||||
captureSources,
|
||||
NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero);
|
||||
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source
|
||||
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.Equal(2, captureSources.CaptureCreationCount);
|
||||
|
||||
await cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureWaitsForMicrophoneToBecomeAvailable()
|
||||
{
|
||||
var captureSources = new InitiallyUnavailableMicrophoneCaptureSourceFactory(
|
||||
new ActiveAudioSource(Pcm16(3_000)));
|
||||
var source = new MicrophoneAudioSource(
|
||||
captureSources,
|
||||
NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero);
|
||||
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source
|
||||
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||
.GetAsyncEnumerator(cancellation.Token);
|
||||
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(3_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.Equal(2, captureSources.CaptureCreationCount);
|
||||
|
||||
await cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
var devices = new SelectableCaptureDevices(selection);
|
||||
var prompts = new AcceptingPrompt();
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero, monitor, selection);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||
try
|
||||
{
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
|
||||
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
|
||||
var nextChunk = chunks.MoveNextAsync().AsTask();
|
||||
Assert.False(nextChunk.IsCompleted);
|
||||
Assert.Null(selection.SelectedDeviceId);
|
||||
|
||||
prompts.Response.SetResult(true);
|
||||
|
||||
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.Equal("new", selection.SelectedDeviceId);
|
||||
Assert.False(lifetime.IsCancellationRequested);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
|
||||
{
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var abandonedCapture = new DisposableAudioSource();
|
||||
var factory = new CallbackCaptureFactory(() =>
|
||||
{
|
||||
lifetime.Cancel();
|
||||
return abandonedCapture;
|
||||
});
|
||||
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
|
||||
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
|
||||
|
||||
Assert.False(await chunks.MoveNextAsync());
|
||||
Assert.True(abandonedCapture.Disposed);
|
||||
}
|
||||
|
||||
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
var devices = new SwitchDuringCreationDevices(selection);
|
||||
var prompts = new AcceptingPrompt();
|
||||
prompts.Response.SetResult(true);
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero, monitor, selection);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||
try
|
||||
{
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal("new", selection.SelectedDeviceId);
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.True(devices.AbandonedCapture.Disposed);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
|
||||
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
private MicrophoneDevice[] available = [new("old", "Built-in")];
|
||||
private int creations;
|
||||
private volatile bool switchObserved;
|
||||
public DisposableAudioSource AbandonedCapture { get; } = new();
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||
{
|
||||
if (selection.SelectedDeviceId == "new") switchObserved = true;
|
||||
return Volatile.Read(ref available);
|
||||
}
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(GetAvailableMicrophones(), available[0]);
|
||||
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
{
|
||||
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
|
||||
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
|
||||
// Let the acceptance callback finish before returning the already-opened source.
|
||||
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
|
||||
throw new TimeoutException("The new microphone was not selected during capture creation.");
|
||||
return AbandonedCapture;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
|
||||
{
|
||||
public bool Disposed { get; private set; }
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
|
||||
public void Dispose() => Disposed = true;
|
||||
}
|
||||
|
||||
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
Shown.TrySetResult(microphone);
|
||||
return Response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
|
||||
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
|
||||
}
|
||||
|
||||
private static byte[] Pcm16(short sample)
|
||||
{
|
||||
return BitConverter.GetBytes(sample);
|
||||
}
|
||||
|
||||
private sealed class SequenceMicrophoneCaptureSourceFactory(params IMeetingAudioSource[] sources)
|
||||
: IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
private readonly Queue<IMeetingAudioSource> sources = new(sources);
|
||||
|
||||
public int CaptureCreationCount { get; private set; }
|
||||
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
{
|
||||
CaptureCreationCount++;
|
||||
return sources.Dequeue();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InitiallyUnavailableMicrophoneCaptureSourceFactory(IMeetingAudioSource availableSource)
|
||||
: IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
public int CaptureCreationCount { get; private set; }
|
||||
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
{
|
||||
CaptureCreationCount++;
|
||||
if (CaptureCreationCount == 1)
|
||||
{
|
||||
throw new InvalidOperationException("No microphone is currently available.");
|
||||
}
|
||||
|
||||
return availableSource;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FailingAfterChunkAudioSource(byte[] pcm) : IMeetingAudioSource
|
||||
{
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Yield();
|
||||
yield return new AudioChunk(pcm, 16000, 1);
|
||||
throw new InvalidOperationException("The active microphone was disconnected.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ActiveAudioSource(byte[] pcm) : IMeetingAudioSource
|
||||
{
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
yield return new AudioChunk(pcm, 16000, 1);
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MicrophoneSelectionTests
|
||||
{
|
||||
[Fact]
|
||||
public void BlankConfiguredMicrophoneUsesDefaultDevice()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
|
||||
var selected = selection.Resolve(
|
||||
configuredDeviceId: null,
|
||||
new MicrophoneDevice("default-id", "default microphone"),
|
||||
[
|
||||
new MicrophoneDevice("default-id", "default microphone"),
|
||||
new MicrophoneDevice("other-id", "other microphone")
|
||||
]);
|
||||
|
||||
Assert.Equal("default-id", selected?.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConfiguredMicrophoneUsesMatchingDevice()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
var selected = selection.Resolve(
|
||||
"other-id",
|
||||
new MicrophoneDevice("default-id", "default microphone"),
|
||||
[
|
||||
new MicrophoneDevice("default-id", "default microphone"),
|
||||
new MicrophoneDevice("other-id", "other microphone")
|
||||
]);
|
||||
|
||||
Assert.Equal("other-id", selected?.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RuntimeMicrophoneSelectionOverridesConfiguredDevice()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
selection.Select("runtime-id");
|
||||
|
||||
var selected = selection.Resolve(
|
||||
"configured-id",
|
||||
new MicrophoneDevice("default-id", "default microphone"),
|
||||
[
|
||||
new MicrophoneDevice("configured-id", "configured microphone"),
|
||||
new MicrophoneDevice("runtime-id", "runtime microphone")
|
||||
]);
|
||||
|
||||
Assert.Equal("runtime-id", selected?.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnavailableDefaultMicrophoneFallsBackToAnotherActiveDevice()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
|
||||
var selected = selection.Resolve(
|
||||
configuredDeviceId: null,
|
||||
new MicrophoneDevice("disconnected-id", "disconnected microphone"),
|
||||
[new MicrophoneDevice("backup-id", "backup microphone")]);
|
||||
|
||||
Assert.Equal("backup-id", selected?.Id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MicrophoneSwitchMonitorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
|
||||
{
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var devices = new MutableDevices();
|
||||
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var switched = false;
|
||||
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||
devices.Available = [new("usb", "USB microphone")];
|
||||
|
||||
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(prompts.WasShown);
|
||||
Assert.False(switched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
|
||||
{
|
||||
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
|
||||
var newDevice = new MicrophoneDevice("new", "USB headset");
|
||||
var devices = new MutableDevices { Available = [oldDevice] };
|
||||
var prompts = new ControlledPrompts();
|
||||
var monitor = CreateMonitor(devices, prompts);
|
||||
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
|
||||
try
|
||||
{
|
||||
devices.Available = [oldDevice, newDevice];
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Equal(newDevice, prompt.Device);
|
||||
Assert.False(switched.Task.IsCompleted);
|
||||
await devices.WaitForPollsAsync();
|
||||
Assert.False(prompts.HasMore);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
await devices.WaitForPollsAsync();
|
||||
Assert.False(prompts.HasMore);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("declined")]
|
||||
[InlineData("removed")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("ended")]
|
||||
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
|
||||
{
|
||||
var device = new MicrophoneDevice("usb", "USB microphone");
|
||||
var devices = new MutableDevices();
|
||||
var prompts = new ControlledPrompts();
|
||||
var monitor = CreateMonitor(devices, prompts,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
|
||||
var switched = false;
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||
try
|
||||
{
|
||||
devices.Available = [device];
|
||||
var prompt = await prompts.NextAsync();
|
||||
if (outcome == "declined")
|
||||
{
|
||||
prompt.Response.SetResult(false);
|
||||
await devices.WaitForPollsAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
if (outcome == "removed") devices.Available = [];
|
||||
if (outcome == "ended") await lifetime.CancelAsync();
|
||||
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
|
||||
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
}
|
||||
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
Assert.False(switched);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
|
||||
TimeSpan? promptLifetime = null) => new(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
|
||||
promptLifetime ?? TimeSpan.FromMinutes(1));
|
||||
|
||||
private sealed class MutableDevices : IMicrophoneDeviceProvider
|
||||
{
|
||||
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
|
||||
public volatile MicrophoneDevice[] Available = [];
|
||||
public Action? BeforeRead { get; set; }
|
||||
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||
{
|
||||
BeforeRead?.Invoke();
|
||||
polls.Writer.TryWrite(true);
|
||||
return Available;
|
||||
}
|
||||
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(Available, Available.FirstOrDefault());
|
||||
|
||||
public async Task WaitForPollsAsync()
|
||||
{
|
||||
while (polls.Reader.TryRead(out _)) { }
|
||||
for (var i = 0; i < 3; i++)
|
||||
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public bool WasShown { get; private set; }
|
||||
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
WasShown = true;
|
||||
beforeAcceptance();
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
|
||||
{
|
||||
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
|
||||
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
var prompt = new Prompt(microphone, cancellationToken);
|
||||
requests.Writer.TryWrite(prompt);
|
||||
return await prompt.Response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
|
||||
{
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -25,4 +25,16 @@ public sealed class NotificationActivationArgumentsTests
|
||||
Assert.Equal("abc123", arguments["promptId"]);
|
||||
Assert.Equal("continue", arguments["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
|
||||
{
|
||||
Assert.Contains(
|
||||
MeetingInactivityPromptActions.All,
|
||||
action => action.Content == "Pause transcription" &&
|
||||
action.ResponseArgument == "pause");
|
||||
Assert.Equal(
|
||||
MeetingInactivityPromptResponse.Pause,
|
||||
MeetingInactivityPromptActions.ParseResponse("pause"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
|
||||
[Fact]
|
||||
public void ExtractAgendaStopsBeforeTeamsJoinInformation()
|
||||
{
|
||||
var agenda = OutlookClassicMeetingMetadataProvider.ExtractAgenda(
|
||||
var agenda = OutlookClassicAppointmentMetadata.ExtractAgenda(
|
||||
"""
|
||||
Review current prototype
|
||||
Decide next backend
|
||||
@@ -25,7 +25,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
|
||||
[Fact]
|
||||
public void NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail()
|
||||
{
|
||||
var attendees = OutlookClassicMeetingMetadataProvider.NormalizeAttendees([
|
||||
var attendees = OutlookClassicAppointmentMetadata.NormalizeAttendees([
|
||||
"Marcus.Altmann@sew-eurodrive.de",
|
||||
"Marcus.Altmann@sew-eurodrive.de <Marcus.Altmann@sew-eurodrive.de>",
|
||||
"Schweigert, Manuel",
|
||||
@@ -37,5 +37,14 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
|
||||
"Schweigert, Manuel"
|
||||
], attendees);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Canceled: Project sync")]
|
||||
[InlineData("Cancelled: Project sync")]
|
||||
[InlineData("Abgesagt: Project sync")]
|
||||
public void SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(string subject)
|
||||
{
|
||||
Assert.True(OutlookClassicCom.SubjectIndicatesCancellation(subject));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -40,7 +40,7 @@ public sealed class OutlookMeetingCandidateSelectorTests
|
||||
{
|
||||
var now = new DateTime(2026, 5, 20, 10, 0, 0);
|
||||
var endingOverlap = new Candidate(now.AddMinutes(-25), now.AddMinutes(2));
|
||||
var upcoming = new Candidate(now.AddMinutes(5), now.AddMinutes(35));
|
||||
var upcoming = new Candidate(now.AddMinutes(1), now.AddMinutes(31));
|
||||
|
||||
var selected = OutlookMeetingCandidateSelector.Select(
|
||||
[endingOverlap, upcoming],
|
||||
|
||||
@@ -1,10 +1,90 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ProjectKnowledgeToolTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("name: Incomplete project")]
|
||||
[InlineData("description: Missing name")]
|
||||
[InlineData("name: ' '\ndescription: Missing name")]
|
||||
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
|
||||
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
|
||||
using var catalog = JsonDocument.Parse(await tools.ListProjects());
|
||||
|
||||
var project = Assert.Single(catalog.RootElement.EnumerateArray());
|
||||
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
|
||||
Assert.Equal("", project.GetProperty("description").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
|
||||
var tools = new MeetingSummaryTools(artifacts,
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
|
||||
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
|
||||
|
||||
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||
$"description: {new string('b', 257)}", from: 3, to: 3);
|
||||
|
||||
Assert.StartsWith("Refused:", rejected);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
|
||||
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||
"description: Updated project purpose", from: 3, to: 3));
|
||||
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
|
||||
Assert.Contains("owner: Ada", updated);
|
||||
Assert.Contains("User-authored project notes.", updated);
|
||||
Assert.Contains("Updated project purpose", await tools.ListProjects());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
|
||||
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
|
||||
|
||||
var catalog = await tools.ListProjects();
|
||||
|
||||
using var json = JsonDocument.Parse(catalog);
|
||||
var projects = json.RootElement.EnumerateArray().ToArray();
|
||||
Assert.Equal(3, projects.Length);
|
||||
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
|
||||
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
|
||||
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
|
||||
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
|
||||
Assert.Equal("", projects[1].GetProperty("description").GetString());
|
||||
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
|
||||
Assert.Equal("", projects[2].GetProperty("description").GetString());
|
||||
Assert.DoesNotContain("Private", catalog);
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
|
||||
{
|
||||
@@ -48,7 +128,7 @@ public sealed class ProjectKnowledgeToolTests
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Contains("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
|
||||
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
|
||||
|
||||
@@ -56,15 +136,15 @@ public sealed class ProjectKnowledgeToolTests
|
||||
|
||||
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
|
||||
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
|
||||
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
||||
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
||||
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
||||
Assert.Equal(
|
||||
"README.md:2 Second alpha line",
|
||||
await tools.Search("alpha"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
|
||||
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
@@ -88,7 +168,8 @@ public sealed class ProjectKnowledgeToolTests
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
|
||||
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -100,8 +181,11 @@ public sealed class ProjectKnowledgeToolTests
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
var projectFile = Path.Combine(projectRoot, "notes.md");
|
||||
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||
var tools = new MeetingSummaryTools(
|
||||
CreateArtifacts(root),
|
||||
artifacts,
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
Vault =
|
||||
@@ -132,8 +216,13 @@ public sealed class ProjectKnowledgeToolTests
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||
var outsidePath = Path.Combine(projectsRoot, "outside.md");
|
||||
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
|
||||
var tools = new MeetingSummaryTools(
|
||||
CreateArtifacts(root),
|
||||
artifacts,
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
Vault =
|
||||
@@ -144,6 +233,7 @@ public sealed class ProjectKnowledgeToolTests
|
||||
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
|
||||
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
|
||||
|
||||
@@ -18,31 +18,8 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote-validation:local",
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
@@ -54,23 +31,109 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
Assert.True(commandRunner.RunCancellationWasObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
|
||||
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
[
|
||||
new LaunchProfile("english", namedOptions),
|
||||
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
|
||||
]),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await commandRunner.WaitForRunAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
|
||||
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(options),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(50));
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
private static MeetingAssistantOptions CreateValidationOptions(string image)
|
||||
{
|
||||
return new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
DockerCommand = "docker",
|
||||
Image = image,
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly IReadOnlyList<LaunchProfile> profiles;
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
|
||||
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
|
||||
{
|
||||
this.profiles = profiles;
|
||||
}
|
||||
|
||||
public LaunchProfile GetRequiredProfile(string? name)
|
||||
{
|
||||
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
|
||||
var profileName = string.IsNullOrWhiteSpace(name)
|
||||
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
|
||||
: name;
|
||||
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfile> GetProfiles()
|
||||
{
|
||||
return [GetRequiredProfile(null)];
|
||||
return profiles;
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
||||
|
||||
@@ -2,6 +2,7 @@ using MeetingAssistant;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
@@ -57,17 +58,49 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner(
|
||||
"""
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
|
||||
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
|
||||
""");
|
||||
var modelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models");
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
|
||||
})
|
||||
.Build();
|
||||
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
|
||||
var validator = CreateValidator(commandRunner, configuredOptions);
|
||||
|
||||
var valid = await validator.ValidateSampleAsync(
|
||||
CreateWav(TimeSpan.FromSeconds(20)),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(valid);
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
bool enabled = true)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
return CreateValidator(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
@@ -76,9 +109,8 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Enabled = enabled,
|
||||
MinimumSingleSpeakerCoverage = 0.90,
|
||||
MinimumMatchingKnownSnippetRatio = 1,
|
||||
Diarization = new PyannoteDiarizationOptions
|
||||
Diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BuildImage = false,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote:local",
|
||||
@@ -94,7 +126,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
MeetingAssistantOptions configuredOptions)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(configuredOptions),
|
||||
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
|
||||
}
|
||||
|
||||
|
||||
@@ -156,9 +156,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var explicitDiarization = new PyannoteDiarizationOptions
|
||||
var explicitDiarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-azure:local",
|
||||
@@ -169,7 +168,7 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
};
|
||||
|
||||
await finalizer.FinalizeAsync(
|
||||
await finalizer.FinalizeEnabledAsync(
|
||||
audioPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
|
||||
explicitDiarization,
|
||||
@@ -186,9 +185,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
|
||||
var diarization = new PyannoteDiarizationOptions
|
||||
var diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-warmup:local",
|
||||
@@ -218,9 +216,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: null);
|
||||
|
||||
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
|
||||
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
Token = "",
|
||||
TokenEnv = "",
|
||||
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,596 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(7, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Single(fixture.Encoder.Requests);
|
||||
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Sabrina", saved.CanonicalName);
|
||||
Assert.Equal(3, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(8, saved.VoiceVectors.Count);
|
||||
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options =>
|
||||
{
|
||||
options.OutlierPruningMinimumVectors = 20;
|
||||
options.OutlierPruningNeighborSimilarity = 0.90;
|
||||
options.OutlierPruningMinimumNeighbors = 3;
|
||||
options.OutlierPruningMinimumClusterRatio = 0.60;
|
||||
});
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(16, saved.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
saved.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
"Chris",
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
var currentRunVectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var liveResult = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
fixture.Encoder.Vectors = currentRunVectors;
|
||||
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
|
||||
{
|
||||
KnownSpeakerMappings = liveResult.SpeakerMappings
|
||||
};
|
||||
if (transcriptAlreadyRelabeled)
|
||||
{
|
||||
finalRequest = finalRequest with { Segments = liveResult.Segments };
|
||||
}
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(10, saved.VoiceVectors.Count);
|
||||
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors =
|
||||
[
|
||||
UnitVector(0),
|
||||
UnitVector(0, 2, 0.01f),
|
||||
UnitVector(0, 3, 0.02f),
|
||||
UnitVector(0, 4, 0.03f),
|
||||
UnitVector(0, 5, 0.04f)
|
||||
];
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
|
||||
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(6, saved.VoiceVectors.Count);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyFinishedSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments[0].Speaker);
|
||||
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
|
||||
Assert.All(
|
||||
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal("John", saved.CanonicalName);
|
||||
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
|
||||
Assert.Equal("John", result.Segments.Single().Speaker);
|
||||
Assert.All(saved.References, reference =>
|
||||
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Manuel", saved.CanonicalName);
|
||||
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(
|
||||
configureSpeaker: options => options.MaxMatchCandidates = 1);
|
||||
await fixture.AddIdentityAsync("Legacy WAV identity", []);
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
|
||||
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(target.Id);
|
||||
Assert.Single(saved.Snippets);
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
|
||||
Assert.Single(await context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private sealed class Fixture : IAsyncDisposable
|
||||
{
|
||||
private readonly string directory;
|
||||
private readonly string databasePath;
|
||||
private readonly SpeakerIdentificationOptions speakerOptions;
|
||||
|
||||
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
|
||||
{
|
||||
this.directory = directory;
|
||||
this.databasePath = databasePath;
|
||||
this.speakerOptions = speakerOptions;
|
||||
}
|
||||
|
||||
public FakeEncoder Encoder { get; } = new();
|
||||
|
||||
public FakeSnippetExtractor SnippetExtractor { get; } = new();
|
||||
|
||||
public string DatabasePath => databasePath;
|
||||
|
||||
public static async Task<Fixture> CreateAsync(
|
||||
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
|
||||
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
|
||||
{
|
||||
var directory = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
var databasePath = Path.Combine(directory, "speaker-identities.db");
|
||||
var speakerOptions = new SpeakerIdentificationOptions
|
||||
{
|
||||
DatabasePath = databasePath,
|
||||
MatchBatchSize = 6,
|
||||
MaxMatchCandidates = 100,
|
||||
MatchIdentityActiveAge = TimeSpan.FromDays(365),
|
||||
MinimumSampleSpeechDuration = TimeSpan.Zero,
|
||||
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
}
|
||||
};
|
||||
configure?.Invoke(speakerOptions.Resemblyzer);
|
||||
configureSpeaker?.Invoke(speakerOptions);
|
||||
await using var context = new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
return new Fixture(directory, databasePath, speakerOptions);
|
||||
}
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService CreateService()
|
||||
{
|
||||
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
|
||||
return new ResemblyzerSpeakerIdentificationService(
|
||||
new TestDbContextFactory(databasePath),
|
||||
SnippetExtractor,
|
||||
Encoder,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(appOptions),
|
||||
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
|
||||
}
|
||||
|
||||
public SpeakerIdentificationRequest CreateRequest(
|
||||
string speaker,
|
||||
int sampleCount,
|
||||
IReadOnlyList<string>? attendees = null,
|
||||
int segmentCount = 1)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, "transcript.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
var segments = Enumerable.Range(0, segmentCount)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 30),
|
||||
TimeSpan.FromSeconds((index + 1) * 30),
|
||||
speaker,
|
||||
"enough useful words for a speaker sample"))
|
||||
.ToList();
|
||||
var segment = segments[0];
|
||||
return new SpeakerIdentificationRequest(
|
||||
Path.Combine(directory, "meeting.wav"),
|
||||
new MeetingNote(
|
||||
Path.Combine(directory, "meeting.md"),
|
||||
new MeetingNoteFrontmatter
|
||||
{
|
||||
Title = "Test",
|
||||
Attendees = attendees?.ToList() ?? ["Chris"],
|
||||
Transcript = transcriptPath,
|
||||
AssistantContext = Path.Combine(directory, "context.md"),
|
||||
Summary = Path.Combine(directory, "summary.md")
|
||||
},
|
||||
""),
|
||||
segments,
|
||||
Enumerable.Range(0, sampleCount)
|
||||
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
|
||||
.ToList());
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> AddIdentityAsync(
|
||||
string? name,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
IReadOnlyList<string>? candidates = null,
|
||||
IReadOnlyList<string>? aliases = null)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
|
||||
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = speakerOptions.Resemblyzer.ModelId,
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"known-{index}",
|
||||
CreatedAt = now.AddMinutes(index)
|
||||
}).ToList()
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
return identity;
|
||||
}
|
||||
|
||||
public async Task AddCurrentRunCandidateAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string candidateName)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
|
||||
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "older-model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(UnitVector(1)),
|
||||
Fingerprint = "older-model-vector",
|
||||
CreatedAt = now
|
||||
}
|
||||
],
|
||||
References =
|
||||
[
|
||||
SpeakerIdentityReferences.Create(
|
||||
request.MeetingNote.Path,
|
||||
request.MeetingNote.Frontmatter.Transcript,
|
||||
now)
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync(identity => identity.Id == id);
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public IReadOnlyList<float[]> Vectors { get; set; } = [];
|
||||
|
||||
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
|
||||
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
|
||||
{
|
||||
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
|
||||
|
||||
public Task<byte[]> ExtractSnippetAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(speakerSegments.ToList());
|
||||
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
var databasePath = Path.Combine(directory, "identities.db");
|
||||
var factory = new TestDbContextFactory(databasePath);
|
||||
await using (var context = factory.CreateDbContext())
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris",
|
||||
DateTimeOffset.UtcNow.AddMonths(-2),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 20, 0.02f))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris duplicate",
|
||||
DateTimeOffset.UtcNow.AddDays(-1),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 40, 0.02f)))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Decoy",
|
||||
DateTimeOffset.UtcNow.AddMonths(-1),
|
||||
[UnitVector(0)],
|
||||
directory));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
};
|
||||
var service = new ResemblyzerSpeakerIdentityMergeService(
|
||||
factory,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
MergeRecentIdentityAge = TimeSpan.FromDays(14),
|
||||
MaxMatchCandidates = 1,
|
||||
MaxSnippetsPerSpeaker = 3,
|
||||
Resemblyzer = resemblyzerOptions
|
||||
}
|
||||
}),
|
||||
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
|
||||
|
||||
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.MatchAttempts);
|
||||
Assert.Equal(1, result.MergedPairs);
|
||||
await using var verification = factory.CreateDbContext();
|
||||
var saved = await verification.SpeakerIdentities
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.SingleAsync(identity => identity.CanonicalName == "Chris");
|
||||
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
|
||||
Assert.Equal("Chris", saved.CanonicalName);
|
||||
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
|
||||
Assert.Equal(20, saved.VoiceVectors.Count);
|
||||
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static SpeakerIdentity CreateIdentity(
|
||||
string name,
|
||||
DateTimeOffset createdAt,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
string directory)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
return new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = createdAt,
|
||||
UpdatedAt = createdAt,
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"{name}-{index}",
|
||||
CreatedAt = createdAt.AddMinutes(index)
|
||||
}).ToList(),
|
||||
References =
|
||||
[
|
||||
new SpeakerIdentityReference
|
||||
{
|
||||
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
|
||||
TranscriptPath = transcriptPath,
|
||||
CreatedAt = createdAt
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primary] = 1;
|
||||
if (secondary is { } index)
|
||||
{
|
||||
vector[index] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static float[] ToVector(byte[] bytes)
|
||||
{
|
||||
var vector = new float[bytes.Length / sizeof(float)];
|
||||
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
|
||||
return vector;
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcherTests
|
||||
{
|
||||
[Fact]
|
||||
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
|
||||
{
|
||||
var matcher = CreateMatcher();
|
||||
var query = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
|
||||
.ToList();
|
||||
var candidates = new[]
|
||||
{
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
|
||||
};
|
||||
|
||||
var result = matcher.Match(query, candidates);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
Assert.True(result.Cohesion >= 0.99);
|
||||
Assert.True(result.BestSimilarity >= 0.99);
|
||||
Assert.True(result.RunnerUpSimilarity < 0.1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FourVectorsDoNotTriggerAutomaticMatching()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 4).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("4/5", result.Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("cohesion", result.Reason);
|
||||
Assert.Null(result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarityBelowThresholdIsRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("best similarity", result.Reason);
|
||||
Assert.Equal(0, result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarCandidatesWithinMarginAreRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
|
||||
]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("margin", result.Reason);
|
||||
Assert.NotNull(result.RunnerUpSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
|
||||
]);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
|
||||
{
|
||||
return new ResemblyzerVoiceClusterMatcher(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
|
||||
{
|
||||
[Fact]
|
||||
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(16, identity.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
identity.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AmbiguousDenseClustersPreserveAllVectors()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(20, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 15)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PruningPreservesOtherModelsAndMalformedStoredRows()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
[UnitVector(2, 60, 0.04f)],
|
||||
"older-model",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = [1],
|
||||
Fingerprint = "malformed",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "zero-magnitude",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
|
||||
{
|
||||
return new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
OutlierPruningMinimumVectors = 20,
|
||||
OutlierPruningNeighborSimilarity = 0.90,
|
||||
OutlierPruningMinimumNeighbors = 3,
|
||||
OutlierPruningMinimumClusterRatio = 0.60
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||
}
|
||||
|
||||
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var identity = new SpeakerIdentity();
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
"resemblyzer-0.1.4-pretrained",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
return identity;
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int secondaryDimension,
|
||||
float secondaryValue)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
vector[secondaryDimension] = secondaryValue;
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(options),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await encoder.WaitForWarmupAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(encoder.CancellationObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisabledWarmupDoesNotInvokeEncoder()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.False(encoder.WarmupStarted);
|
||||
}
|
||||
|
||||
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool WarmupStarted { get; private set; }
|
||||
|
||||
public bool CancellationObserved { get; private set; }
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
WarmupStarted = true;
|
||||
started.TrySetResult();
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
CancellationObserved = true;
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitForWarmupAsync()
|
||||
{
|
||||
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,5 +9,6 @@ public sealed class ScreenshotOcrOptionsTests
|
||||
{
|
||||
Assert.Contains("exactly who is in the meeting", ScreenshotOcrOptions.DefaultPrompt);
|
||||
Assert.Contains("partial", ScreenshotOcrOptions.DefaultPrompt);
|
||||
Assert.Contains("\"attendees\"", ScreenshotOcrOptions.DefaultPrompt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,22 @@ namespace MeetingAssistant.Tests;
|
||||
public sealed class SpeakerAudioSampleCollectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
|
||||
public void DefaultSampleDurationsAreTenAndSixtySeconds()
|
||||
{
|
||||
var options = new SpeakerIdentificationOptions();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
|
||||
@@ -37,8 +46,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
|
||||
|
||||
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
|
||||
@@ -48,6 +57,61 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
|
||||
|
||||
var sample = Assert.Single(collector.Snapshot());
|
||||
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
|
||||
|
||||
Assert.Empty(collector.Snapshot());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
|
||||
|
||||
var sample = collector.TryAdd(Segment(
|
||||
0,
|
||||
70,
|
||||
"Guest01",
|
||||
"one two three four five six seven eight nine ten."));
|
||||
|
||||
Assert.NotNull(sample);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
|
||||
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
|
||||
{
|
||||
@@ -55,8 +119,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1),
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
logger: logger);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
@@ -69,6 +133,47 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.Contains("minimum duration", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
|
||||
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
|
||||
|
||||
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
|
||||
|
||||
Assert.Collection(
|
||||
samples,
|
||||
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
|
||||
|
||||
Assert.Single(collector.Snapshot());
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(
|
||||
double start,
|
||||
double end,
|
||||
|
||||
@@ -107,6 +107,9 @@ public sealed class SpeakerIdentityMergeServiceTests
|
||||
return new SpeakerIdentityMergeService(
|
||||
new TestSpeakerIdentityDbContextFactory(dbPath),
|
||||
Matcher,
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions(),
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerIdentityMergerTests
|
||||
{
|
||||
[Fact]
|
||||
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var target = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("oldest", 1, now.AddMinutes(-4)),
|
||||
Vector("duplicate", 2, now.AddMinutes(-3))
|
||||
]
|
||||
};
|
||||
var source = new SpeakerIdentity
|
||||
{
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("duplicate", 2, now.AddMinutes(-2)),
|
||||
Vector("newer", 3, now.AddMinutes(-1)),
|
||||
Vector("newest", 4, now)
|
||||
]
|
||||
};
|
||||
|
||||
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
|
||||
|
||||
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
|
||||
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerRecognitionFeatureSelectionTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
|
||||
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
|
||||
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
|
||||
bool enabled,
|
||||
Type expectedIdentificationType,
|
||||
Type expectedMergeType,
|
||||
int expectedMinimumSamples,
|
||||
bool expectedNonOverlappingSamples)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
await using var factory = new WebApplicationFactory<Program>()
|
||||
.WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
|
||||
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
|
||||
});
|
||||
});
|
||||
builder.ConfigureTestServices(services =>
|
||||
{
|
||||
services.RemoveAll<IResemblyzerVoiceEncoder>();
|
||||
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
|
||||
});
|
||||
});
|
||||
|
||||
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
|
||||
|
||||
Assert.IsType(expectedIdentificationType, service);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
|
||||
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
|
||||
Assert.IsType(expectedMergeType, mergeService);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
|
||||
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
|
||||
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
|
||||
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerSampleSpanSelectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13),
|
||||
Segment(20, 70)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Equal(3, span.Count);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Empty(span);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
|
||||
{
|
||||
var segments = Enumerable.Range(0, 6)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 10),
|
||||
TimeSpan.FromSeconds((index + 1) * 10),
|
||||
"Guest01",
|
||||
$"segment {index} has enough words"))
|
||||
.ToList();
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
segments,
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(5, spans.Count);
|
||||
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
|
||||
Assert.All(
|
||||
spans.Zip(spans.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 10),
|
||||
Segment(9, 19),
|
||||
Segment(20, 30)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(2, spans.Count);
|
||||
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Empty(spans);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[Segment(0, 70)],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(double start, double end)
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(start),
|
||||
TimeSpan.FromSeconds(end),
|
||||
"Guest01",
|
||||
"enough words for this segment");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerVoiceVectorPersistenceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-a",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
var identity = await context.SpeakerIdentities
|
||||
.Include(candidate => candidate.VoiceVectors)
|
||||
.SingleAsync();
|
||||
var vector = Assert.Single(identity.VoiceVectors);
|
||||
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
|
||||
Assert.Equal(256, vector.Dimensions);
|
||||
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
|
||||
|
||||
context.SpeakerIdentities.Remove(identity);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
await using var context = new SpeakerIdentityDbContext(options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
context.SpeakerVoiceVectors.AddRange(
|
||||
CreateVector(identity.Id, "same-vector"),
|
||||
CreateVector(identity.Id, "same-vector"));
|
||||
|
||||
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identityId,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Notifications;
|
||||
using MeetingAssistant.Summary;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SummaryAgentRequestTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("denied")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("shutdown")]
|
||||
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
|
||||
{
|
||||
var prompts = new ControlledPrompts();
|
||||
var editor = new RecordingEditor();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
await requests.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
|
||||
var prompt = await prompts.NextAsync();
|
||||
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
}
|
||||
finally { await requests.StopAsync(CancellationToken.None); }
|
||||
Assert.Empty(editor.Prompts);
|
||||
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InvalidRequestsAreRefusedBeforeNotification()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
|
||||
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
|
||||
Assert.False(prompts.HasMore);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var prompts = new ControlledPrompts();
|
||||
var editor = new RecordingEditor();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
await requests.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
|
||||
|
||||
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
|
||||
|
||||
Assert.StartsWith("Requested", result);
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
|
||||
Assert.Empty(editor.Prompts);
|
||||
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
|
||||
Assert.True(tools.SummaryWasWritten);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
Assert.False(prompt.Response.TrySetResult(true));
|
||||
Assert.Single(editor.Prompts);
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
finally { await requests.StopAsync(CancellationToken.None); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("denied")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("canceled")]
|
||||
[InlineData("removed")]
|
||||
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
|
||||
var prompt = await prompts.NextAsync();
|
||||
|
||||
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||
if (outcome == "canceled") await cancellation.CancelAsync();
|
||||
if (outcome == "removed")
|
||||
{
|
||||
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
|
||||
prompt.Response.SetResult(true);
|
||||
}
|
||||
if (outcome == "canceled")
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
|
||||
else
|
||||
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
|
||||
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
|
||||
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
|
||||
Assert.False(request.IsCompleted);
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
|
||||
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
|
||||
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.StartsWith("Approved", result);
|
||||
Assert.Contains("Alpha instructions", result);
|
||||
Assert.Contains("Alpha Platform", result);
|
||||
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
Assert.Contains("Edited by user", note);
|
||||
Assert.Contains("custom: keep-me", note);
|
||||
Assert.Contains("User's latest notes.", note);
|
||||
Assert.Contains("Existing", note);
|
||||
Assert.Contains("Alpha", note);
|
||||
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
|
||||
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
|
||||
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
|
||||
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
|
||||
Assert.False(prompts.HasMore);
|
||||
}
|
||||
|
||||
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var project = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(project);
|
||||
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
|
||||
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
|
||||
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
|
||||
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
|
||||
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
}
|
||||
|
||||
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
|
||||
{
|
||||
public List<string> Prompts { get; } = [];
|
||||
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Show() { }
|
||||
public void ShowWithPrompt(string initialPrompt)
|
||||
{
|
||||
Prompts.Add(initialPrompt);
|
||||
Started.TrySetResult(initialPrompt);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledPrompts : IAgentApprovalPromptService
|
||||
{
|
||||
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
|
||||
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new PendingPrompt(prompt, cancellationToken);
|
||||
requests.Writer.TryWrite(request);
|
||||
try { return await request.Response.Task.WaitAsync(cancellationToken); }
|
||||
finally { request.Removed.TrySetResult(); }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
|
||||
{
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -13,46 +13,112 @@ public sealed class TaskbarIconTests
|
||||
{
|
||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(),
|
||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")]);
|
||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
|
||||
[new MicrophoneDevice("integrated", "integrated microphone")],
|
||||
"integrated");
|
||||
|
||||
Assert.Equal(RecordingProcessState.Idle, menu.State);
|
||||
Assert.Contains(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.EditRules &&
|
||||
item.Text == "Settings and logs");
|
||||
Assert.Contains(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.StartRecording &&
|
||||
item.ProfileName == "default" &&
|
||||
item.Text == "Start meeting recording (default)\tCtrl+Alt+M");
|
||||
Assert.Contains(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.StartRecording &&
|
||||
item.ProfileName == "english" &&
|
||||
item.Text == "Start meeting recording (english)\tCtrl+Alt+L");
|
||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
|
||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
|
||||
AssertMenuLayout(
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
|
||||
("Start meeting recording (default)\tCtrl+Alt+M", MeetingTaskbarAction.StartRecording, false),
|
||||
("Start meeting recording (english)\tCtrl+Alt+L", MeetingTaskbarAction.StartRecording, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
Assert.Equal(
|
||||
["default", "english"],
|
||||
menu.Items
|
||||
.Where(item => item.Action == MeetingTaskbarAction.StartRecording)
|
||||
.Select(item => item.ProfileName));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordingMenuOffersStopAbortAndOtherProfileSwitches()
|
||||
public void MenuOffersMicrophoneSubmenuWithEffectiveDeviceChecked()
|
||||
{
|
||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(),
|
||||
[Profile("default")],
|
||||
[
|
||||
new MicrophoneDevice("integrated", "integrated microphone"),
|
||||
new MicrophoneDevice("other", "other microphone")
|
||||
],
|
||||
"integrated");
|
||||
|
||||
var microphoneMenu = Assert.Single(menu.Items, item => item.Text == "Microphone");
|
||||
|
||||
Assert.Equal(MeetingTaskbarAction.OpenSubmenu, microphoneMenu.Action);
|
||||
Assert.NotNull(microphoneMenu.Items);
|
||||
Assert.Contains(microphoneMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.SelectMicrophone &&
|
||||
item.MicrophoneDeviceId == "integrated" &&
|
||||
item.Text == "integrated microphone" &&
|
||||
item.IsChecked);
|
||||
Assert.Contains(microphoneMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.SelectMicrophone &&
|
||||
item.MicrophoneDeviceId == "other" &&
|
||||
item.Text == "other microphone" &&
|
||||
!item.IsChecked);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordingMenuPrioritizesFinishMeetingInDedicatedSection()
|
||||
{
|
||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L"), Profile("french", "Ctrl+Alt+F")]);
|
||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
|
||||
[new MicrophoneDevice("integrated", "integrated microphone")],
|
||||
"integrated");
|
||||
|
||||
Assert.Equal(RecordingProcessState.Recording, menu.State);
|
||||
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
|
||||
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
|
||||
Assert.Contains(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
||||
item.ProfileName == "english" &&
|
||||
item.Text == "Switch to english\tCtrl+Alt+L");
|
||||
Assert.Contains(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
||||
item.ProfileName == "french" &&
|
||||
item.Text == "Switch to french\tCtrl+Alt+F");
|
||||
Assert.DoesNotContain(menu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
||||
item.ProfileName == "default");
|
||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StartRecording);
|
||||
AssertMenuLayout(
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
Assert.Equal(
|
||||
"english",
|
||||
Assert.Single(menu.Items, item => item.Action == MeetingTaskbarAction.SwitchProfile).ProfileName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordingMenuKeepsFinishMeetingIsolatedWithoutMicrophones()
|
||||
{
|
||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||
[Profile("default")]);
|
||||
|
||||
AssertMenuLayout(
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
|
||||
{
|
||||
var runningMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||
[Profile("default")]);
|
||||
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
|
||||
[Profile("default")]);
|
||||
|
||||
Assert.Contains(runningMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.PauseTranscription &&
|
||||
item.Text == "Pause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
|
||||
item.Text == "Unpause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.StopRecording &&
|
||||
item.Text == "Finish meeting");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -108,7 +174,7 @@ public sealed class TaskbarIconTests
|
||||
{
|
||||
Assert.Equal(
|
||||
requiresConfirmation,
|
||||
MeetingTaskbarExitPolicy.RequiresConfirmation(Status(state: state)));
|
||||
MeetingTaskbarExitPolicy.RequiresConfirmation(state));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -145,10 +211,20 @@ public sealed class TaskbarIconTests
|
||||
});
|
||||
}
|
||||
|
||||
private static void AssertMenuLayout(
|
||||
MeetingTaskbarMenu menu,
|
||||
params (string Text, MeetingTaskbarAction Action, bool StartsSection)[] expected)
|
||||
{
|
||||
Assert.Equal(
|
||||
expected,
|
||||
menu.Items.Select(item => (item.Text, item.Action, item.StartsSection)));
|
||||
}
|
||||
|
||||
private static RecordingStatus Status(
|
||||
bool isRecording = false,
|
||||
RecordingProcessState state = RecordingProcessState.Idle,
|
||||
string? profile = null)
|
||||
string? profile = null,
|
||||
bool isPaused = false)
|
||||
{
|
||||
return new RecordingStatus(
|
||||
isRecording,
|
||||
@@ -157,7 +233,8 @@ public sealed class TaskbarIconTests
|
||||
state == RecordingProcessState.Idle ? null : "context.md",
|
||||
state == RecordingProcessState.Idle ? null : "summary.md",
|
||||
state,
|
||||
profile);
|
||||
profile,
|
||||
isPaused);
|
||||
}
|
||||
|
||||
[SupportedOSPlatform("windows")]
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
internal sealed class TransformingAttendeeWorkflowEngine : IMeetingWorkflowEngine
|
||||
{
|
||||
private readonly string source;
|
||||
private readonly string replacement;
|
||||
|
||||
public TransformingAttendeeWorkflowEngine(string source, string replacement)
|
||||
{
|
||||
this.source = source;
|
||||
this.replacement = replacement;
|
||||
}
|
||||
|
||||
public List<string> AttendeeRequests { get; } = [];
|
||||
|
||||
public Task RunAsync(
|
||||
MeetingWorkflowEvent workflowEvent,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<string> TransformTranscriptLineAsync(
|
||||
MeetingWorkflowEvent workflowEvent,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(workflowEvent.TranscriptLineText ?? "");
|
||||
}
|
||||
|
||||
public Task<string> TransformAttendeeAsync(
|
||||
MeetingWorkflowEvent workflowEvent,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
AttendeeRequests.Add(workflowEvent.AttendeeName ?? "");
|
||||
return Task.FromResult(string.Equals(workflowEvent.AttendeeName, source, StringComparison.OrdinalIgnoreCase)
|
||||
? replacement
|
||||
: workflowEvent.AttendeeName ?? "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class VenvResemblyzerVoiceEncoderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
|
||||
{
|
||||
using var fixture = new EncoderFixture();
|
||||
|
||||
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName == "python-test"
|
||||
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("torch==2.14.0+cpu")
|
||||
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("Resemblyzer==0.1.4")
|
||||
&& command.Arguments.Contains("--no-deps"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
warmupExitCode: 21,
|
||||
warmupError: "incompatible environment");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
|
||||
|
||||
Assert.Empty(Directory.EnumerateFiles(
|
||||
fixture.RuntimeFolder,
|
||||
".ready",
|
||||
SearchOption.AllDirectories));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
|
||||
{
|
||||
var first = new float[256];
|
||||
first[0] = 2;
|
||||
var second = new float[256];
|
||||
second[1] = 3;
|
||||
using var fixture = new EncoderFixture(VectorOutput([first, second]));
|
||||
|
||||
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, vectors.Count);
|
||||
Assert.Equal(1f, vectors[0][0], 5);
|
||||
Assert.Equal(1f, vectors[1][1], 5);
|
||||
var encodingCommand = Assert.Single(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Any(
|
||||
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
|
||||
Assert.EndsWith(
|
||||
Path.Combine("Scripts", "python.exe"),
|
||||
encodingCommand.FileName,
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal(2, encodingCommand.Arguments.Count);
|
||||
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongVectorDimension()
|
||||
{
|
||||
var vector = new float[255];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("255 dimensions", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsMalformedVectorJson()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsNonFiniteVector()
|
||||
{
|
||||
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
|
||||
using var fixture = new EncoderFixture(
|
||||
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongResultCount()
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
|
||||
{
|
||||
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("zero magnitude", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncReportsFailedLocalCommand()
|
||||
{
|
||||
using var fixture = new EncoderFixture("", 17, "runtime failed");
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("exit code 17", error.Message);
|
||||
Assert.Contains("runtime failed", error.Message);
|
||||
}
|
||||
|
||||
private static string VectorOutput(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
return $"""
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
|
||||
{JsonSerializer.Serialize(vectors)}
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
|
||||
""";
|
||||
}
|
||||
|
||||
private sealed class EncoderFixture : IDisposable
|
||||
{
|
||||
public EncoderFixture(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
RuntimeFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"resemblyzer");
|
||||
Runner = new CapturingCommandRunner(
|
||||
encodingOutput,
|
||||
encodingExitCode,
|
||||
encodingError,
|
||||
warmupExitCode,
|
||||
warmupError);
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
|
||||
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
|
||||
Encoder = new VenvResemblyzerVoiceEncoder(
|
||||
Runner,
|
||||
Options.Create(options),
|
||||
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
|
||||
}
|
||||
|
||||
public CapturingCommandRunner Runner { get; }
|
||||
|
||||
public VenvResemblyzerVoiceEncoder Encoder { get; }
|
||||
|
||||
public string RuntimeFolder { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(RuntimeFolder))
|
||||
{
|
||||
Directory.Delete(RuntimeFolder, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly string encodingOutput;
|
||||
private readonly int encodingExitCode;
|
||||
private readonly string encodingError;
|
||||
private readonly int warmupExitCode;
|
||||
private readonly string warmupError;
|
||||
|
||||
public CapturingCommandRunner(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
this.encodingOutput = encodingOutput;
|
||||
this.encodingExitCode = encodingExitCode;
|
||||
this.encodingError = encodingError;
|
||||
this.warmupExitCode = warmupExitCode;
|
||||
this.warmupError = warmupError;
|
||||
}
|
||||
|
||||
public List<CapturedCommand> Commands { get; } = [];
|
||||
|
||||
public Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
|
||||
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
|
||||
if (isEncoding)
|
||||
{
|
||||
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
|
||||
}
|
||||
|
||||
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
|
||||
return Task.FromResult(isWarmup
|
||||
? new CommandResult(warmupExitCode, string.Empty, warmupError)
|
||||
: new CommandResult(0, string.Empty, string.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
Key = "summary-key",
|
||||
KeyEnv = "SUMMARY_KEY",
|
||||
Model = "summary-model",
|
||||
UseStreaming = true,
|
||||
EnableThinking = true,
|
||||
ReasoningEffort = ReasoningEffortOption.High,
|
||||
ReconnectionAttempts = 7,
|
||||
@@ -33,6 +34,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
var editor = new WorkflowRulesEditorOptions
|
||||
{
|
||||
Model = "editor-model",
|
||||
UseStreaming = false,
|
||||
EnableThinking = false,
|
||||
MaxOutputTokens = 50
|
||||
};
|
||||
@@ -43,6 +45,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Equal("summary-key", effective.Key);
|
||||
Assert.Equal("SUMMARY_KEY", effective.KeyEnv);
|
||||
Assert.Equal("editor-model", effective.Model);
|
||||
Assert.False(effective.UseStreaming);
|
||||
Assert.False(effective.EnableThinking);
|
||||
Assert.Equal(ReasoningEffortOption.High, effective.ReasoningEffort);
|
||||
Assert.Equal(7, effective.ReconnectionAttempts);
|
||||
@@ -442,7 +445,8 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
Assert.Equal("summary body", await File.ReadAllTextAsync(Path.Combine(summariesRoot, "daily-summary.md")));
|
||||
Assert.Equal("one\ntwo\nthree", await File.ReadAllTextAsync(Path.Combine(transcriptsRoot, "daily-transcript.md")));
|
||||
var note = await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md"));
|
||||
var note = (await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md")))
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
Assert.Contains("title: Fixed", note);
|
||||
Assert.Contains("projects:\n- Alpha", note);
|
||||
Assert.Contains("Existing body.", note);
|
||||
@@ -636,6 +640,28 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsRefuseSideEffectingAttendeeAddedRules()
|
||||
{
|
||||
var fixture = await CreateRulesEditorFixtureAsync();
|
||||
|
||||
var result = await fixture.Tools.WriteRules("""
|
||||
rules:
|
||||
- name: attendee-side-effect
|
||||
on:
|
||||
- attendee_added:
|
||||
contains: Contoso
|
||||
steps:
|
||||
- uses: add_context
|
||||
value: '@Model.Attendee.Name'
|
||||
""", replace_file: true);
|
||||
|
||||
Assert.StartsWith("Refused: workflow rules are invalid.", result);
|
||||
Assert.Contains("attendee-side-effect", result);
|
||||
Assert.Contains("attendee.name", result);
|
||||
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsAcceptAndParseMaskedProfanityRedactionRule()
|
||||
{
|
||||
@@ -721,12 +747,45 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Contains("read_logs", instructions);
|
||||
Assert.Contains("read_spec_file", instructions);
|
||||
Assert.Contains("list_recent_summaries", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("When correcting a meeting note that has project references", instructions);
|
||||
Assert.Contains("read that project's AGENTS.md", instructions);
|
||||
Assert.Contains("new project reference", instructions);
|
||||
Assert.Contains("matching correction", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
WorkflowRulesEditor = new WorkflowRulesEditorOptions
|
||||
{
|
||||
InitialPrompt = "Custom interactive agent instructions."
|
||||
}
|
||||
};
|
||||
|
||||
var instructions = await builder.BuildAsync(options, CancellationToken.None);
|
||||
|
||||
Assert.Contains("Custom interactive agent instructions.", instructions);
|
||||
Assert.Contains("meeting-specific memory", instructions);
|
||||
Assert.Contains("fix or investigate a meeting or summary", instructions);
|
||||
Assert.Contains("read the matching assistant context", instructions);
|
||||
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
|
||||
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsCrudAndSearchSpeakerIdentities()
|
||||
{
|
||||
@@ -750,6 +809,32 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
|
||||
{
|
||||
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
|
||||
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identity.Id,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-1",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
await fixture.Context.SaveChangesAsync();
|
||||
var tools = fixture.CreateTools();
|
||||
|
||||
var searchResult = await tools.SearchIdentities("Sabrina");
|
||||
var readResult = await tools.ReadIdentity(identity.Id);
|
||||
|
||||
Assert.Contains("\"sampleCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", readResult);
|
||||
Assert.DoesNotContain("vector-1", readResult);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
|
||||
{
|
||||
@@ -758,7 +843,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
|
||||
|
||||
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
|
||||
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
@@ -846,15 +931,37 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Equal("", viewModel.Draft);
|
||||
Assert.Equal(
|
||||
[new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "Rename ABS Daily")],
|
||||
viewModel.Messages.ToArray());
|
||||
viewModel.Messages.Select(item => item.Message).OfType<WorkflowRulesEditorChatMessage>().ToArray());
|
||||
|
||||
pipeline.Release.SetResult();
|
||||
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.False(viewModel.IsThinking);
|
||||
Assert.Equal(2, viewModel.Messages.Count);
|
||||
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[1].Role);
|
||||
Assert.Equal("Updated rules.", viewModel.Messages[1].Content);
|
||||
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
|
||||
{
|
||||
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
|
||||
{
|
||||
Draft = "My unfinished draft"
|
||||
};
|
||||
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
|
||||
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
|
||||
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
|
||||
|
||||
var task = requested.SendPromptAsync(prompt);
|
||||
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(requested.IsThinking);
|
||||
Assert.Empty(pipeline.LastConversation);
|
||||
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
|
||||
Assert.Equal("My unfinished draft", existing.Draft);
|
||||
Assert.Empty(existing.Messages);
|
||||
pipeline.Release.SetResult();
|
||||
await task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
Assert.False(requested.IsThinking);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -870,20 +977,19 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
Assert.False(viewModel.IsThinking);
|
||||
Assert.Equal("", viewModel.Draft);
|
||||
Assert.Equal(
|
||||
[
|
||||
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"),
|
||||
new WorkflowRulesEditorChatMessage(
|
||||
WorkflowRulesEditorChatRole.Agent,
|
||||
"Settings and logs failed: The request timed out.")
|
||||
],
|
||||
viewModel.Messages.ToArray());
|
||||
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"), viewModel.Messages[0].Message);
|
||||
AssertCompletedActivity(
|
||||
viewModel,
|
||||
["Thinking..."],
|
||||
"Meeting Summary Agent failed: The request timed out.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelShowsReconnectStatusReportedByPipeline()
|
||||
{
|
||||
var pipeline = new StatusReportingRulesEditorPipeline("Updated rules.");
|
||||
var pipeline = new ReportingRulesEditorPipeline(
|
||||
"Updated rules.",
|
||||
WorkflowRulesEditorActivityUpdate.Status("Reconnecting..."));
|
||||
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
|
||||
{
|
||||
Draft = "check logs"
|
||||
@@ -900,7 +1006,146 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
Assert.False(viewModel.IsThinking);
|
||||
Assert.Equal("Thinking...", viewModel.ActivityMessage);
|
||||
Assert.Equal("Updated rules.", viewModel.Messages[1].Content);
|
||||
AssertCompletedActivity(viewModel, ["Thinking...", "Reconnecting..."], "Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelShowsToolCallsAbovePersistentThinkingLine()
|
||||
{
|
||||
var pipeline = new ReportingRulesEditorPipeline(
|
||||
"Updated rules.",
|
||||
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"));
|
||||
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
|
||||
{
|
||||
Draft = "check logs"
|
||||
};
|
||||
|
||||
var sendTask = viewModel.SendAsync();
|
||||
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(viewModel.IsThinking);
|
||||
Assert.Equal("Thinking...", viewModel.ActivityMessage);
|
||||
Assert.Equal(["Called tool: read_logs"], viewModel.ActivityMessages.ToArray());
|
||||
|
||||
pipeline.Release.SetResult();
|
||||
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.False(viewModel.IsThinking);
|
||||
Assert.Empty(viewModel.ActivityMessages);
|
||||
AssertCompletedActivity(viewModel, ["Thinking...", "Called tool: read_logs"], "Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelShowsVisibleThinkingOutputInActivityExpander()
|
||||
{
|
||||
var pipeline = new ReportingRulesEditorPipeline(
|
||||
"Updated rules.",
|
||||
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."),
|
||||
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"),
|
||||
WorkflowRulesEditorActivityUpdate.Thinking("Matched the rule to the user request."));
|
||||
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
|
||||
{
|
||||
Draft = "check logs"
|
||||
};
|
||||
|
||||
var sendTask = viewModel.SendAsync();
|
||||
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(viewModel.IsThinking);
|
||||
Assert.Equal("Thinking...", viewModel.ActivityMessage);
|
||||
Assert.Equal(
|
||||
[
|
||||
"Checked the recent logs.",
|
||||
"Called tool: read_logs",
|
||||
"Matched the rule to the user request."
|
||||
],
|
||||
viewModel.ActivityMessages.ToArray());
|
||||
|
||||
pipeline.Release.SetResult();
|
||||
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.False(viewModel.IsThinking);
|
||||
Assert.Empty(viewModel.ActivityMessages);
|
||||
AssertCompletedActivity(
|
||||
viewModel,
|
||||
[
|
||||
"Checked the recent logs.",
|
||||
"Called tool: read_logs",
|
||||
"Matched the rule to the user request."
|
||||
],
|
||||
"Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelMarshalsBackgroundActivityUpdatesToCapturedContext()
|
||||
{
|
||||
var previousContext = SynchronizationContext.Current;
|
||||
var context = new RecordingSynchronizationContext();
|
||||
var pipeline = new BackgroundReportingRulesEditorPipeline(
|
||||
"Updated rules.",
|
||||
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."));
|
||||
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
|
||||
{
|
||||
Draft = "check logs"
|
||||
};
|
||||
|
||||
SynchronizationContext.SetSynchronizationContext(context);
|
||||
var sendTask = viewModel.SendAsync();
|
||||
SynchronizationContext.SetSynchronizationContext(previousContext);
|
||||
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(viewModel.IsThinking);
|
||||
Assert.Equal(["Checked the recent logs."], viewModel.ActivityMessages.ToArray());
|
||||
Assert.Equal(1, context.SendCount);
|
||||
|
||||
pipeline.Release.SetResult();
|
||||
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
AssertCompletedActivity(viewModel, ["Checked the recent logs."], "Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelKeepsActivityItemsOutOfModelConversation()
|
||||
{
|
||||
var pipeline = new BlockingRulesEditorPipeline("Updated rules.");
|
||||
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
|
||||
{
|
||||
Draft = "first"
|
||||
};
|
||||
|
||||
var firstSend = viewModel.SendAsync();
|
||||
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
pipeline.Release.SetResult();
|
||||
await firstSend.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
pipeline.Reset("Second response.");
|
||||
viewModel.Draft = "second";
|
||||
var secondSend = viewModel.SendAsync();
|
||||
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.Equal(
|
||||
[
|
||||
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "first"),
|
||||
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, "Updated rules.")
|
||||
],
|
||||
pipeline.LastConversation);
|
||||
|
||||
pipeline.Release.SetResult();
|
||||
await secondSend.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
private static void AssertCompletedActivity(
|
||||
WorkflowRulesEditorChatViewModel viewModel,
|
||||
IReadOnlyList<string> expectedActivityLines,
|
||||
string expectedAgentMessage)
|
||||
{
|
||||
Assert.Equal(3, viewModel.Messages.Count);
|
||||
Assert.Equal(WorkflowRulesEditorConversationItemKind.Activity, viewModel.Messages[1].Kind);
|
||||
Assert.StartsWith("Worked for ", viewModel.Messages[1].Content);
|
||||
Assert.Equal(expectedActivityLines, viewModel.Messages[1].ActivityLines);
|
||||
Assert.Equal(WorkflowRulesEditorConversationItemKind.Message, viewModel.Messages[2].Kind);
|
||||
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[2].Message?.Role);
|
||||
Assert.Equal(expectedAgentMessage, viewModel.Messages[2].Message?.Content);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -1185,31 +1430,36 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
private sealed class BlockingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
|
||||
{
|
||||
private readonly string response;
|
||||
private string response;
|
||||
|
||||
public BlockingRulesEditorPipeline(string response)
|
||||
{
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource Started { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource Release { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public IReadOnlyList<WorkflowRulesEditorChatMessage> LastConversation { get; private set; } = [];
|
||||
|
||||
public void Reset(string nextResponse)
|
||||
{
|
||||
response = nextResponse;
|
||||
Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
|
||||
public async Task<WorkflowRulesEditorChatResult> SendAsync(
|
||||
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
|
||||
string userMessage,
|
||||
CancellationToken cancellationToken,
|
||||
Action<string>? statusChanged = null)
|
||||
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
|
||||
{
|
||||
LastConversation = conversation;
|
||||
Started.SetResult();
|
||||
await Release.Task.WaitAsync(cancellationToken);
|
||||
return new WorkflowRulesEditorChatResult(
|
||||
response,
|
||||
conversation
|
||||
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
|
||||
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response))
|
||||
.ToList());
|
||||
return new WorkflowRulesEditorChatResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1226,19 +1476,23 @@ public sealed class WorkflowRulesEditorTests
|
||||
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
|
||||
string userMessage,
|
||||
CancellationToken cancellationToken,
|
||||
Action<string>? statusChanged = null)
|
||||
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
|
||||
{
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StatusReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
|
||||
private sealed class ReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
|
||||
{
|
||||
private readonly string response;
|
||||
private readonly IReadOnlyList<WorkflowRulesEditorActivityUpdate> updates;
|
||||
|
||||
public StatusReportingRulesEditorPipeline(string response)
|
||||
public ReportingRulesEditorPipeline(
|
||||
string response,
|
||||
params WorkflowRulesEditorActivityUpdate[] updates)
|
||||
{
|
||||
this.response = response;
|
||||
this.updates = updates;
|
||||
}
|
||||
|
||||
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
@@ -1249,17 +1503,57 @@ public sealed class WorkflowRulesEditorTests
|
||||
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
|
||||
string userMessage,
|
||||
CancellationToken cancellationToken,
|
||||
Action<string>? statusChanged = null)
|
||||
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
|
||||
{
|
||||
statusChanged?.Invoke("Reconnecting...");
|
||||
foreach (var update in updates)
|
||||
{
|
||||
activityChanged?.Invoke(update);
|
||||
}
|
||||
|
||||
Reported.SetResult();
|
||||
await Release.Task.WaitAsync(cancellationToken);
|
||||
return new WorkflowRulesEditorChatResult(
|
||||
response,
|
||||
conversation
|
||||
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
|
||||
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response))
|
||||
.ToList());
|
||||
return new WorkflowRulesEditorChatResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BackgroundReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
|
||||
{
|
||||
private readonly string response;
|
||||
private readonly WorkflowRulesEditorActivityUpdate update;
|
||||
|
||||
public BackgroundReportingRulesEditorPipeline(
|
||||
string response,
|
||||
WorkflowRulesEditorActivityUpdate update)
|
||||
{
|
||||
this.response = response;
|
||||
this.update = update;
|
||||
}
|
||||
|
||||
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public async Task<WorkflowRulesEditorChatResult> SendAsync(
|
||||
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
|
||||
string userMessage,
|
||||
CancellationToken cancellationToken,
|
||||
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
|
||||
{
|
||||
await Task.Run(() => activityChanged?.Invoke(update), cancellationToken);
|
||||
Reported.SetResult();
|
||||
await Release.Task.WaitAsync(cancellationToken);
|
||||
return new WorkflowRulesEditorChatResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingSynchronizationContext : SynchronizationContext
|
||||
{
|
||||
public int SendCount { get; private set; }
|
||||
|
||||
public override void Send(SendOrPostCallback callback, object? state)
|
||||
{
|
||||
SendCount++;
|
||||
callback(state);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Calendar;
|
||||
@@ -84,7 +85,8 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
foreach (var meeting in cachedMeetings.OrderBy(meeting => meeting.Start))
|
||||
{
|
||||
var promptKey = GetPromptKey(meeting);
|
||||
if (promptedMeetingKeys.Contains(promptKey) ||
|
||||
if (meeting.IsCanceled ||
|
||||
promptedMeetingKeys.Contains(promptKey) ||
|
||||
!IsInPromptWindow(meeting, now, promptOptions))
|
||||
{
|
||||
continue;
|
||||
@@ -96,8 +98,10 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
meeting.Subject,
|
||||
meeting.Start);
|
||||
await promptService.ShowPromptAsync(
|
||||
new MeetingStartPromptRequest(meeting),
|
||||
HandlePromptResponseAsync,
|
||||
new MeetingStartPromptRequest(
|
||||
meeting,
|
||||
recordingController.CurrentStatus.IsRecording && meeting.Metadata is not null),
|
||||
(response, token) => HandlePromptResponseAsync(meeting, response, token),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -133,9 +137,22 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
}
|
||||
|
||||
private async Task HandlePromptResponseAsync(
|
||||
CalendarMeeting meeting,
|
||||
MeetingStartPromptResponse response,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (response == MeetingStartPromptResponse.AttachMetadataToCurrentMeeting)
|
||||
{
|
||||
if (meeting.Metadata is not null)
|
||||
{
|
||||
await recordingController.AttachMetadataToCurrentMeetingAsync(
|
||||
meeting.Metadata,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (response != MeetingStartPromptResponse.Record)
|
||||
{
|
||||
return;
|
||||
@@ -146,7 +163,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
await recordingController.StopAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await recordingController.StartAsync(cancellationToken);
|
||||
await recordingController.StartFromPromptAsync(meeting.Metadata, cancellationToken);
|
||||
}
|
||||
|
||||
private void ResetPromptedMeetingsIfDayChanged(DateOnly today)
|
||||
@@ -183,6 +200,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
var syncDelay = nextSyncAt - now;
|
||||
|
||||
var nextMeetingStart = cachedMeetings
|
||||
.Where(meeting => !meeting.IsCanceled)
|
||||
.Where(meeting => !promptedMeetingKeys.Contains(GetPromptKey(meeting)))
|
||||
.Where(meeting => meeting.End > now)
|
||||
.Select(meeting => meeting.Start <= now ? now : meeting.Start)
|
||||
@@ -228,7 +246,9 @@ public sealed record CalendarMeeting(
|
||||
string Id,
|
||||
string Subject,
|
||||
DateTimeOffset Start,
|
||||
DateTimeOffset End);
|
||||
DateTimeOffset End,
|
||||
MeetingMetadata? Metadata = null,
|
||||
bool IsCanceled = false);
|
||||
|
||||
public interface IMeetingStartPromptService
|
||||
{
|
||||
@@ -238,12 +258,15 @@ public interface IMeetingStartPromptService
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingStartPromptRequest(CalendarMeeting Meeting);
|
||||
public sealed record MeetingStartPromptRequest(
|
||||
CalendarMeeting Meeting,
|
||||
bool CanAttachToCurrentMeeting = false);
|
||||
|
||||
public enum MeetingStartPromptResponse
|
||||
{
|
||||
Record,
|
||||
Skip
|
||||
Skip,
|
||||
AttachMetadataToCurrentMeeting
|
||||
}
|
||||
|
||||
public interface IMeetingPromptRecordingController
|
||||
@@ -252,6 +275,14 @@ public interface IMeetingPromptRecordingController
|
||||
|
||||
Task<RecordingStatus> StartAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task<RecordingStatus> StartFromPromptAsync(
|
||||
MeetingMetadata? metadata,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||
MeetingMetadata metadata,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
@@ -271,6 +302,20 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo
|
||||
return coordinator.StartAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> StartFromPromptAsync(
|
||||
MeetingMetadata? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||
MeetingMetadata metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return coordinator.AttachMetadataToCurrentMeetingAsync(metadata, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return coordinator.StopAsync(cancellationToken);
|
||||
|
||||
@@ -60,18 +60,21 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
|
||||
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
|
||||
if (end <= dayStart ||
|
||||
start < dayStart ||
|
||||
!IsTeamsAppointment(item))
|
||||
OutlookClassicCom.IsCanceledAppointment(item) ||
|
||||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawSubject = Convert.ToString(OutlookClassicCom.GetProperty(item, "Subject"))?.Trim();
|
||||
var subject = string.IsNullOrWhiteSpace(rawSubject) ? "Teams meeting" : rawSubject;
|
||||
var subject = OutlookClassicAppointmentMetadata.ReadSubject(item);
|
||||
var localEnd = OutlookClassicCom.ToLocalOffset(end);
|
||||
meetings.Add(new CalendarMeeting(
|
||||
GetAppointmentId(item, start, end, subject),
|
||||
subject,
|
||||
OutlookClassicCom.ToLocalOffset(start),
|
||||
OutlookClassicCom.ToLocalOffset(end)));
|
||||
localEnd,
|
||||
OutlookClassicAppointmentMetadata.CreateMetadata(item, end),
|
||||
IsCanceled: false));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -98,13 +101,5 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
|
||||
? $"{start:O}|{end:O}|{subject}"
|
||||
: $"{id}|{start:O}";
|
||||
}
|
||||
|
||||
private static bool IsTeamsAppointment(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -118,21 +118,10 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
||||
string promptId,
|
||||
MeetingStartPromptRequest request)
|
||||
{
|
||||
var yesButton = new ToastButton()
|
||||
.SetContent("Yes")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "record")
|
||||
.SetBackgroundActivation();
|
||||
var yesButton = BuildResponseButton("Yes", promptId, "record");
|
||||
var noButton = BuildResponseButton("No", promptId, "skip");
|
||||
|
||||
var noButton = new ToastButton()
|
||||
.SetContent("No")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "skip")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
return new ToastContentBuilder()
|
||||
var notification = new ToastContentBuilder()
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
@@ -141,6 +130,30 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
||||
.AddText($"{request.Meeting.Subject} starts at {request.Meeting.Start.LocalDateTime:t}.")
|
||||
.AddButton(yesButton)
|
||||
.AddButton(noButton);
|
||||
|
||||
if (request.CanAttachToCurrentMeeting)
|
||||
{
|
||||
notification.AddButton(
|
||||
BuildResponseButton(
|
||||
"Add metadata to current meeting",
|
||||
promptId,
|
||||
"attach-metadata"));
|
||||
}
|
||||
|
||||
return notification;
|
||||
}
|
||||
|
||||
private static ToastButton BuildResponseButton(
|
||||
string content,
|
||||
string promptId,
|
||||
string response)
|
||||
{
|
||||
return new ToastButton()
|
||||
.SetContent(content)
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", response)
|
||||
.SetBackgroundActivation();
|
||||
}
|
||||
|
||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||
@@ -166,10 +179,14 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
||||
return;
|
||||
}
|
||||
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
||||
string.Equals(responseValue, "record", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingStartPromptResponse.Record
|
||||
: MeetingStartPromptResponse.Skip;
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||
? responseValue.ToLowerInvariant() switch
|
||||
{
|
||||
"record" => MeetingStartPromptResponse.Record,
|
||||
"attach-metadata" => MeetingStartPromptResponse.AttachMetadataToCurrentMeeting,
|
||||
_ => MeetingStartPromptResponse.Skip
|
||||
}
|
||||
: MeetingStartPromptResponse.Skip;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
|
||||
@@ -32,7 +32,14 @@ Calendar-Feature is refined, Golem feature needs more work.
|
||||
|
||||
PROJECT.md should use this structure:
|
||||
|
||||
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: Project display name
|
||||
description: Concise project purpose and distinguishing context.
|
||||
---
|
||||
|
||||
# Executive Summary
|
||||
|
||||
<Elevator Pitch>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.LaunchProfiles;
|
||||
|
||||
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
var options = BindDefaultOptions();
|
||||
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ValidateSpeakerSampleDurations(options, DefaultProfileName);
|
||||
return new LaunchProfile(DefaultProfileName, options);
|
||||
}
|
||||
|
||||
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
|
||||
profileSection.Bind(options);
|
||||
ApplyArrayOverrides(profileSection, options);
|
||||
ValidateSpeakerSampleDurations(options, profileName);
|
||||
return new LaunchProfile(profileName, options);
|
||||
}
|
||||
|
||||
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
: name.Trim();
|
||||
}
|
||||
|
||||
private static void ValidateSpeakerSampleDurations(
|
||||
MeetingAssistantOptions options,
|
||||
string profileName)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification,
|
||||
$"Launch profile '{profileName}' has invalid speaker sample durations");
|
||||
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification.Resemblyzer,
|
||||
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
|
||||
}
|
||||
|
||||
private static void ApplyArrayOverrides(
|
||||
IConfigurationSection profileSection,
|
||||
MeetingAssistantOptions options)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||
<MicrosoftSpeechVersion>1.50.0</MicrosoftSpeechVersion>
|
||||
<MicrosoftSpeechVersion>1.52.0</MicrosoftSpeechVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
@@ -20,18 +20,19 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.11.0" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
|
||||
<PackageReference Include="DiffPlex" Version="1.9.0" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.9" />
|
||||
<PackageReference Include="NAudio" Version="2.3.0" />
|
||||
<PackageReference Include="NCalcSync" Version="5.12.0" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
|
||||
<PackageReference Include="NAudio" Version="3.1.0" />
|
||||
<PackageReference Include="NCalcSync" Version="7.2.0" />
|
||||
<PackageReference Include="RazorLight" Version="2.3.1" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.9" />
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
|
||||
<PackageReference Include="Whisper.net" Version="1.9.1" />
|
||||
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
|
||||
<PackageReference Include="YamlDotNet" Version="18.0.0" />
|
||||
<PackageReference Include="YamlDotNet" Version="18.1.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
@@ -49,8 +50,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\aec_v1.fpie" Link="runtimes\win-x64\native\MASmodels\aec_v1.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\pns_avg4.fpie" Link="runtimes\win-x64\native\MASmodels\pns_avg4.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\*.fpie" Link="runtimes\win-x64\native\MASmodels\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -64,6 +64,7 @@
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||
<Compile Remove="Recording\WindowsMicrophoneDeviceProvider.cs" />
|
||||
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
|
||||
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
|
||||
<Compile Remove="Taskbar\UnoTaskbarIconService.Windows.cs" />
|
||||
|
||||
@@ -59,6 +59,7 @@ public sealed class ScreenshotOcrOptions
|
||||
|
||||
Identify who appears to be talking, who appears to be presenting, and what is being presented when visible.
|
||||
If people or participant tiles are visible, state whether it is clear from the screenshot exactly who is in the meeting or whether the visible people are only a partial participant result.
|
||||
Include any attendee names that are clearly visible or strongly deduced from the screenshot in the meeting-assistant metadata. The attendee list may be partial.
|
||||
If the screenshot contains slides, capture the visible text in clean markdown.
|
||||
If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately.
|
||||
If it contains another kind of shared screen or scene, describe the scene and the relevant visible information.
|
||||
@@ -67,9 +68,9 @@ public sealed class ScreenshotOcrOptions
|
||||
Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently.
|
||||
End your response with exactly one fenced json block containing meeting-assistant metadata, for example:
|
||||
```json
|
||||
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } }
|
||||
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, "attendees": ["Ada Lovelace"] }
|
||||
```
|
||||
Use `{ "crop": null }` when no confident presentation/shared-screen crop exists.
|
||||
Use `{ "crop": null, "attendees": [] }` when no confident presentation/shared-screen crop exists and no attendees are visible or deducible.
|
||||
Do not invent information that is not visible in the screenshot.
|
||||
""";
|
||||
|
||||
@@ -125,6 +126,8 @@ public sealed class RecordingOptions
|
||||
|
||||
public int Channels { get; set; } = 1;
|
||||
|
||||
public string? MicrophoneDeviceId { get; set; }
|
||||
|
||||
public double MicrophoneMixGain { get; set; } = 1;
|
||||
|
||||
public double SystemAudioMixGain { get; set; } = 1;
|
||||
@@ -162,6 +165,8 @@ public sealed class RecordingInactivitySafeguardOptions
|
||||
|
||||
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
|
||||
|
||||
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
|
||||
|
||||
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
|
||||
@@ -178,10 +183,8 @@ public sealed class WhisperLocalOptions
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions
|
||||
public class PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string DockerCommand { get; set; } = "docker";
|
||||
|
||||
public string BaseImage { get; set; } = "python:3.11-slim";
|
||||
@@ -205,6 +208,11 @@ public sealed class PyannoteDiarizationOptions
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
|
||||
public sealed class AzureSpeechOptions
|
||||
{
|
||||
public string Endpoint { get; set; } = "";
|
||||
@@ -337,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
|
||||
|
||||
public int MaxSnippetsPerSpeaker { get; set; } = 3;
|
||||
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
|
||||
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
|
||||
|
||||
@@ -352,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
|
||||
public AzureSpeechOptions AzureSpeech { get; set; } = new();
|
||||
|
||||
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
|
||||
|
||||
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public int RequiredVectorsPerSpeaker { get; set; } = 5;
|
||||
|
||||
public int MaxVectorsPerIdentity { get; set; } = 1000;
|
||||
|
||||
public int OutlierPruningMinimumVectors { get; set; } = 20;
|
||||
|
||||
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
|
||||
|
||||
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
|
||||
|
||||
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
|
||||
|
||||
public double MinimumClusterCohesion { get; set; } = 0.75;
|
||||
|
||||
public double MinimumIdentitySimilarity { get; set; } = 0.75;
|
||||
|
||||
public double MinimumSimilarityMargin { get; set; } = 0.05;
|
||||
|
||||
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
|
||||
|
||||
public string PackageVersion { get; set; } = "0.1.4";
|
||||
|
||||
public string PythonCommand { get; set; } = "python";
|
||||
|
||||
public string TorchVersion { get; set; } = "2.14.0+cpu";
|
||||
|
||||
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
|
||||
|
||||
public string WebRtcVadVersion { get; set; } = "2.0.14";
|
||||
|
||||
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
|
||||
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
|
||||
}
|
||||
|
||||
public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
@@ -362,9 +411,8 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
|
||||
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
|
||||
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new()
|
||||
public PyannoteRuntimeOptions Diarization { get; set; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
CommandTimeout = TimeSpan.FromHours(1),
|
||||
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
|
||||
};
|
||||
@@ -380,6 +428,8 @@ public sealed class AgentOptions
|
||||
|
||||
public string Model { get; set; } = "chatgpt/gpt-5.5";
|
||||
|
||||
public bool UseStreaming { get; set; } = true;
|
||||
|
||||
public bool EnableThinking { get; set; } = true;
|
||||
|
||||
public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium;
|
||||
@@ -411,6 +461,8 @@ public sealed class WorkflowRulesEditorOptions
|
||||
|
||||
public string? Model { get; set; }
|
||||
|
||||
public bool? UseStreaming { get; set; }
|
||||
|
||||
public bool? EnableThinking { get; set; }
|
||||
|
||||
public ReasoningEffortOption? ReasoningEffort { get; set; }
|
||||
@@ -439,6 +491,7 @@ public sealed class WorkflowRulesEditorOptions
|
||||
Key = string.IsNullOrWhiteSpace(Key) ? defaults.Key : Key,
|
||||
KeyEnv = string.IsNullOrWhiteSpace(KeyEnv) ? defaults.KeyEnv : KeyEnv!,
|
||||
Model = string.IsNullOrWhiteSpace(Model) ? defaults.Model : Model!,
|
||||
UseStreaming = UseStreaming ?? defaults.UseStreaming,
|
||||
EnableThinking = EnableThinking ?? defaults.EnableThinking,
|
||||
ReasoningEffort = ReasoningEffort ?? defaults.ReasoningEffort,
|
||||
ReconnectionAttempts = ReconnectionAttempts ?? defaults.ReconnectionAttempts,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using YamlDotNet.Core;
|
||||
using YamlDotNet.Serialization;
|
||||
|
||||
@@ -113,7 +115,24 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
||||
|
||||
private static string EscapeQuoted(string value)
|
||||
{
|
||||
return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
|
||||
var escaped = new StringBuilder(value.Length + 2);
|
||||
escaped.Append('"');
|
||||
foreach (var character in value)
|
||||
{
|
||||
escaped.Append(character switch
|
||||
{
|
||||
'\\' => "\\\\",
|
||||
'"' => "\\\"",
|
||||
'\r' => "\\r",
|
||||
'\n' => "\\n",
|
||||
'\t' => "\\t",
|
||||
< ' ' => "\\x" + ((int)character).ToString("X2", CultureInfo.InvariantCulture),
|
||||
_ => character.ToString()
|
||||
});
|
||||
}
|
||||
|
||||
escaped.Append('"');
|
||||
return escaped.ToString();
|
||||
}
|
||||
|
||||
private static string EscapeNullableDateTime(DateTimeOffset? value)
|
||||
@@ -135,7 +154,7 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
||||
return "\"\"";
|
||||
}
|
||||
|
||||
if (value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal))
|
||||
if (RequiresQuotedScalar(value))
|
||||
{
|
||||
return EscapeQuoted(value);
|
||||
}
|
||||
@@ -143,6 +162,49 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
||||
return value;
|
||||
}
|
||||
|
||||
private static bool RequiresQuotedScalar(string value)
|
||||
{
|
||||
if (char.IsWhiteSpace(value[0]) || char.IsWhiteSpace(value[^1]))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (value.Contains(':', StringComparison.Ordinal) ||
|
||||
value.Contains('\'', StringComparison.Ordinal) ||
|
||||
value.Contains(" #", StringComparison.Ordinal) ||
|
||||
value.Any(char.IsControl))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IsYamlIndicator(value[0]))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IsYamlCoreSchemaKeyword(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, out _) ||
|
||||
double.TryParse(value, CultureInfo.InvariantCulture, out _);
|
||||
}
|
||||
|
||||
private static bool IsYamlIndicator(char character)
|
||||
{
|
||||
return character is '-' or '?' or ':' or ',' or '[' or ']' or '{' or '}' or
|
||||
'#' or '&' or '*' or '!' or '|' or '>' or '\'' or '"' or '%' or '@' or '`';
|
||||
}
|
||||
|
||||
private static bool IsYamlCoreSchemaKeyword(string value)
|
||||
{
|
||||
return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(value, "false", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(value, "null", StringComparison.OrdinalIgnoreCase) ||
|
||||
value == "~";
|
||||
}
|
||||
|
||||
private sealed class MeetingNoteYaml
|
||||
{
|
||||
[YamlMember(Alias = "title")]
|
||||
|
||||
@@ -8,4 +8,13 @@ internal static class MeetingAttendeeNames
|
||||
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
|
||||
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
|
||||
}
|
||||
|
||||
public static List<string> NormalizeDistinct(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(attendee => attendee.Trim())
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,4 +14,24 @@ public static class MeetingNoteActionLinks
|
||||
var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}";
|
||||
return $"[{label}]({url})";
|
||||
}
|
||||
|
||||
public static string CreateScreenshotOcrRetryLink(
|
||||
string publicBaseUrl,
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
string label = "Retry screenshot OCR")
|
||||
{
|
||||
var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl)
|
||||
? "http://localhost:5090"
|
||||
: publicBaseUrl.TrimEnd('/');
|
||||
var url = $"{baseUrl}/meetings/screenshot-ocr/retry" +
|
||||
$"?meetingNotePath={Uri.EscapeDataString(artifacts.MeetingNotePath)}" +
|
||||
$"&transcriptPath={Uri.EscapeDataString(artifacts.TranscriptPath)}" +
|
||||
$"&assistantContextPath={Uri.EscapeDataString(artifacts.AssistantContextPath)}" +
|
||||
$"&summaryPath={Uri.EscapeDataString(artifacts.SummaryPath)}" +
|
||||
$"&screenshotPath={Uri.EscapeDataString(screenshotPath)}" +
|
||||
$"&screenshotId={Uri.EscapeDataString(screenshotId)}";
|
||||
return $"[{label}]({url})";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
#if WINDOWS
|
||||
using MeetingAssistant.Calendar;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
|
||||
internal static class OutlookClassicAppointmentMetadata
|
||||
{
|
||||
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
|
||||
{
|
||||
var title = ReadSubject(appointment);
|
||||
return new MeetingMetadata(
|
||||
title,
|
||||
ReadAttendees(appointment),
|
||||
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
|
||||
OutlookClassicCom.ToLocalOffset(end));
|
||||
}
|
||||
|
||||
public static string ReadSubject(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
|
||||
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
|
||||
}
|
||||
|
||||
public static bool IsTeamsAppointment(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
|
||||
}
|
||||
|
||||
public static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> ReadAttendees(object appointment)
|
||||
{
|
||||
var attendees = new List<string>();
|
||||
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
|
||||
if (!string.IsNullOrWhiteSpace(organizer))
|
||||
{
|
||||
attendees.Add(organizer.Trim());
|
||||
}
|
||||
|
||||
object? recipients = null;
|
||||
try
|
||||
{
|
||||
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
|
||||
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
|
||||
for (var index = 1; index <= count; index++)
|
||||
{
|
||||
object? recipient = null;
|
||||
try
|
||||
{
|
||||
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
|
||||
var formatted = FormatRecipient(recipient!);
|
||||
if (!string.IsNullOrWhiteSpace(formatted))
|
||||
{
|
||||
attendees.Add(formatted);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
OutlookClassicCom.ReleaseComObject(recipient);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
OutlookClassicCom.ReleaseComObject(recipients);
|
||||
}
|
||||
|
||||
return NormalizeAttendees(attendees);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static bool IsTeamsJoinLine(string line)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim();
|
||||
var mailtoPrefix = "mailto:";
|
||||
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
return attendee[(emailStart + 1)..emailEnd].Trim();
|
||||
}
|
||||
|
||||
return attendee.Trim();
|
||||
}
|
||||
|
||||
private static string FormatRecipient(object recipient)
|
||||
{
|
||||
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
|
||||
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(email) ||
|
||||
email.Contains('/'))
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -108,6 +108,37 @@ internal static class OutlookClassicCom
|
||||
}
|
||||
}
|
||||
|
||||
public static bool IsCanceledAppointment(object appointment)
|
||||
{
|
||||
var meetingStatus = TryGetProperty(appointment, "MeetingStatus");
|
||||
if (meetingStatus is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
var status = Convert.ToInt32(meetingStatus);
|
||||
if (status is 5 or 7)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (exception is FormatException or InvalidCastException or OverflowException)
|
||||
{
|
||||
// Fall back to subject prefixes when Outlook exposes a non-numeric status value.
|
||||
}
|
||||
}
|
||||
|
||||
var subject = Convert.ToString(TryGetProperty(appointment, "Subject")) ?? "";
|
||||
return SubjectIndicatesCancellation(subject);
|
||||
}
|
||||
|
||||
internal static bool SubjectIndicatesCancellation(string subject)
|
||||
{
|
||||
var normalized = subject.TrimStart();
|
||||
return normalized.StartsWith("Canceled:", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Cancelled:", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Abgesagt:", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static DateTimeOffset ToLocalOffset(DateTime value)
|
||||
{
|
||||
return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local));
|
||||
|
||||
@@ -121,7 +121,9 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
|
||||
}
|
||||
|
||||
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
|
||||
if (end < windowStart || !IsTeamsAppointment(item))
|
||||
if (end < windowStart ||
|
||||
OutlookClassicCom.IsCanceledAppointment(item) ||
|
||||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -150,15 +152,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
|
||||
return null;
|
||||
}
|
||||
|
||||
var appointment = selected.Appointment;
|
||||
var title = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var attendees = ReadAttendees(appointment);
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return new MeetingMetadata(
|
||||
title.Trim(),
|
||||
attendees,
|
||||
ExtractAgenda(body),
|
||||
OutlookClassicCom.ToLocalOffset(selected.End));
|
||||
return OutlookClassicAppointmentMetadata.CreateMetadata(selected.Appointment, selected.End);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -169,158 +163,6 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
|
||||
}
|
||||
}
|
||||
|
||||
internal static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static bool IsTeamsJoinLine(string line)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
|
||||
}
|
||||
|
||||
private static bool IsTeamsAppointment(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ReadAttendees(object appointment)
|
||||
{
|
||||
var attendees = new List<string>();
|
||||
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
|
||||
if (!string.IsNullOrWhiteSpace(organizer))
|
||||
{
|
||||
attendees.Add(organizer.Trim());
|
||||
}
|
||||
|
||||
object? recipients = null;
|
||||
try
|
||||
{
|
||||
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
|
||||
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
|
||||
for (var index = 1; index <= count; index++)
|
||||
{
|
||||
object? recipient = null;
|
||||
try
|
||||
{
|
||||
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
|
||||
var formatted = FormatRecipient(recipient!);
|
||||
if (!string.IsNullOrWhiteSpace(formatted))
|
||||
{
|
||||
attendees.Add(formatted);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
OutlookClassicCom.ReleaseComObject(recipient);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
OutlookClassicCom.ReleaseComObject(recipients);
|
||||
}
|
||||
|
||||
return NormalizeAttendees(attendees);
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim();
|
||||
var mailtoPrefix = "mailto:";
|
||||
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
return attendee[(emailStart + 1)..emailEnd].Trim();
|
||||
}
|
||||
|
||||
return attendee.Trim();
|
||||
}
|
||||
|
||||
private static string FormatRecipient(object recipient)
|
||||
{
|
||||
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
|
||||
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(email) ||
|
||||
email.Contains('/'))
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
private sealed record OutlookAppointmentCandidate(
|
||||
object Appointment,
|
||||
DateTime Start,
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
namespace MeetingAssistant.Notifications;
|
||||
|
||||
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
|
||||
|
||||
public interface IAgentApprovalPromptService
|
||||
{
|
||||
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
|
||||
{
|
||||
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
|
||||
}
|
||||
|
||||
public sealed class AgentApprovalService
|
||||
{
|
||||
private readonly IAgentApprovalPromptService prompts;
|
||||
private readonly ILogger<AgentApprovalService> logger;
|
||||
private readonly TimeSpan timeout;
|
||||
|
||||
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
|
||||
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
|
||||
|
||||
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
|
||||
{
|
||||
this.prompts = prompts;
|
||||
this.logger = logger;
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
|
||||
{
|
||||
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
deadline.CancelAfter(timeout);
|
||||
try
|
||||
{
|
||||
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
|
||||
.WaitAsync(deadline.Token);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return accepted && !deadline.IsCancellationRequested;
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await deadline.CancelAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
#if WINDOWS
|
||||
using System.Collections.Concurrent;
|
||||
using CommunityToolkit.WinUI.Notifications;
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Notifications;
|
||||
|
||||
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
|
||||
: IAgentApprovalPromptService, IDisposable
|
||||
{
|
||||
private const string Source = "summary-agent-approval";
|
||||
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||
private readonly object gate = new();
|
||||
private bool registered;
|
||||
private bool disposed;
|
||||
|
||||
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = Guid.NewGuid().ToString("N");
|
||||
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
try
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (disposed) return false;
|
||||
if (!registered)
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||
registered = true;
|
||||
}
|
||||
pending[id] = response;
|
||||
new ToastContentBuilder()
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText(prompt.Title)
|
||||
.AddText(prompt.Message)
|
||||
.AddButton(new ToastButton().SetContent("Yes")
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||
.AddButton(new ToastButton().SetContent("No")
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.AddArgument("response", "no").SetBackgroundActivation())
|
||||
.Show(toast =>
|
||||
{
|
||||
toast.Tag = id;
|
||||
toast.Group = Source;
|
||||
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
|
||||
});
|
||||
}
|
||||
|
||||
return await response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
pending.TryRemove(id, out _);
|
||||
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
|
||||
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
|
||||
}
|
||||
}
|
||||
|
||||
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||
{
|
||||
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||
if (arguments.GetValueOrDefault("source") != Source ||
|
||||
!arguments.TryGetValue("requestId", out var id) ||
|
||||
!pending.TryRemove(id, out var response)) return;
|
||||
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
disposed = true;
|
||||
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||
registered = false;
|
||||
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||
pending.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
+184
-4
@@ -4,6 +4,7 @@ using MeetingAssistant.Hotkeys;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Logging;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Notifications;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Speakers;
|
||||
@@ -16,9 +17,21 @@ using Microsoft.Extensions.Options;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
|
||||
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
|
||||
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
|
||||
builder.Services.AddOptions<MeetingAssistantOptions>()
|
||||
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
|
||||
builder.Services.AddSingleton<WindowsMicrophoneDeviceProvider>();
|
||||
builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
|
||||
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
|
||||
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
|
||||
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
|
||||
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
|
||||
builder.Services.AddSingleton<MicrophoneAudioSource>();
|
||||
builder.Services.AddSingleton<SystemAudioSource>();
|
||||
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||
@@ -30,6 +43,8 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
|
||||
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||
@@ -70,7 +85,10 @@ builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreen
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
|
||||
builder.Services.AddSingleton<IMeetingScreenshotService, MeetingScreenshotService>();
|
||||
builder.Services.AddSingleton<MeetingScreenshotService>();
|
||||
builder.Services.AddSingleton<IMeetingScreenshotService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
|
||||
builder.Services.AddSingleton<IScreenshotOcrRetryRunner>(sp => sp.GetRequiredService<MeetingScreenshotService>());
|
||||
builder.Services.AddSingleton<IMeetingNoteImageOcrService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
|
||||
builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOptions) =>
|
||||
{
|
||||
var appOptions = services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value;
|
||||
@@ -82,8 +100,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
|
||||
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
|
||||
builder.Services.AddSingleton(services =>
|
||||
{
|
||||
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer;
|
||||
return resemblyzer.Enabled
|
||||
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
|
||||
: SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
});
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
|
||||
: services.GetRequiredService<SpeakerIdentityService>());
|
||||
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
|
||||
: services.GetRequiredService<SpeakerIdentityMergeService>());
|
||||
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
|
||||
@@ -93,7 +138,11 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
|
||||
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
|
||||
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
|
||||
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
|
||||
#if WINDOWS
|
||||
@@ -103,6 +152,9 @@ builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRule
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<AgentApprovalService>();
|
||||
builder.Services.AddSingleton<WorkflowChangeRequestService>();
|
||||
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
|
||||
builder.Services.AddSingleton<AsrDiagnosticService>();
|
||||
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
|
||||
@@ -125,6 +177,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
|
||||
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
||||
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
||||
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
||||
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddHostedService<GlobalHotkeyService>();
|
||||
builder.Services.AddHostedService<UnoTaskbarIconService>();
|
||||
@@ -410,6 +463,81 @@ app.MapGet("/profiles/{launchProfile}/meetings/summary/retry", async (
|
||||
launchProfiles,
|
||||
cancellationToken));
|
||||
|
||||
app.MapPost("/meetings/screenshot-ocr/retry", async (
|
||||
ScreenshotOcrRetryRequest request,
|
||||
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
CancellationToken cancellationToken) =>
|
||||
await RetryScreenshotOcrAsync(
|
||||
null,
|
||||
request,
|
||||
screenshotOcrRetryRunner,
|
||||
options.Value,
|
||||
null,
|
||||
cancellationToken));
|
||||
app.MapPost("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
|
||||
string launchProfile,
|
||||
ScreenshotOcrRetryRequest request,
|
||||
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
CancellationToken cancellationToken) =>
|
||||
await RetryScreenshotOcrAsync(
|
||||
launchProfile,
|
||||
request,
|
||||
screenshotOcrRetryRunner,
|
||||
options.Value,
|
||||
launchProfiles,
|
||||
cancellationToken));
|
||||
app.MapGet("/meetings/screenshot-ocr/retry", async (
|
||||
string meetingNotePath,
|
||||
string transcriptPath,
|
||||
string assistantContextPath,
|
||||
string summaryPath,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
CancellationToken cancellationToken) =>
|
||||
await RetryScreenshotOcrAsync(
|
||||
null,
|
||||
new ScreenshotOcrRetryRequest(
|
||||
meetingNotePath,
|
||||
transcriptPath,
|
||||
assistantContextPath,
|
||||
summaryPath,
|
||||
screenshotPath,
|
||||
screenshotId),
|
||||
screenshotOcrRetryRunner,
|
||||
options.Value,
|
||||
null,
|
||||
cancellationToken));
|
||||
app.MapGet("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
|
||||
string launchProfile,
|
||||
string meetingNotePath,
|
||||
string transcriptPath,
|
||||
string assistantContextPath,
|
||||
string summaryPath,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
CancellationToken cancellationToken) =>
|
||||
await RetryScreenshotOcrAsync(
|
||||
launchProfile,
|
||||
new ScreenshotOcrRetryRequest(
|
||||
meetingNotePath,
|
||||
transcriptPath,
|
||||
assistantContextPath,
|
||||
summaryPath,
|
||||
screenshotPath,
|
||||
screenshotId),
|
||||
screenshotOcrRetryRunner,
|
||||
options.Value,
|
||||
launchProfiles,
|
||||
cancellationToken));
|
||||
|
||||
static async Task<IResult> RetrySummaryAsync(
|
||||
string? launchProfile,
|
||||
string? summaryPath,
|
||||
@@ -438,6 +566,50 @@ static async Task<IResult> RetrySummaryAsync(
|
||||
});
|
||||
}
|
||||
|
||||
static async Task<IResult> RetryScreenshotOcrAsync(
|
||||
string? launchProfile,
|
||||
ScreenshotOcrRetryRequest request,
|
||||
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
|
||||
MeetingAssistantOptions defaultOptions,
|
||||
ILaunchProfileOptionsProvider? launchProfiles,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.MeetingNotePath) ||
|
||||
string.IsNullOrWhiteSpace(request.TranscriptPath) ||
|
||||
string.IsNullOrWhiteSpace(request.AssistantContextPath) ||
|
||||
string.IsNullOrWhiteSpace(request.SummaryPath) ||
|
||||
string.IsNullOrWhiteSpace(request.ScreenshotPath) ||
|
||||
string.IsNullOrWhiteSpace(request.ScreenshotId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "Meeting artifact paths, screenshot path, and screenshot id are required." });
|
||||
}
|
||||
|
||||
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
request.MeetingNotePath,
|
||||
request.TranscriptPath,
|
||||
request.AssistantContextPath,
|
||||
request.SummaryPath);
|
||||
var trigger = await screenshotOcrRetryRunner.TriggerOcrRetryAsync(
|
||||
artifacts,
|
||||
request.ScreenshotPath,
|
||||
request.ScreenshotId,
|
||||
profileOptions,
|
||||
cancellationToken);
|
||||
if (trigger is null)
|
||||
{
|
||||
return Results.NotFound(new { error = "No retryable screenshot OCR block was found for the requested screenshot." });
|
||||
}
|
||||
|
||||
return Results.Accepted(value: new
|
||||
{
|
||||
assistantContextPath = trigger.AssistantContextPath,
|
||||
screenshotPath = trigger.ScreenshotPath,
|
||||
screenshotId = trigger.ScreenshotId,
|
||||
state = "ocr retrying"
|
||||
});
|
||||
}
|
||||
|
||||
app.MapPost("/asr/transcribe-file", async (
|
||||
AsrDiagnosticRequest request,
|
||||
AsrDiagnosticService diagnostics,
|
||||
@@ -544,6 +716,14 @@ public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null);
|
||||
|
||||
public sealed record SummaryRetryRequest(string SummaryPath);
|
||||
|
||||
public sealed record ScreenshotOcrRetryRequest(
|
||||
string MeetingNotePath,
|
||||
string TranscriptPath,
|
||||
string AssistantContextPath,
|
||||
string SummaryPath,
|
||||
string ScreenshotPath,
|
||||
string ScreenshotId);
|
||||
|
||||
public sealed record SpeakerIdentityMergeDiagnosticRequest(double? RecentDays = null);
|
||||
|
||||
public sealed record WorkflowConfigurationReloadResponse(string? RulesPath);
|
||||
|
||||
@@ -66,32 +66,22 @@ public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
|
||||
return items;
|
||||
}
|
||||
|
||||
public async Task CompleteAsync(string id, CancellationToken cancellationToken)
|
||||
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = GetBacklogFolder(options);
|
||||
var path = GetItemPath(folder, id);
|
||||
var path = GetItemPath(folder, item.Id);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
OfflineTranscriptionBacklogItem? item = null;
|
||||
try
|
||||
{
|
||||
item = JsonSerializer.Deserialize<OfflineTranscriptionBacklogItem>(
|
||||
await File.ReadAllTextAsync(path, cancellationToken),
|
||||
JsonOptions);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
File.Delete(path);
|
||||
if (item is not null && File.Exists(item.AudioPath))
|
||||
if (File.Exists(item.AudioPath))
|
||||
{
|
||||
DeleteCompletedAudio(item.AudioPath);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string GetBacklogFolder(MeetingAssistantOptions options)
|
||||
|
||||
@@ -6,6 +6,8 @@ public interface IMeetingInactivityPromptService
|
||||
MeetingInactivityPromptRequest request,
|
||||
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task DismissAllAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingInactivityPromptRequest(
|
||||
@@ -18,9 +20,32 @@ public enum MeetingInactivityPromptResponse
|
||||
{
|
||||
Dismissed,
|
||||
Continue,
|
||||
Pause,
|
||||
Stop
|
||||
}
|
||||
|
||||
internal sealed record MeetingInactivityPromptAction(
|
||||
string Content,
|
||||
string ResponseArgument,
|
||||
MeetingInactivityPromptResponse Response);
|
||||
|
||||
internal static class MeetingInactivityPromptActions
|
||||
{
|
||||
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
|
||||
[
|
||||
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
|
||||
new("No", "continue", MeetingInactivityPromptResponse.Continue),
|
||||
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
|
||||
];
|
||||
|
||||
public static MeetingInactivityPromptResponse ParseResponse(string? value)
|
||||
{
|
||||
return All.FirstOrDefault(action =>
|
||||
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||
?.Response ?? MeetingInactivityPromptResponse.Continue;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
|
||||
{
|
||||
public Task ShowStopPromptAsync(
|
||||
@@ -30,4 +55,9 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
// The caller owns the returned source and disposes it when it implements IDisposable,
|
||||
// including when capture is canceled before enumeration starts.
|
||||
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMicrophoneDeviceProvider
|
||||
{
|
||||
IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones();
|
||||
|
||||
MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMicrophoneSwitchPromptService
|
||||
{
|
||||
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(false);
|
||||
}
|
||||
@@ -33,6 +33,17 @@ public interface ITranscriptStore
|
||||
string line,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
async Task AppendLinesAsync(
|
||||
TranscriptSession session,
|
||||
IReadOnlyList<string> lines,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var line in lines)
|
||||
{
|
||||
await AppendLineAsync(session, line, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
|
||||
@@ -28,11 +28,13 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
||||
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
||||
private readonly IMeetingScreenshotService screenshotService;
|
||||
private readonly IMeetingNoteImageOcrService meetingNoteImageOcrService;
|
||||
private readonly IMeetingRunArtifactCleaner artifactCleaner;
|
||||
private readonly IMeetingInactivityPromptService inactivityPromptService;
|
||||
private readonly IMeetingInactivityClock inactivityClock;
|
||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
|
||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
private RecordingRun? currentRun;
|
||||
@@ -58,10 +60,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
ILaunchProfileOptionsProvider? launchProfiles = null,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
|
||||
IMeetingScreenshotService? screenshotService = null,
|
||||
IMeetingNoteImageOcrService? meetingNoteImageOcrService = null,
|
||||
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
||||
IMeetingInactivityPromptService? inactivityPromptService = null,
|
||||
IMeetingInactivityClock? inactivityClock = null,
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
|
||||
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
|
||||
{
|
||||
this.audioSource = audioSource;
|
||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||
@@ -78,11 +82,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
|
||||
this.meetingNoteImageOcrService = meetingNoteImageOcrService ?? NoopMeetingNoteImageOcrService.Instance;
|
||||
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
|
||||
this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService();
|
||||
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||
this.options = options.Value;
|
||||
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
|
||||
?? SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -93,7 +100,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
currentArtifacts?.AssistantContextPath,
|
||||
currentArtifacts?.SummaryPath,
|
||||
GetProcessState(currentRun),
|
||||
currentRun?.LaunchProfileName);
|
||||
currentRun?.LaunchProfileName,
|
||||
IsRecording && currentRun?.IsTranscriptionPaused == true);
|
||||
|
||||
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
|
||||
|
||||
@@ -142,12 +150,66 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public async Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return await StartAsync(null, cancellationToken);
|
||||
return await StartAsync(null, null, suppressMetadataLookup: false, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> StartFromPromptAsync(
|
||||
MeetingMetadata? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||
MeetingMetadata metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var run = currentRun;
|
||||
if (run is null || run.IsCaptureStopping)
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||
run.Artifacts,
|
||||
await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken),
|
||||
metadata,
|
||||
run.Options,
|
||||
cancellationToken);
|
||||
currentMeetingNote = meetingNote;
|
||||
await transcriptStore.UpdateMetadataAsync(
|
||||
run.Session,
|
||||
run.Artifacts,
|
||||
meetingNote,
|
||||
cancellationToken);
|
||||
run.MarkPromptMetadataAttached();
|
||||
|
||||
logger.LogInformation(
|
||||
"Attached prompted calendar metadata to active meeting {MeetingNotePath}",
|
||||
run.MeetingNotePath);
|
||||
return CurrentStatus;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> StartAsync(
|
||||
string? launchProfileName,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await StartAsync(launchProfileName, null, suppressMetadataLookup: false, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> StartAsync(
|
||||
string? launchProfileName,
|
||||
MeetingMetadata? suppliedMetadata,
|
||||
bool suppressMetadataLookup,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
@@ -164,14 +226,16 @@ public sealed class MeetingRecordingCoordinator
|
||||
var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken);
|
||||
var assistantContextPath = GetAssistantContextPath(startedAt, runOptions);
|
||||
var summaryPath = GetSummaryPath(startedAt, runOptions);
|
||||
var meetingNote = MeetingNoteTemplate.Create(
|
||||
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
|
||||
startTime: startedAt,
|
||||
attendees: [],
|
||||
transcriptPath: currentSession.TranscriptPath,
|
||||
assistantContextPath: assistantContextPath,
|
||||
summaryPath: summaryPath);
|
||||
|
||||
currentMeetingNote = await meetingNoteStore.SaveAsync(
|
||||
MeetingNoteTemplate.Create(
|
||||
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
|
||||
startTime: startedAt,
|
||||
attendees: [],
|
||||
transcriptPath: currentSession.TranscriptPath,
|
||||
assistantContextPath: assistantContextPath,
|
||||
summaryPath: summaryPath),
|
||||
meetingNote,
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
currentArtifacts = new MeetingSessionArtifacts(
|
||||
@@ -179,12 +243,27 @@ public sealed class MeetingRecordingCoordinator
|
||||
currentSession.TranscriptPath,
|
||||
assistantContextPath,
|
||||
summaryPath);
|
||||
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, currentMeetingNote, "", null, cancellationToken);
|
||||
await meetingArtifactStore.CreateAssistantContextAsync(
|
||||
currentArtifacts,
|
||||
currentMeetingNote,
|
||||
"",
|
||||
null,
|
||||
cancellationToken);
|
||||
await meetingWorkflowEngine.RunAsync(
|
||||
MeetingWorkflowEvent.Created(currentArtifacts),
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
|
||||
if (suppliedMetadata is not null)
|
||||
{
|
||||
currentMeetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||
currentArtifacts,
|
||||
currentMeetingNote,
|
||||
suppliedMetadata,
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
|
||||
currentArtifacts,
|
||||
currentMeetingNote,
|
||||
@@ -215,10 +294,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
startedAt,
|
||||
launchProfile.Name,
|
||||
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
|
||||
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
|
||||
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
|
||||
logger);
|
||||
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
|
||||
if (runOptions.Recording.InactivitySafeguard.Enabled)
|
||||
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
|
||||
{
|
||||
_ = Task.Run(
|
||||
() => RunInactivitySafeguardAsync(run),
|
||||
@@ -226,9 +307,20 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
|
||||
currentRun = run;
|
||||
_ = Task.Run(
|
||||
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
|
||||
CancellationToken.None);
|
||||
if (suppliedMetadata is null && !suppressMetadataLookup)
|
||||
{
|
||||
_ = Task.Run(
|
||||
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
|
||||
CancellationToken.None);
|
||||
}
|
||||
else
|
||||
{
|
||||
await TransitionMeetingAsync(
|
||||
run,
|
||||
AssistantContextState.Transcribing,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
logger.LogInformation("Meeting recording started");
|
||||
|
||||
return CurrentStatus;
|
||||
@@ -249,11 +341,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return await StopAsync(null, cancellationToken);
|
||||
return await StopAsync(null, null, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> StopAsync(
|
||||
DateTimeOffset? inferredEndTime,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
DateTimeOffset? expectedPauseStartedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -262,13 +357,30 @@ public sealed class MeetingRecordingCoordinator
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (currentRun is null || currentRun.IsCaptureStopping)
|
||||
if (currentRun is null ||
|
||||
currentRun.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
run = currentRun;
|
||||
if (inferredEndTime is not null)
|
||||
var inactivityStopClaimed = expectedActivityVersion is not null;
|
||||
if (expectedActivityVersion is not null &&
|
||||
(inferredEndTime is null ||
|
||||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
|
||||
if (expectedPauseStartedAt is not null &&
|
||||
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
|
||||
{
|
||||
run.SetInferredEndTime(inferredEndTime.Value);
|
||||
}
|
||||
@@ -278,7 +390,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
{
|
||||
run.Abort();
|
||||
}
|
||||
else
|
||||
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
|
||||
{
|
||||
run.StopCapture();
|
||||
}
|
||||
@@ -304,6 +416,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
|
||||
{
|
||||
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
if (IsAzureSpeechRun(run))
|
||||
{
|
||||
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
|
||||
@@ -324,6 +442,65 @@ public sealed class MeetingRecordingCoordinator
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var run = currentRun;
|
||||
if (run is null ||
|
||||
run.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var changed = false;
|
||||
await run.PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var changedAt = inactivityClock.Now;
|
||||
changed = run.TrySetTranscriptionPaused(
|
||||
isPaused,
|
||||
changedAt,
|
||||
expectedActivityVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
run.PipelineGate.Release();
|
||||
}
|
||||
|
||||
if (changed && isPaused)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (changed)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Meeting transcription {PauseState}",
|
||||
isPaused ? "paused" : "unpaused");
|
||||
}
|
||||
|
||||
return CurrentStatus;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -445,7 +622,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
TimeSpan.Zero,
|
||||
TimeSpan.Zero,
|
||||
"System",
|
||||
$"Transcription profile changed to {targetProfile.Name}.");
|
||||
$"Transcription profile changed to {targetProfile.Name}. Speaker recognition and identities reset.");
|
||||
run.AddLiveSegment(marker);
|
||||
await AppendTranscriptSegmentAsync(run, marker, cancellationToken);
|
||||
|
||||
@@ -576,8 +753,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
chunk.Channels);
|
||||
}
|
||||
|
||||
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
|
||||
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
|
||||
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -616,7 +792,6 @@ public sealed class MeetingRecordingCoordinator
|
||||
run.ResetSpeakerIdentification();
|
||||
}
|
||||
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
var sample = run.TryAddSpeakerSample(segment);
|
||||
if (sample is not null)
|
||||
{
|
||||
@@ -633,6 +808,56 @@ public sealed class MeetingRecordingCoordinator
|
||||
var relabeledSegment = run.Relabel(segment);
|
||||
run.AddLiveSegment(relabeledSegment);
|
||||
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
if (!string.IsNullOrWhiteSpace(segment.Text))
|
||||
{
|
||||
// Switching profiles holds the coordinator gate while draining this reader.
|
||||
// Notification cleanup must not make transcript draining wait for that gate.
|
||||
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsForRunAsync(
|
||||
RecordingRun run,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Pending cleanup is no longer needed after this run stops capturing.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await inactivityPromptService.DismissAllAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -642,14 +867,23 @@ public sealed class MeetingRecordingCoordinator
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var formatted = TranscriptLineFormatter.Format(segment);
|
||||
var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId &&
|
||||
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference)
|
||||
? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken)
|
||||
: segment.Kind == TranscriptionSegmentKind.Marker &&
|
||||
segment.MarkerId is { } existingMarkerId &&
|
||||
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference)
|
||||
? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken)
|
||||
: await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
|
||||
TranscriptLineReference lineReference;
|
||||
if (segment.ReplacesMarkerId is { } replacesMarkerId &&
|
||||
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference))
|
||||
{
|
||||
lineReference = await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken);
|
||||
}
|
||||
else if (segment.Kind == TranscriptionSegmentKind.Marker &&
|
||||
segment.MarkerId is { } existingMarkerId &&
|
||||
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference))
|
||||
{
|
||||
lineReference = await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
lineReference = await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
|
||||
}
|
||||
|
||||
if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId)
|
||||
{
|
||||
run.SetTranscriptMarker(markerId, lineReference);
|
||||
@@ -751,7 +985,10 @@ public sealed class MeetingRecordingCoordinator
|
||||
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
||||
{
|
||||
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
|
||||
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
|
||||
var promptThresholds = safeguardOptions.Enabled
|
||||
? GetInactivityPromptThresholds(safeguardOptions)
|
||||
: [];
|
||||
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
|
||||
var promptedThresholds = new HashSet<TimeSpan>();
|
||||
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
|
||||
|
||||
@@ -759,7 +996,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
|
||||
string.Join(", ", promptThresholds),
|
||||
safeguardOptions.AutoStopAfter,
|
||||
safeguardOptions.CheckInterval);
|
||||
checkInterval);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -775,9 +1012,38 @@ public sealed class MeetingRecordingCoordinator
|
||||
snapshot.LastTranscriptActivityAt);
|
||||
}
|
||||
|
||||
var pauseStartedAt = snapshot.TranscriptionPausedAt;
|
||||
if (pauseStartedAt is not null)
|
||||
{
|
||||
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
|
||||
if (maximumPauseDuration > TimeSpan.Zero &&
|
||||
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
|
||||
maximumPauseDuration);
|
||||
var stopStatus = await StopAsync(
|
||||
null,
|
||||
run,
|
||||
null,
|
||||
pauseStartedAt,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = inactivityClock.Now;
|
||||
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
|
||||
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
if (safeguardOptions.Enabled &&
|
||||
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
inactivityDuration >= safeguardOptions.AutoStopAfter)
|
||||
{
|
||||
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
|
||||
@@ -785,8 +1051,16 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
|
||||
inactivityDuration,
|
||||
inferredEndTime);
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
return;
|
||||
var stopStatus = await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
snapshot.ActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var threshold in promptThresholds)
|
||||
@@ -803,6 +1077,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
inactivityDuration,
|
||||
threshold,
|
||||
inferredEndTime);
|
||||
var promptActivityVersion = snapshot.ActivityVersion;
|
||||
await inactivityPromptService.ShowStopPromptAsync(
|
||||
new MeetingInactivityPromptRequest(
|
||||
inactivityDuration,
|
||||
@@ -818,16 +1093,36 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
if (response == MeetingInactivityPromptResponse.Stop)
|
||||
{
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
}
|
||||
else if (response == MeetingInactivityPromptResponse.Pause)
|
||||
{
|
||||
await SetTranscriptionPausedAsync(
|
||||
true,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
CancellationToken.None);
|
||||
}
|
||||
},
|
||||
run.CaptureCancellation);
|
||||
|
||||
var currentActivity = run.GetTranscriptActivitySnapshot();
|
||||
if (currentActivity.IsTranscriptionPaused ||
|
||||
currentActivity.ActivityVersion != promptActivityVersion)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(run.CaptureCancellation);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
GetInactivitySafeguardCheckInterval(safeguardOptions),
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
@@ -895,11 +1190,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
try
|
||||
{
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
|
||||
if (checkpoint is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
|
||||
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -955,7 +1250,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.IsAborted)
|
||||
if (run.IsAborted || run.HasAttachedPromptMetadata)
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -963,25 +1258,21 @@ public sealed class MeetingRecordingCoordinator
|
||||
await gate.WaitAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
if (run.IsAborted)
|
||||
if (run.IsAborted || run.HasAttachedPromptMetadata)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
|
||||
await ApplyMeetingMetadataAsync(meetingNote, metadata, run.Options, CancellationToken.None);
|
||||
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
|
||||
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||
run.Artifacts,
|
||||
await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None),
|
||||
metadata,
|
||||
run.Options,
|
||||
CancellationToken.None);
|
||||
if (currentMeetingNote?.Path == meetingNote.Path)
|
||||
{
|
||||
currentMeetingNote = meetingNote;
|
||||
}
|
||||
|
||||
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
|
||||
run.Artifacts,
|
||||
meetingNote,
|
||||
metadata.Agenda,
|
||||
metadata.ScheduledEnd,
|
||||
CancellationToken.None);
|
||||
logger.LogInformation(
|
||||
"Applied Outlook meeting metadata to {MeetingNotePath}",
|
||||
run.MeetingNotePath);
|
||||
@@ -1021,6 +1312,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
|
||||
private async Task ApplyMeetingMetadataAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingNote meetingNote,
|
||||
MeetingMetadata metadata,
|
||||
MeetingAssistantOptions options,
|
||||
@@ -1043,10 +1335,65 @@ public sealed class MeetingRecordingCoordinator
|
||||
return;
|
||||
}
|
||||
|
||||
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
|
||||
var canonicalized = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
|
||||
meetingNote.Frontmatter.Attendees = await TransformAttendeesAsync(
|
||||
artifacts,
|
||||
canonicalized,
|
||||
options,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<MeetingNote> ApplyAndPersistMeetingMetadataAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingNote meetingNote,
|
||||
MeetingMetadata metadata,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await ApplyMeetingMetadataAsync(
|
||||
artifacts,
|
||||
meetingNote,
|
||||
metadata,
|
||||
options,
|
||||
cancellationToken);
|
||||
var savedMeetingNote = await meetingNoteStore.SaveAsync(
|
||||
meetingNote,
|
||||
options,
|
||||
cancellationToken);
|
||||
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
|
||||
artifacts,
|
||||
savedMeetingNote,
|
||||
metadata.Agenda,
|
||||
metadata.ScheduledEnd,
|
||||
cancellationToken);
|
||||
return savedMeetingNote;
|
||||
}
|
||||
|
||||
private async Task<List<string>> TransformAttendeesAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
IReadOnlyList<string> attendees,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var transformed = new List<string>();
|
||||
foreach (var attendee in attendees)
|
||||
{
|
||||
var value = await TransformAttendeeAsync(artifacts, attendee, options, cancellationToken);
|
||||
var storageValue = string.Equals(value, attendee, StringComparison.Ordinal)
|
||||
? attendee.Trim()
|
||||
: NormalizeAttendeeName(value);
|
||||
var normalized = NormalizeAttendeeName(storageValue);
|
||||
if (!string.IsNullOrWhiteSpace(normalized) &&
|
||||
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
transformed.Add(storageValue);
|
||||
}
|
||||
}
|
||||
|
||||
return transformed;
|
||||
}
|
||||
|
||||
private async Task<List<string>> CanonicalizeAttendeesAsync(
|
||||
IReadOnlyList<string> attendees,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -1361,8 +1708,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
continue;
|
||||
}
|
||||
|
||||
var transformedDisplayName = await TransformAttendeeAsync(
|
||||
artifacts,
|
||||
match.DisplayName.Trim(),
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
var displayName = string.Equals(transformedDisplayName, match.DisplayName.Trim(), StringComparison.Ordinal)
|
||||
? match.DisplayName.Trim()
|
||||
: NormalizeAttendeeName(transformedDisplayName);
|
||||
var acceptedNames = match.AcceptedNames
|
||||
.Append(match.DisplayName)
|
||||
.Append(displayName)
|
||||
.Select(NormalizeAttendeeName)
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
@@ -1381,7 +1737,6 @@ public sealed class MeetingRecordingCoordinator
|
||||
continue;
|
||||
}
|
||||
|
||||
var displayName = match.DisplayName.Trim();
|
||||
meetingNote.Frontmatter.Attendees.Add(displayName);
|
||||
existingNames.Add(NormalizeAttendeeName(displayName));
|
||||
changed = true;
|
||||
@@ -1441,6 +1796,18 @@ public sealed class MeetingRecordingCoordinator
|
||||
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
|
||||
}
|
||||
|
||||
private async Task<string> TransformAttendeeAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string attendee,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await meetingWorkflowEngine.TransformAttendeeAsync(
|
||||
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
|
||||
options,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static bool RemoveDuplicateAcceptedAliases(
|
||||
List<string> attendees,
|
||||
string displayName,
|
||||
@@ -1505,6 +1872,10 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
try
|
||||
{
|
||||
await meetingNoteImageOcrService.ProcessMeetingNoteImageEmbedsAsync(
|
||||
run.Artifacts,
|
||||
run.Options,
|
||||
cancellationToken);
|
||||
await screenshotService.WaitForPendingOcrAsync(
|
||||
run.Artifacts,
|
||||
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
|
||||
@@ -1651,6 +2022,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
|
||||
}
|
||||
|
||||
private static TimeSpan GetInactivitySafeguardCheckInterval(
|
||||
RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
@@ -1707,6 +2083,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
private TimeSpan? lastTranscriptSegmentEnd;
|
||||
private long transcriptActivityVersion;
|
||||
private DateTimeOffset? inferredEndTime;
|
||||
private bool isTranscriptionPaused;
|
||||
private DateTimeOffset? transcriptionPausedAt;
|
||||
|
||||
public RecordingRun(
|
||||
CancellationTokenSource captureCancellation,
|
||||
@@ -1722,6 +2100,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
string launchProfileName,
|
||||
TimeSpan liveSampleBufferDuration,
|
||||
int maxSpeakerSamples,
|
||||
bool requireNonOverlappingSpeakerSamples,
|
||||
ILogger logger)
|
||||
{
|
||||
CaptureCancellationSource = captureCancellation;
|
||||
@@ -1742,8 +2121,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
liveSampleBufferDuration,
|
||||
maxSpeakerSamples,
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleSegmentGap,
|
||||
logger);
|
||||
options.SpeakerIdentification.MaximumSampleDuration,
|
||||
logger,
|
||||
requireNonOverlappingSpeakerSamples);
|
||||
}
|
||||
|
||||
public CancellationTokenSource CaptureCancellationSource { get; }
|
||||
@@ -1788,6 +2168,19 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public bool HasSwitchedProfile { get; private set; }
|
||||
|
||||
public bool HasAttachedPromptMetadata { get; private set; }
|
||||
|
||||
public bool IsTranscriptionPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
return isTranscriptionPaused;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||
|
||||
public DateTimeOffset? InferredEndTime
|
||||
@@ -1806,6 +2199,42 @@ public sealed class MeetingRecordingCoordinator
|
||||
CaptureCancellationSource.Cancel();
|
||||
}
|
||||
|
||||
public void MarkPromptMetadataAttached()
|
||||
{
|
||||
HasAttachedPromptMetadata = true;
|
||||
}
|
||||
|
||||
public bool TrySetTranscriptionPaused(
|
||||
bool isPaused,
|
||||
DateTimeOffset changedAt,
|
||||
long? expectedActivityVersion)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (expectedActivityVersion is not null &&
|
||||
transcriptActivityVersion != expectedActivityVersion.Value)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isTranscriptionPaused == isPaused)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
isTranscriptionPaused = isPaused;
|
||||
transcriptionPausedAt = isPaused ? changedAt : null;
|
||||
|
||||
if (!isPaused)
|
||||
{
|
||||
lastTranscriptActivityAt = changedAt;
|
||||
}
|
||||
|
||||
transcriptActivityVersion++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Abort()
|
||||
{
|
||||
IsAborted = true;
|
||||
@@ -1834,13 +2263,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
await PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
AppendAudio(chunk);
|
||||
await Pipeline.WriteAsync(chunk, cancellationToken);
|
||||
var routedChunk = IsTranscriptionPaused
|
||||
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
|
||||
: chunk;
|
||||
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
|
||||
AppendAudio(routedChunk);
|
||||
await Pipeline.WriteAsync(routedChunk, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -2024,7 +2457,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
return new TranscriptActivitySnapshot(
|
||||
lastTranscriptActivityAt,
|
||||
lastTranscriptSegmentEnd,
|
||||
transcriptActivityVersion);
|
||||
transcriptActivityVersion,
|
||||
isTranscriptionPaused,
|
||||
transcriptionPausedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2044,6 +2479,35 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
inferredEndTime = endTime;
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void AppendAudio(AudioChunk chunk)
|
||||
{
|
||||
speakerSampleCollector.AppendAudio(chunk);
|
||||
@@ -2067,9 +2531,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
return speakerSampleCollector.Snapshot();
|
||||
}
|
||||
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
var speakers = GetUnmappedSampleSpeakers();
|
||||
var speakers = GetUnmappedSampleSpeakers(samples);
|
||||
if (speakers.Count == 0)
|
||||
{
|
||||
return null;
|
||||
@@ -2077,7 +2543,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
var checkpoint = new LiveIdentificationCheckpoint(
|
||||
speakers,
|
||||
BuildAttendeeSignature(meetingNote));
|
||||
BuildAttendeeSignature(meetingNote),
|
||||
Options.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? speakers.Select(speaker => samples.Count(sample =>
|
||||
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
|
||||
: []);
|
||||
lock (liveIdentificationGate)
|
||||
{
|
||||
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
|
||||
@@ -2135,9 +2605,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
return GetSpeakerSamplesSnapshot()
|
||||
return samples
|
||||
.Select(sample => sample.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -2173,7 +2643,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
public sealed record TranscriptActivitySnapshot(
|
||||
DateTimeOffset LastTranscriptActivityAt,
|
||||
TimeSpan? LastTranscriptSegmentEnd,
|
||||
long ActivityVersion);
|
||||
long ActivityVersion,
|
||||
bool IsTranscriptionPaused,
|
||||
DateTimeOffset? TranscriptionPausedAt);
|
||||
|
||||
private static int StateRank(AssistantContextState state)
|
||||
{
|
||||
@@ -2191,11 +2663,13 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public sealed record LiveIdentificationCheckpoint(
|
||||
IReadOnlyList<string> Speakers,
|
||||
string AttendeeSignature)
|
||||
string AttendeeSignature,
|
||||
IReadOnlyList<int> SampleCounts)
|
||||
{
|
||||
public bool Matches(LiveIdentificationCheckpoint other)
|
||||
{
|
||||
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
|
||||
SampleCounts.SequenceEqual(other.SampleCounts) &&
|
||||
Speakers.Count == other.Speakers.Count &&
|
||||
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -2228,7 +2702,8 @@ public sealed record RecordingStatus(
|
||||
string? AssistantContextPath,
|
||||
string? SummaryPath,
|
||||
RecordingProcessState State = RecordingProcessState.Idle,
|
||||
string? LaunchProfile = null);
|
||||
string? LaunchProfile = null,
|
||||
bool IsPaused = false);
|
||||
|
||||
public enum RecordingProcessState
|
||||
{
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
{
|
||||
private static readonly TimeSpan DefaultRecoveryDelay = TimeSpan.FromSeconds(1);
|
||||
private readonly IMicrophoneCaptureSourceFactory captureSources;
|
||||
private readonly ILogger<MicrophoneAudioSource> logger;
|
||||
private readonly TimeSpan recoveryDelay;
|
||||
private readonly MicrophoneSwitchMonitor? switchMonitor;
|
||||
private readonly MicrophoneDeviceSelection? selection;
|
||||
|
||||
public MicrophoneAudioSource(
|
||||
IMicrophoneCaptureSourceFactory captureSources,
|
||||
ILogger<MicrophoneAudioSource> logger,
|
||||
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||
MicrophoneDeviceSelection? selection = null)
|
||||
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
|
||||
{
|
||||
}
|
||||
|
||||
internal MicrophoneAudioSource(
|
||||
IMicrophoneCaptureSourceFactory captureSources,
|
||||
ILogger<MicrophoneAudioSource> logger,
|
||||
TimeSpan recoveryDelay,
|
||||
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||
MicrophoneDeviceSelection? selection = null)
|
||||
{
|
||||
this.captureSources = captureSources;
|
||||
this.logger = logger;
|
||||
this.recoveryDelay = recoveryDelay;
|
||||
this.switchMonitor = switchMonitor;
|
||||
this.selection = selection;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
MeetingAssistantOptions options,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
using var session = new CaptureSession(selection, cancellationToken);
|
||||
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
|
||||
try
|
||||
{
|
||||
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
|
||||
{
|
||||
yield return chunk;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
session.Stop();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
|
||||
MeetingAssistantOptions options,
|
||||
CaptureSession session,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var failedAttempts = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var captureToken = session.BeginCapture();
|
||||
IMeetingAudioSource? captureSource = null;
|
||||
IAsyncEnumerator<AudioChunk>? capture = null;
|
||||
Exception? failure = null;
|
||||
|
||||
try
|
||||
{
|
||||
captureSource = captureSources.CreateCapture(options);
|
||||
capture = captureSource
|
||||
.CaptureAsync(options, captureToken)
|
||||
.GetAsyncEnumerator(captureToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
failure = exception;
|
||||
}
|
||||
|
||||
using var captureOwner = captureSource as IDisposable;
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (capture is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (!captureToken.IsCancellationRequested)
|
||||
{
|
||||
var hasNext = false;
|
||||
try
|
||||
{
|
||||
hasNext = await capture.MoveNextAsync();
|
||||
}
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
failure = exception;
|
||||
}
|
||||
|
||||
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (failedAttempts > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Microphone capture recovered after {FailedAttemptCount} failed attempt(s)",
|
||||
failedAttempts);
|
||||
failedAttempts = 0;
|
||||
}
|
||||
|
||||
yield return capture.Current;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
await capture.DisposeAsync();
|
||||
}
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
failure ??= exception;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var switchRequested = captureToken.IsCancellationRequested;
|
||||
session.EndCapture();
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (switchRequested)
|
||||
{
|
||||
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
|
||||
continue;
|
||||
}
|
||||
|
||||
failedAttempts++;
|
||||
logger.LogWarning(
|
||||
failure,
|
||||
"Microphone capture stopped unexpectedly; re-resolving an available microphone in {RecoveryDelay}",
|
||||
recoveryDelay);
|
||||
|
||||
if (!await WaitForRecoveryAsync(cancellationToken))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> WaitForRecoveryAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(recoveryDelay, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
|
||||
{
|
||||
private readonly object gate = new();
|
||||
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
private CancellationTokenSource? capture;
|
||||
|
||||
public CancellationToken Token => lifetime.Token;
|
||||
|
||||
public CancellationToken BeginCapture()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
|
||||
return capture.Token;
|
||||
}
|
||||
}
|
||||
|
||||
public void EndCapture()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
capture?.Dispose();
|
||||
capture = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
|
||||
selection.Select(device.Id);
|
||||
capture?.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
lock (gate) lifetime.Cancel();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Stop();
|
||||
EndCapture();
|
||||
lifetime.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed record MicrophoneDevice(
|
||||
string Id,
|
||||
string Name);
|
||||
|
||||
public sealed record MicrophoneDeviceSnapshot(
|
||||
IReadOnlyList<MicrophoneDevice> Available,
|
||||
MicrophoneDevice? Current);
|
||||
@@ -0,0 +1,56 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class MicrophoneDeviceSelection
|
||||
{
|
||||
private readonly object sync = new();
|
||||
private string? selectedDeviceId;
|
||||
|
||||
public string? SelectedDeviceId
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (sync)
|
||||
{
|
||||
return selectedDeviceId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Select(string deviceId)
|
||||
{
|
||||
lock (sync)
|
||||
{
|
||||
selectedDeviceId = string.IsNullOrWhiteSpace(deviceId)
|
||||
? null
|
||||
: deviceId.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
public MicrophoneDevice? Resolve(
|
||||
string? configuredDeviceId,
|
||||
MicrophoneDevice? defaultDevice,
|
||||
IReadOnlyList<MicrophoneDevice> availableDevices)
|
||||
{
|
||||
var selected = SelectedDeviceId;
|
||||
return FindById(availableDevices, selected) ??
|
||||
FindById(availableDevices, configuredDeviceId) ??
|
||||
FindById(availableDevices, defaultDevice?.Id) ??
|
||||
availableDevices.FirstOrDefault() ??
|
||||
defaultDevice;
|
||||
}
|
||||
|
||||
private static MicrophoneDevice? FindById(
|
||||
IReadOnlyList<MicrophoneDevice> devices,
|
||||
string? id)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return devices.FirstOrDefault(device => string.Equals(
|
||||
device.Id,
|
||||
id.Trim(),
|
||||
StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class MicrophoneSwitchMonitor
|
||||
{
|
||||
private readonly IMicrophoneDeviceProvider devices;
|
||||
private readonly IMicrophoneSwitchPromptService prompts;
|
||||
private readonly ILogger<MicrophoneSwitchMonitor> logger;
|
||||
private readonly TimeSpan pollInterval;
|
||||
private readonly TimeSpan promptLifetime;
|
||||
|
||||
public MicrophoneSwitchMonitor(
|
||||
IMicrophoneDeviceProvider devices,
|
||||
IMicrophoneSwitchPromptService prompts,
|
||||
ILogger<MicrophoneSwitchMonitor> logger)
|
||||
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
|
||||
{
|
||||
}
|
||||
|
||||
internal MicrophoneSwitchMonitor(
|
||||
IMicrophoneDeviceProvider devices,
|
||||
IMicrophoneSwitchPromptService prompts,
|
||||
ILogger<MicrophoneSwitchMonitor> logger,
|
||||
TimeSpan pollInterval,
|
||||
TimeSpan promptLifetime)
|
||||
{
|
||||
this.devices = devices;
|
||||
this.prompts = prompts;
|
||||
this.logger = logger;
|
||||
this.pollInterval = pollInterval;
|
||||
this.promptLifetime = promptLifetime;
|
||||
}
|
||||
|
||||
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
await Task.Delay(pollInterval, cancellationToken);
|
||||
var available = ReadDevices();
|
||||
if (available is null) continue;
|
||||
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var (id, prompt) in pending.ToArray())
|
||||
{
|
||||
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
|
||||
if (prompt.Task.IsCompleted)
|
||||
{
|
||||
prompt.Cancellation.Dispose();
|
||||
pending.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var device in available)
|
||||
{
|
||||
if (known.Add(device.Id))
|
||||
{
|
||||
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
promptCancellation.CancelAfter(promptLifetime);
|
||||
pending.Add(device.Id, new PendingPrompt(promptCancellation,
|
||||
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
|
||||
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
|
||||
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
|
||||
&& !cancellationToken.IsCancellationRequested
|
||||
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
|
||||
&& !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
switchMicrophone(device, cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
|
||||
}
|
||||
}
|
||||
|
||||
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
|
||||
{
|
||||
try
|
||||
{
|
||||
return devices.GetAvailableMicrophones();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not check for newly available microphones");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
|
||||
}
|
||||
@@ -4,40 +4,34 @@ using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
|
||||
{
|
||||
private readonly ILogger<MicrophoneAudioSource> logger;
|
||||
private readonly IWaveIn capture;
|
||||
private readonly string sourceName;
|
||||
private readonly ILogger logger;
|
||||
private int disposed;
|
||||
|
||||
public MicrophoneAudioSource(ILogger<MicrophoneAudioSource> logger)
|
||||
public NaudioCaptureAudioSource(
|
||||
IWaveIn capture,
|
||||
string sourceName,
|
||||
ILogger logger)
|
||||
{
|
||||
this.capture = capture;
|
||||
this.sourceName = sourceName;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||
return CaptureWith(cancellationToken);
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
public void Dispose()
|
||||
{
|
||||
return CaptureAsync(new WasapiCapture(), options, cancellationToken);
|
||||
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
|
||||
}
|
||||
|
||||
private IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
IWaveIn capture,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
|
||||
return CaptureWith(capture, "microphone", logger, cancellationToken);
|
||||
}
|
||||
|
||||
internal static async IAsyncEnumerable<AudioChunk> CaptureWith(
|
||||
IWaveIn capture,
|
||||
string sourceName,
|
||||
ILogger logger,
|
||||
private async IAsyncEnumerable<AudioChunk> CaptureWith(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var channel = Channel.CreateUnbounded<AudioChunk>();
|
||||
@@ -92,7 +86,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
}
|
||||
finally
|
||||
{
|
||||
capture.Dispose();
|
||||
Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -120,6 +114,6 @@ public sealed class SystemAudioSource : IMeetingAudioSource
|
||||
WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels)
|
||||
};
|
||||
|
||||
return MicrophoneAudioSource.CaptureWith(capture, "system", logger, cancellationToken);
|
||||
return new NaudioCaptureAudioSource(capture, "system", logger).CaptureAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ public interface IOfflineTranscriptionBacklog
|
||||
|
||||
Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task CompleteAsync(string id, CancellationToken cancellationToken);
|
||||
Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record OfflineTranscriptionBacklogItem(
|
||||
@@ -38,7 +38,7 @@ public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
|
||||
return Task.FromResult<IReadOnlyList<OfflineTranscriptionBacklogItem>>([]);
|
||||
}
|
||||
|
||||
public Task CompleteAsync(string id, CancellationToken cancellationToken)
|
||||
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
@@ -4,14 +4,11 @@ using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class OfflineTranscriptionBacklogProcessor
|
||||
{
|
||||
private const int MaxChunkDurationMilliseconds = 1000;
|
||||
|
||||
private readonly IOfflineTranscriptionBacklog backlog;
|
||||
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
|
||||
private readonly ITranscriptStore transcriptStore;
|
||||
@@ -72,7 +69,7 @@ public sealed class OfflineTranscriptionBacklogProcessor
|
||||
try
|
||||
{
|
||||
await ProcessAsync(item, cancellationToken);
|
||||
await backlog.CompleteAsync(item.Id, cancellationToken);
|
||||
await backlog.CompleteAsync(item, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}",
|
||||
item.Id,
|
||||
@@ -120,8 +117,25 @@ public sealed class OfflineTranscriptionBacklogProcessor
|
||||
pipelineOptions,
|
||||
cancellationToken);
|
||||
|
||||
if (!finishedSegments.Any(segment =>
|
||||
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
|
||||
}
|
||||
|
||||
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
|
||||
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
|
||||
await transcriptStore.AppendLinesAsync(session,
|
||||
[
|
||||
"",
|
||||
"## Offline transcription recovery",
|
||||
"",
|
||||
"> The following transcript was recovered by replaying the entire recording. "
|
||||
+ "It may duplicate earlier transcript passages. "
|
||||
+ "Treat repeated passages as the same discussion when summarizing.",
|
||||
"",
|
||||
.. lines
|
||||
], cancellationToken);
|
||||
|
||||
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
|
||||
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
|
||||
@@ -215,29 +229,9 @@ public sealed class OfflineTranscriptionBacklogProcessor
|
||||
ISpeechRecognitionPipeline pipeline,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var reader = new WaveFileReader(audioPath);
|
||||
var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond;
|
||||
var chunkSize = Math.Max(
|
||||
reader.WaveFormat.BlockAlign,
|
||||
bytesPerSecond * MaxChunkDurationMilliseconds / 1000);
|
||||
chunkSize -= chunkSize % reader.WaveFormat.BlockAlign;
|
||||
|
||||
var buffer = new byte[chunkSize];
|
||||
while (true)
|
||||
await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(audioPath, cancellationToken: cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken);
|
||||
if (read == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await pipeline.WriteAsync(
|
||||
new AudioChunk(
|
||||
buffer[..read],
|
||||
reader.WaveFormat.SampleRate,
|
||||
reader.WaveFormat.Channels),
|
||||
cancellationToken);
|
||||
await pipeline.WriteAsync(chunk, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public static class PcmWavAudioChunkReader
|
||||
{
|
||||
private const int MaxChunkDurationMilliseconds = 1000;
|
||||
|
||||
public static async IAsyncEnumerable<AudioChunk> ReadChunksAsync(
|
||||
string path,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
var reader = new WaveFileReader(path);
|
||||
try
|
||||
{
|
||||
var format = reader.WaveFormat;
|
||||
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
|
||||
}
|
||||
|
||||
var chunkSize = Math.Max(
|
||||
format.BlockAlign,
|
||||
format.AverageBytesPerSecond * MaxChunkDurationMilliseconds / 1000);
|
||||
chunkSize -= chunkSize % format.BlockAlign;
|
||||
|
||||
var buffer = new byte[chunkSize];
|
||||
int read;
|
||||
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
|
||||
{
|
||||
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
reader.Dispose();
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
// NAudio can throw while disposing reader streams from async iterators after all chunks were read.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
private readonly object gate = new();
|
||||
private readonly RollingAudioBuffer audioBuffer;
|
||||
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly int maxSamplesPerSpeaker;
|
||||
private readonly TimeSpan minimumUninterruptedSpeechDuration;
|
||||
private readonly TimeSpan maximumSegmentGap;
|
||||
private readonly TimeSpan minimumSampleSpeechDuration;
|
||||
private readonly TimeSpan maximumSampleDuration;
|
||||
private readonly bool requireNonOverlappingSamples;
|
||||
private readonly ILogger? logger;
|
||||
private PendingSpeakerSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
|
||||
: this(
|
||||
bufferDuration,
|
||||
maxSamplesPerSpeaker,
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(1),
|
||||
logger: null)
|
||||
{
|
||||
}
|
||||
private SpeakerSampleSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(
|
||||
TimeSpan bufferDuration,
|
||||
int maxSamplesPerSpeaker,
|
||||
TimeSpan minimumUninterruptedSpeechDuration,
|
||||
TimeSpan maximumSegmentGap,
|
||||
ILogger? logger = null)
|
||||
TimeSpan minimumSampleSpeechDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
ILogger? logger = null,
|
||||
bool requireNonOverlappingSamples = false)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
minimumSampleSpeechDuration,
|
||||
maximumSampleDuration,
|
||||
"Speaker sample collector configuration is invalid");
|
||||
audioBuffer = new RollingAudioBuffer(bufferDuration);
|
||||
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
|
||||
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
|
||||
? minimumUninterruptedSpeechDuration
|
||||
: TimeSpan.Zero;
|
||||
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
|
||||
? maximumSegmentGap
|
||||
: TimeSpan.Zero;
|
||||
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
|
||||
this.maximumSampleDuration = maximumSampleDuration;
|
||||
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
lock (gate)
|
||||
{
|
||||
samplesBySpeaker.Clear();
|
||||
lastAcceptedEndBySpeaker.Clear();
|
||||
pendingSpan = null;
|
||||
audioBuffer.Reset();
|
||||
}
|
||||
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return null;
|
||||
}
|
||||
|
||||
TranscriptionSegment sampleSegment;
|
||||
PendingSpanReset? reset;
|
||||
SpeakerSampleSpan? sampleSpan;
|
||||
SpeakerSampleSpan? previousSpan;
|
||||
lock (gate)
|
||||
{
|
||||
(sampleSegment, reset) = ExtendPendingSpan(segment);
|
||||
if (requireNonOverlappingSamples &&
|
||||
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
|
||||
segment.Start < lastAcceptedEnd)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
|
||||
segment.Speaker,
|
||||
lastAcceptedEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
|
||||
}
|
||||
|
||||
if (reset is not null)
|
||||
if (sampleSpan is null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
|
||||
reset.PreviousSpeaker,
|
||||
segment.Speaker,
|
||||
reset.PreviousEnd,
|
||||
segment.Start,
|
||||
reset.Gap,
|
||||
maximumSegmentGap);
|
||||
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
|
||||
segment.Speaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
|
||||
var sampleSegment = sampleSpan.ToSegment();
|
||||
|
||||
if (previousSpan is not null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
|
||||
previousSpan.Speaker,
|
||||
segment.Speaker,
|
||||
previousSpan.End,
|
||||
segment.Start);
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
|
||||
if (!score.Accepted)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
sampleSegment.Speaker,
|
||||
score.Reason,
|
||||
sampleSpan.SpeechDuration,
|
||||
sampleSegment.End - sampleSegment.Start,
|
||||
minimumUninterruptedSpeechDuration,
|
||||
minimumSampleSpeechDuration,
|
||||
score.WordCount);
|
||||
return null;
|
||||
}
|
||||
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
|
||||
lock (gate)
|
||||
{
|
||||
if (requireNonOverlappingSamples &&
|
||||
ReferenceEquals(pendingSpan, sampleSpan))
|
||||
{
|
||||
pendingSpan = null;
|
||||
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
|
||||
}
|
||||
|
||||
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
|
||||
{
|
||||
samples = [];
|
||||
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
{
|
||||
if (pendingSpan is null ||
|
||||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
|
||||
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
|
||||
{
|
||||
var reset = pendingSpan is null
|
||||
? null
|
||||
: new PendingSpanReset(
|
||||
pendingSpan.Speaker,
|
||||
pendingSpan.End,
|
||||
segment.Start - pendingSpan.End);
|
||||
pendingSpan = new PendingSpeakerSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
segment.End,
|
||||
[segment.Text]);
|
||||
return (pendingSpan.ToSegment(), reset);
|
||||
pendingSpan = extended;
|
||||
return (pendingSpan, null);
|
||||
}
|
||||
|
||||
pendingSpan = pendingSpan.Extend(segment);
|
||||
return (pendingSpan.ToSegment(), null);
|
||||
var previousSpan = pendingSpan;
|
||||
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
return (pendingSpan, previousSpan);
|
||||
}
|
||||
|
||||
private static SampleScore Score(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan minimumUninterruptedSpeechDuration)
|
||||
TimeSpan speechDuration,
|
||||
TimeSpan minimumSampleSpeechDuration)
|
||||
{
|
||||
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
|
||||
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
|
||||
var durationSeconds = speechDuration.TotalSeconds;
|
||||
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
|
||||
{
|
||||
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
|
||||
}
|
||||
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
|
||||
}
|
||||
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
|
||||
var wordScore = Math.Min(words / 60.0, 1);
|
||||
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
|
||||
segment.Text.TrimEnd().EndsWith('?') ||
|
||||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
|
||||
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
|
||||
|
||||
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
|
||||
|
||||
private sealed record PendingSpeakerSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
IReadOnlyList<string> TextParts)
|
||||
{
|
||||
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
|
||||
{
|
||||
return this with
|
||||
{
|
||||
End = segment.End > End ? segment.End : End,
|
||||
TextParts = TextParts.Append(segment.Text).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,25 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task AppendLinesAsync(
|
||||
TranscriptSession session,
|
||||
IReadOnlyList<string> lines,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (lines.Count == 0)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
return WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
() => File.AppendAllTextAsync(
|
||||
session.TranscriptPath,
|
||||
string.Concat(lines.Select(line => line + Environment.NewLine)),
|
||||
cancellationToken),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
|
||||
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
private const string NotificationGroup = "meeting-inactivity";
|
||||
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
|
||||
private readonly object registrationGate = new();
|
||||
private readonly object notificationGate = new();
|
||||
private bool registered;
|
||||
|
||||
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
|
||||
@@ -37,24 +37,27 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
try
|
||||
{
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
lock (notificationGate)
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -76,6 +79,36 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (pendingPrompts.IsEmpty)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
pendingPrompts.Clear();
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
|
||||
logger.LogInformation("Dismissed all native Windows inactivity notifications");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
@@ -83,29 +116,28 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (!registered)
|
||||
lock (notificationGate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (!registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
registered = false;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void EnsureRegistered()
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (registrationGate)
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
@@ -122,29 +154,25 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
string promptId,
|
||||
MeetingInactivityPromptRequest request)
|
||||
{
|
||||
var yesButton = new ToastButton()
|
||||
.SetContent("Yes")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "stop")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
var noButton = new ToastButton()
|
||||
.SetContent("No")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "continue")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
return new ToastContentBuilder()
|
||||
var builder = new ToastContentBuilder()
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText("Stop meeting?")
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
|
||||
.AddButton(yesButton)
|
||||
.AddButton(noButton);
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
|
||||
|
||||
foreach (var action in MeetingInactivityPromptActions.All)
|
||||
{
|
||||
builder.AddButton(new ToastButton()
|
||||
.SetContent(action.Content)
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", action.ResponseArgument)
|
||||
.SetBackgroundActivation());
|
||||
}
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||
@@ -177,10 +205,9 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return;
|
||||
}
|
||||
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
||||
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingInactivityPromptResponse.Stop
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||
? MeetingInactivityPromptActions.ParseResponse(responseValue)
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using NAudio.CoreAudioApi;
|
||||
using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
private readonly MicrophoneDeviceSelection selection;
|
||||
private readonly ILogger<WindowsMicrophoneDeviceProvider> logger;
|
||||
|
||||
public WindowsMicrophoneDeviceProvider(
|
||||
MicrophoneDeviceSelection selection,
|
||||
ILogger<WindowsMicrophoneDeviceProvider> logger)
|
||||
{
|
||||
this.selection = selection;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var enumerator = new MMDeviceEnumerator();
|
||||
return enumerator
|
||||
.EnumerateAudioEndPoints(DataFlow.Capture, DeviceState.Active)
|
||||
.Select(ToMicrophoneDevice)
|
||||
.OrderBy(device => device.Name, StringComparer.CurrentCultureIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not enumerate microphone capture endpoints");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
{
|
||||
var devices = GetAvailableMicrophones();
|
||||
return new MicrophoneDeviceSnapshot(
|
||||
devices,
|
||||
selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices));
|
||||
}
|
||||
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
{
|
||||
var current = GetMicrophoneSnapshot(options).Current;
|
||||
IWaveIn capture;
|
||||
if (current is null)
|
||||
{
|
||||
logger.LogInformation("Starting microphone capture from Windows default capture endpoint");
|
||||
capture = new WasapiCapture();
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})",
|
||||
current.Name,
|
||||
current.Id);
|
||||
using var enumerator = new MMDeviceEnumerator();
|
||||
capture = new WasapiCapture(enumerator.GetDevice(current.Id));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
capture.WaveFormat = new WaveFormat(
|
||||
options.Recording.SampleRate,
|
||||
16,
|
||||
options.Recording.Channels);
|
||||
return new NaudioCaptureAudioSource(capture, "microphone", logger);
|
||||
}
|
||||
catch
|
||||
{
|
||||
capture.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static MicrophoneDevice? GetDefaultMicrophone()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var enumerator = new MMDeviceEnumerator();
|
||||
return ToMicrophoneDevice(enumerator.GetDefaultAudioEndpoint(DataFlow.Capture, Role.Console));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static MicrophoneDevice ToMicrophoneDevice(MMDevice device)
|
||||
{
|
||||
return new MicrophoneDevice(device.ID, device.FriendlyName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
#if WINDOWS
|
||||
using System.Collections.Concurrent;
|
||||
using CommunityToolkit.WinUI.Notifications;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
|
||||
{
|
||||
private const string Source = "meeting-microphone-switch";
|
||||
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
|
||||
private readonly object gate = new();
|
||||
private bool registered;
|
||||
|
||||
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = Guid.NewGuid().ToString("N");
|
||||
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
try
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (!registered)
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||
registered = true;
|
||||
}
|
||||
|
||||
pending[id] = response;
|
||||
new ToastContentBuilder()
|
||||
.AddArgument("source", Source)
|
||||
.AddArgument("promptId", id)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText("New microphone available")
|
||||
.AddText($"Switch this meeting to {microphone.Name}?")
|
||||
.AddButton(new ToastButton().SetContent("Yes")
|
||||
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||
.AddButton(new ToastButton().SetContent("No")
|
||||
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||
.AddArgument("response", "no").SetBackgroundActivation())
|
||||
.Show(toast =>
|
||||
{
|
||||
toast.Tag = id;
|
||||
toast.Group = Source;
|
||||
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
|
||||
});
|
||||
}
|
||||
|
||||
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
|
||||
return await response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
pending.TryRemove(id, out _);
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.History.Remove(id, Source);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||
{
|
||||
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||
if (arguments.GetValueOrDefault("source") != Source ||
|
||||
!arguments.TryGetValue("promptId", out var id) ||
|
||||
!pending.TryRemove(id, out var response)) return;
|
||||
|
||||
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||
registered = false;
|
||||
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||
pending.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -1,14 +1,13 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
|
||||
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
|
||||
|
||||
@@ -39,20 +38,30 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
: options.Agent.Model;
|
||||
var key = ResolveApiKey(options);
|
||||
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
|
||||
using var httpClient = CreateHttpClient();
|
||||
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
|
||||
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
|
||||
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
|
||||
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
|
||||
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
|
||||
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
||||
}
|
||||
|
||||
var text = ParseOutputText(responseJson);
|
||||
var httpClient = CreateHttpClient();
|
||||
httpClient.BaseAddress = LiteLlmResponsesChatClient.NormalizeEndpoint(new Uri(endpoint));
|
||||
using var chatClient = new LiteLlmResponsesChatClient(
|
||||
httpClient,
|
||||
key,
|
||||
model,
|
||||
enableThinking: false,
|
||||
reasoningEffort: "none",
|
||||
reconnectionAttempts: options.Agent.ReconnectionAttempts,
|
||||
reconnectionDelay: options.Agent.ReconnectionDelay,
|
||||
logger: logger,
|
||||
firstRequestIsUser: false,
|
||||
useStreaming: options.Agent.UseStreaming);
|
||||
var response = await chatClient.GetResponseAsync(
|
||||
[
|
||||
new ChatMessage(
|
||||
ChatRole.User,
|
||||
[
|
||||
new TextContent(CreatePrompt(prompt, imageBytes)),
|
||||
new DataContent(imageBytes, "image/png")
|
||||
])
|
||||
],
|
||||
cancellationToken: cancellationToken);
|
||||
var text = response.Text ?? string.Empty;
|
||||
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
|
||||
return ParseOcrResult(text);
|
||||
}
|
||||
@@ -64,35 +73,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
: new HttpClient(httpMessageHandlerFactory());
|
||||
}
|
||||
|
||||
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
|
||||
{
|
||||
return new JsonObject
|
||||
{
|
||||
["model"] = model,
|
||||
["store"] = false,
|
||||
["input"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["role"] = "user",
|
||||
["content"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_text",
|
||||
["text"] = prompt
|
||||
},
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_image",
|
||||
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static string CreatePrompt(string prompt, byte[] imageBytes)
|
||||
{
|
||||
return TryReadPngDimensions(imageBytes, out var width, out var height)
|
||||
@@ -104,41 +84,69 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
private static ScreenshotOcrResult ParseOcrResult(string text)
|
||||
{
|
||||
ScreenshotCropCoordinates? crop = null;
|
||||
IReadOnlyList<string> attendees = [];
|
||||
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
|
||||
{
|
||||
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
|
||||
if (TryParseMetadata(match.Groups["json"].Value, out var parsedCrop, out var parsedAttendees))
|
||||
{
|
||||
crop = parsedCrop;
|
||||
attendees = parsedAttendees;
|
||||
return "";
|
||||
}
|
||||
|
||||
return match.Value;
|
||||
}).Trim();
|
||||
return new ScreenshotOcrResult(cleaned, crop);
|
||||
return new ScreenshotOcrResult(cleaned, crop, attendees);
|
||||
}
|
||||
|
||||
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
|
||||
private static bool TryParseMetadata(
|
||||
string json,
|
||||
out ScreenshotCropCoordinates? crop,
|
||||
out IReadOnlyList<string> attendees)
|
||||
{
|
||||
crop = null;
|
||||
attendees = [];
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
|
||||
cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind != JsonValueKind.Object ||
|
||||
!TryGetInt(cropElement, "x", out var x) ||
|
||||
!TryGetInt(cropElement, "y", out var y) ||
|
||||
!TryGetInt(cropElement, "width", out var width) ||
|
||||
!TryGetInt(cropElement, "height", out var height))
|
||||
if (document.RootElement.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
var hasMetadata = false;
|
||||
if (document.RootElement.TryGetProperty("attendees", out var attendeesElement))
|
||||
{
|
||||
hasMetadata = true;
|
||||
if (attendeesElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
attendees = MeetingAttendeeNames.NormalizeDistinct(attendeesElement
|
||||
.EnumerateArray()
|
||||
.Where(element => element.ValueKind == JsonValueKind.String)
|
||||
.Select(element => element.GetString() ?? ""));
|
||||
}
|
||||
}
|
||||
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement))
|
||||
{
|
||||
return hasMetadata;
|
||||
}
|
||||
|
||||
hasMetadata = true;
|
||||
if (cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind == JsonValueKind.Object &&
|
||||
TryGetInt(cropElement, "x", out var x) &&
|
||||
TryGetInt(cropElement, "y", out var y) &&
|
||||
TryGetInt(cropElement, "width", out var width) &&
|
||||
TryGetInt(cropElement, "height", out var height))
|
||||
{
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
@@ -181,46 +189,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
bytes[offset + 3];
|
||||
}
|
||||
|
||||
private static string ParseOutputText(string responseJson)
|
||||
{
|
||||
using var document = JsonDocument.Parse(responseJson);
|
||||
var root = document.RootElement;
|
||||
var parts = new List<string>();
|
||||
if (root.TryGetProperty("output_text", out var outputText) &&
|
||||
outputText.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(outputText.GetString()))
|
||||
{
|
||||
parts.Add(outputText.GetString()!);
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in output.EnumerateArray())
|
||||
{
|
||||
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var block in content.EnumerateArray())
|
||||
{
|
||||
if (block.TryGetProperty("type", out var type) &&
|
||||
type.GetString() == "output_text" &&
|
||||
block.TryGetProperty("text", out var text) &&
|
||||
text.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(text.GetString()))
|
||||
{
|
||||
parts.Add(text.GetString()!);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts.Count == 0
|
||||
? ""
|
||||
: string.Join(Environment.NewLine + Environment.NewLine, parts);
|
||||
}
|
||||
|
||||
private static string ResolveApiKey(MeetingAssistantOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
|
||||
@@ -249,17 +217,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
|
||||
}
|
||||
|
||||
private static Uri NormalizeEndpoint(Uri endpoint)
|
||||
{
|
||||
var value = endpoint.ToString().TrimEnd('/');
|
||||
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
value += "/v1";
|
||||
}
|
||||
|
||||
return new Uri(value + "/");
|
||||
}
|
||||
|
||||
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex JsonCodeBlockRegex();
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@ using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
@@ -22,7 +25,24 @@ public interface IScreenshotOcrClient
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
|
||||
public sealed record ScreenshotOcrResult
|
||||
{
|
||||
public ScreenshotOcrResult(
|
||||
string text,
|
||||
ScreenshotCropCoordinates? crop,
|
||||
IReadOnlyList<string>? attendees = null)
|
||||
{
|
||||
Text = text;
|
||||
Crop = crop;
|
||||
Attendees = attendees ?? [];
|
||||
}
|
||||
|
||||
public string Text { get; init; }
|
||||
|
||||
public ScreenshotCropCoordinates? Crop { get; init; }
|
||||
|
||||
public IReadOnlyList<string> Attendees { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
@@ -46,11 +66,36 @@ public interface IMeetingScreenshotService
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IMeetingNoteImageOcrService
|
||||
{
|
||||
Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IScreenshotOcrRetryRunner
|
||||
{
|
||||
Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingScreenshotCaptureResult(
|
||||
string ScreenshotPath,
|
||||
TimeSpan MeetingTimestamp,
|
||||
bool OcrStarted);
|
||||
|
||||
public sealed record MeetingScreenshotOcrRetryTriggerResult(
|
||||
string AssistantContextPath,
|
||||
string ScreenshotPath,
|
||||
string ScreenshotId);
|
||||
|
||||
public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
|
||||
|
||||
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
public static NoopMeetingScreenshotService Instance { get; } = new();
|
||||
@@ -80,12 +125,32 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
|
||||
{
|
||||
public static NoopMeetingNoteImageOcrService Instance { get; } = new();
|
||||
|
||||
public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed partial class MeetingScreenshotService :
|
||||
IMeetingScreenshotService,
|
||||
IScreenshotOcrRetryRunner,
|
||||
IMeetingNoteImageOcrService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IMeetingNoteStore meetingNoteStore;
|
||||
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
|
||||
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
||||
private readonly IScreenshotOcrClient ocrClient;
|
||||
private readonly ILogger<MeetingScreenshotService> logger;
|
||||
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
|
||||
@@ -94,11 +159,17 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
public MeetingScreenshotService(
|
||||
IActiveWindowScreenshotCapture screenshotCapture,
|
||||
IMeetingArtifactStore artifactStore,
|
||||
IMeetingNoteStore meetingNoteStore,
|
||||
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ILogger<MeetingScreenshotService> logger)
|
||||
ILogger<MeetingScreenshotService> logger,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
{
|
||||
this.screenshotCapture = screenshotCapture;
|
||||
this.artifactStore = artifactStore;
|
||||
this.meetingNoteStore = meetingNoteStore;
|
||||
this.attendeeCanonicalizer = attendeeCanonicalizer;
|
||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||
this.ocrClient = ocrClient;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -135,7 +206,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.CapturedScreenshot,
|
||||
ocrCancellation.Token));
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -212,6 +289,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(screenshotId) ||
|
||||
string.IsNullOrWhiteSpace(screenshotPath) ||
|
||||
!File.Exists(screenshotPath) ||
|
||||
!File.Exists(artifacts.AssistantContextPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var replaced = await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
"_OCR retry pending..._",
|
||||
cancellationToken,
|
||||
preserveMarkers: true,
|
||||
appendWhenMissing: false);
|
||||
if (!replaced)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.CapturedScreenshot,
|
||||
ocrCancellation.Token));
|
||||
return new MeetingScreenshotOcrRetryTriggerResult(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
screenshotId);
|
||||
}
|
||||
|
||||
public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
|
||||
{
|
||||
return new MeetingNoteImageOcrQueueResult(0);
|
||||
}
|
||||
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||
var embeds = FindMeetingNoteImageEmbeds(
|
||||
meetingNote.UserNotes,
|
||||
artifacts.MeetingNotePath,
|
||||
options);
|
||||
var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
|
||||
foreach (var embed in embeds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var ocrId = Guid.NewGuid().ToString("N");
|
||||
await artifactStore.AppendAssistantContextAsync(
|
||||
artifacts,
|
||||
CreateMeetingNoteImageMarkdown(
|
||||
ocrId,
|
||||
embed.OriginalEmbed,
|
||||
embed.ImagePath,
|
||||
artifacts.AssistantContextPath),
|
||||
cancellationToken);
|
||||
queuedEmbeds.Add((embed, ocrId));
|
||||
}
|
||||
|
||||
foreach (var (embed, ocrId) in queuedEmbeds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
embed.ImagePath,
|
||||
ocrId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.MeetingNoteImage,
|
||||
ocrCancellation.Token));
|
||||
}
|
||||
|
||||
return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
|
||||
}
|
||||
|
||||
private async Task<string> SaveScreenshotAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
byte[] pngBytes,
|
||||
@@ -234,6 +405,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
ScreenshotOcrPolicy policy,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
|
||||
@@ -248,17 +420,27 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
: options.Screenshots.Ocr.Prompt,
|
||||
options,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = policy.AllowCrop
|
||||
? await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token)
|
||||
: "";
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
cropMarkdown +
|
||||
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
|
||||
CancellationToken.None);
|
||||
if (policy.AddAttendees)
|
||||
{
|
||||
await AddOcrAttendeesAsync(
|
||||
artifacts,
|
||||
result.Attendees,
|
||||
options,
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -269,8 +451,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
CancellationToken.None);
|
||||
CreateOcrFailureMarkdown(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
policy.IncludeRetryLink),
|
||||
CancellationToken.None,
|
||||
preserveMarkers: true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -278,16 +467,93 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
CancellationToken.None);
|
||||
CreateOcrFailureMarkdown(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
policy.IncludeRetryLink),
|
||||
CancellationToken.None,
|
||||
preserveMarkers: true);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplaceOcrPlaceholderAsync(
|
||||
private async Task AddOcrAttendeesAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
IReadOnlyList<string> attendees,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
|
||||
if (additions.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||
var transformedAdditions = await TransformAttendeesAsync(
|
||||
artifacts,
|
||||
additions,
|
||||
options,
|
||||
cancellationToken);
|
||||
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
|
||||
meetingNote.Frontmatter.Attendees.Concat(transformedAdditions).ToList(),
|
||||
cancellationToken);
|
||||
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
meetingNote.Frontmatter.Attendees = canonicalized.ToList();
|
||||
await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
|
||||
additions.Count,
|
||||
artifacts.MeetingNotePath);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
|
||||
artifacts.MeetingNotePath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<string>> TransformAttendeesAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
IReadOnlyList<string> attendees,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var transformed = new List<string>();
|
||||
foreach (var attendee in attendees)
|
||||
{
|
||||
var value = await meetingWorkflowEngine.TransformAttendeeAsync(
|
||||
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
|
||||
options,
|
||||
cancellationToken);
|
||||
var normalized = MeetingAttendeeNames.NormalizeDisplayName(value);
|
||||
if (!string.IsNullOrWhiteSpace(normalized) &&
|
||||
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
transformed.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return transformed;
|
||||
}
|
||||
|
||||
private async Task<bool> ReplaceOcrPlaceholderAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotId,
|
||||
string replacement,
|
||||
CancellationToken cancellationToken)
|
||||
CancellationToken cancellationToken,
|
||||
bool preserveMarkers = false,
|
||||
bool appendWhenMissing = true)
|
||||
{
|
||||
var fileLock = contextFileLocks.GetOrAdd(
|
||||
NormalizeContextKey(assistantContextPath),
|
||||
@@ -297,28 +563,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
if (!File.Exists(assistantContextPath))
|
||||
{
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
||||
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
var startMarker = CreateOcrStartMarker(screenshotId);
|
||||
var endMarker = CreateOcrEndMarker(screenshotId);
|
||||
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
|
||||
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
|
||||
if (startIndex < 0 || endIndex < startIndex)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
return;
|
||||
if (appendWhenMissing)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
endIndex += endMarker.Length;
|
||||
var updated = content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
var updated = preserveMarkers
|
||||
? content[..startIndex] +
|
||||
startMarker +
|
||||
Environment.NewLine +
|
||||
replacement.TrimEnd() +
|
||||
Environment.NewLine +
|
||||
endMarker +
|
||||
content[endIndex..]
|
||||
: content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -326,6 +605,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateOcrFailureMarkdown(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
string status,
|
||||
bool includeRetryLink = true)
|
||||
{
|
||||
var markdown = status.TrimEnd();
|
||||
if (includeRetryLink)
|
||||
{
|
||||
markdown += Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
|
||||
options.Api.PublicBaseUrl,
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId);
|
||||
}
|
||||
|
||||
return markdown;
|
||||
}
|
||||
|
||||
private async Task<string> TryCreateCropMarkdownAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotPath,
|
||||
@@ -435,13 +737,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
return content +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
$"<!-- screenshot-ocr:{screenshotId} -->" +
|
||||
Environment.NewLine +
|
||||
"_OCR pending..._" +
|
||||
Environment.NewLine +
|
||||
$"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
CreateOcrBlock(screenshotId, "_OCR pending..._");
|
||||
}
|
||||
|
||||
private static string CreateMeetingNoteImageMarkdown(
|
||||
string ocrId,
|
||||
string originalEmbed,
|
||||
string imagePath,
|
||||
string assistantContextPath)
|
||||
{
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
imagePath));
|
||||
return "## Meeting Note Image" +
|
||||
Environment.NewLine +
|
||||
"Image from meeting note." +
|
||||
Environment.NewLine +
|
||||
$"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
|
||||
Environment.NewLine +
|
||||
$"" +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
CreateOcrBlock(ocrId, "_OCR pending..._");
|
||||
}
|
||||
|
||||
private static string CreateOcrBlock(string screenshotId, string content)
|
||||
{
|
||||
return CreateOcrStartMarker(screenshotId) +
|
||||
Environment.NewLine +
|
||||
content.TrimEnd() +
|
||||
Environment.NewLine +
|
||||
CreateOcrEndMarker(screenshotId);
|
||||
}
|
||||
|
||||
private static string CreateOcrStartMarker(string screenshotId)
|
||||
{
|
||||
return $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static string CreateOcrEndMarker(string screenshotId)
|
||||
{
|
||||
return $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
|
||||
string userNotes,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(userNotes))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var result = new List<MeetingNoteImageEmbed>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
|
||||
{
|
||||
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
||||
}
|
||||
|
||||
foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
|
||||
{
|
||||
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void AddResolvedEmbed(
|
||||
List<MeetingNoteImageEmbed> embeds,
|
||||
HashSet<string> seen,
|
||||
string originalEmbed,
|
||||
string target,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
|
||||
if (imagePath is null || !seen.Add(imagePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
|
||||
}
|
||||
|
||||
private static string? ResolveMeetingNoteImagePath(
|
||||
string target,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var cleaned = CleanEmbedTarget(target);
|
||||
if (string.IsNullOrWhiteSpace(cleaned) ||
|
||||
cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
|
||||
cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
|
||||
!IsSupportedImagePath(cleaned))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var candidates = new List<string>();
|
||||
if (Path.IsPathRooted(cleaned))
|
||||
{
|
||||
candidates.Add(VaultPath.Resolve(cleaned));
|
||||
}
|
||||
else
|
||||
{
|
||||
candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
|
||||
candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
|
||||
}
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
|
||||
{
|
||||
try
|
||||
{
|
||||
var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
|
||||
return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
|
||||
.FirstOrDefault(IsSupportedImagePath);
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CleanEmbedTarget(string target)
|
||||
{
|
||||
var cleaned = target.Trim();
|
||||
if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
|
||||
cleaned.EndsWith(">", StringComparison.Ordinal) &&
|
||||
cleaned.Length > 1)
|
||||
{
|
||||
cleaned = cleaned[1..^1].Trim();
|
||||
}
|
||||
|
||||
var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
|
||||
if (pipeIndex >= 0)
|
||||
{
|
||||
cleaned = cleaned[..pipeIndex].Trim();
|
||||
}
|
||||
|
||||
var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
|
||||
if (hashIndex >= 0)
|
||||
{
|
||||
cleaned = cleaned[..hashIndex].Trim();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
|
||||
}
|
||||
catch (UriFormatException)
|
||||
{
|
||||
return cleaned.Replace('/', Path.DirectorySeparatorChar);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsSupportedImagePath(string path)
|
||||
{
|
||||
return Path.GetExtension(path).ToLowerInvariant() switch
|
||||
{
|
||||
".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex ObsidianImageEmbedRegex();
|
||||
|
||||
[GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex MarkdownImageEmbedRegex();
|
||||
|
||||
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(
|
||||
@@ -479,4 +957,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
}
|
||||
|
||||
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
|
||||
|
||||
private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
|
||||
{
|
||||
public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
|
||||
AllowCrop: true,
|
||||
AddAttendees: true,
|
||||
IncludeRetryLink: true);
|
||||
|
||||
public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
|
||||
AllowCrop: false,
|
||||
AddAttendees: false,
|
||||
IncludeRetryLink: false);
|
||||
}
|
||||
|
||||
private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
|
||||
}
|
||||
|
||||
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
return [];
|
||||
}
|
||||
|
||||
return await finalizer.FinalizeAsync(
|
||||
return await finalizer.FinalizeEnabledAsync(
|
||||
wavPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
||||
options.Diarization,
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
ResemblyzerVoiceClusterMatcher clusterMatcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentificationService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.encoder = encoder;
|
||||
this.clusterMatcher = clusterMatcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ApplySpeakerOverrideAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string sourceSpeaker,
|
||||
string targetSpeaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled ||
|
||||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
|
||||
string.IsNullOrWhiteSpace(targetSpeaker) ||
|
||||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = sourceSpeaker.Trim();
|
||||
var targetName = targetSpeaker.Trim();
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var sourceCandidate = identities
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
|
||||
if (target is null && sourceCandidate is null && vectors.Count == 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
|
||||
sourceLabel,
|
||||
targetName);
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (target is null)
|
||||
{
|
||||
target = sourceCandidate ?? new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now
|
||||
};
|
||||
if (target.Id == 0)
|
||||
{
|
||||
context.SpeakerIdentities.Add(target);
|
||||
}
|
||||
}
|
||||
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||
{
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
sourceCandidate,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||
}
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
|
||||
outlierPruner.AddAndPrune(
|
||||
target,
|
||||
vectors,
|
||||
now);
|
||||
target.UpdatedAt = now;
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
target.References,
|
||||
sourceLabel,
|
||||
targetName,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteSpeakerIdentityAsync(
|
||||
string identity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities.FirstOrDefault(candidate =>
|
||||
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
|
||||
if (target is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
context.SpeakerIdentities.Remove(target);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
bool final,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return EmptyResult(request.Segments);
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var knownMappings = request.KnownSpeakerMappings ??
|
||||
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
if (final && knownMappings.Count > 0)
|
||||
{
|
||||
await PersistMappedSpeakerEvidenceAsync(
|
||||
context,
|
||||
request,
|
||||
knownMappings,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
var identifiedNames = knownMappings.Values
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var segmentSpeaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
identifiedNames.Add(segmentSpeaker.Trim());
|
||||
}
|
||||
|
||||
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var pendingAudits = new List<PendingIdentificationAudit>();
|
||||
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAutomaticVectorsAsync(
|
||||
request,
|
||||
speaker,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
|
||||
speaker,
|
||||
vectors.Count,
|
||||
resemblyzerOptions.RequiredVectorsPerSpeaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (identity, decision) = await FindMatchAsync(
|
||||
context,
|
||||
attendees,
|
||||
identifiedNames,
|
||||
vectors,
|
||||
cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, vectors));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousCanonicalName = identity.CanonicalName;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees);
|
||||
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var displayName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(displayName))
|
||||
{
|
||||
mappings[speaker] = displayName;
|
||||
identifiedNames.Add(displayName);
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
identifiedNames.Add(acceptedName);
|
||||
}
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
displayName,
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
LearnUnmatchedSpeakers(
|
||||
context,
|
||||
request.MeetingNote,
|
||||
attendees,
|
||||
matchedAcceptedNames,
|
||||
unmatchedSpeakers,
|
||||
pendingAudits);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await AppendAuditsAsync(pendingAudits, cancellationToken);
|
||||
var relabeled = request.Segments
|
||||
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
|
||||
? segment with { Speaker = name }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> identifiedNames,
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var identities = await LoadIdentities(context)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var candidates = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
|
||||
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
|
||||
candidate.Identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
candidate.Identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
var match = clusterMatcher.Match(queryVectors, candidates);
|
||||
return (
|
||||
match.IdentityId is { } identityId
|
||||
? identities.Single(identity => identity.Id == identityId)
|
||||
: null,
|
||||
match);
|
||||
}
|
||||
|
||||
private void LearnUnmatchedSpeakers(
|
||||
SpeakerIdentityDbContext context,
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> matchedAcceptedNames,
|
||||
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
|
||||
ICollection<PendingIdentificationAudit> pendingAudits)
|
||||
{
|
||||
var candidates = attendees
|
||||
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (speaker, vectors) in unmatchedSpeakers)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates
|
||||
.Select(name => new SpeakerCandidateName { Name = name })
|
||||
.ToList(),
|
||||
References = [CreateReference(meetingNote, now)]
|
||||
};
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
logger.LogInformation(
|
||||
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
|
||||
speaker,
|
||||
identity.VoiceVectors.Count,
|
||||
string.Join(", ", candidates));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AppendAuditsAsync(
|
||||
IEnumerable<PendingIdentificationAudit> pendingAudits,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
audit.Identity.References,
|
||||
audit.Speaker,
|
||||
audit.Name,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (samples.Count < requiredCount)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task PersistMappedSpeakerEvidenceAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
SpeakerIdentificationRequest request,
|
||||
IReadOnlyDictionary<string, string> knownMappings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
foreach (var (speaker, mappedName) in knownMappings)
|
||||
{
|
||||
var identity = identities
|
||||
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
|
||||
.OrderBy(candidate =>
|
||||
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(candidate => candidate.Id)
|
||||
.FirstOrDefault();
|
||||
if (identity is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
|
||||
speaker,
|
||||
mappedName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
|
||||
if (vectors.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
var vectorUpdate = outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
|
||||
vectorUpdate.AddedCount,
|
||||
speaker,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback: true,
|
||||
cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
int maxSamples,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var suppliedSamples = request.Samples?
|
||||
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(sample => sample.WavBytes.Length > 0)
|
||||
.OrderByDescending(sample => sample.Score)
|
||||
.Take(maxSamples)
|
||||
.ToList()
|
||||
?? [];
|
||||
var wavSamples = suppliedSamples
|
||||
.Select(sample => sample.WavBytes)
|
||||
.ToList();
|
||||
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
|
||||
{
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
request.Segments.Count);
|
||||
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
|
||||
{
|
||||
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
span,
|
||||
cancellationToken);
|
||||
if (wavBytes.Length > 0)
|
||||
{
|
||||
wavSamples.Add(wavBytes);
|
||||
}
|
||||
|
||||
if (wavSamples.Count >= maxSamples)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
|
||||
wavSamples.Count,
|
||||
maxSamples,
|
||||
speaker,
|
||||
suppliedSamples.Count,
|
||||
wavSamples.Count - suppliedSamples.Count);
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
private static bool OverlapsSuppliedSample(
|
||||
IReadOnlyList<TranscriptionSegment> span,
|
||||
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
|
||||
{
|
||||
if (span.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var start = span[0].Start;
|
||||
var end = span[^1].End;
|
||||
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
|
||||
}
|
||||
|
||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
|
||||
}
|
||||
|
||||
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
private sealed record PendingIdentificationAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string Speaker,
|
||||
string Name);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ResemblyzerVoiceClusterMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ResemblyzerVoiceClusterMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||
TimeSpan? recentIdentityAge,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var recentIds = identities
|
||||
.Where(identity => identity.CreatedAt >= cutoff)
|
||||
.Select(identity => identity.Id)
|
||||
.ToList();
|
||||
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var attempts = 0;
|
||||
var mergedPairs = 0;
|
||||
var pendingAudits = new List<PendingMergeAudit>();
|
||||
|
||||
foreach (var sourceId in recentIds)
|
||||
{
|
||||
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
|
||||
source,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)
|
||||
.Take(required * 2)
|
||||
.ToList();
|
||||
if (sourceVectors.Count < required * 2)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
|
||||
source.Id,
|
||||
sourceVectors.Count,
|
||||
required * 2);
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidates = identities
|
||||
.Where(identity => identity.Id != source.Id)
|
||||
.Where(identity => identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(identity => new ResemblyzerVoiceVectorCandidate(
|
||||
identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
|
||||
if (!first.Accepted || first.IdentityId is not { } targetId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
|
||||
if (!second.Accepted || second.IdentityId != targetId)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
|
||||
source.Id,
|
||||
first.IdentityId,
|
||||
second.IdentityId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
|
||||
if (target is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
pendingAudits.Add(new PendingMergeAudit(
|
||||
target,
|
||||
targetName,
|
||||
sourceName));
|
||||
context.SpeakerIdentities.Remove(source);
|
||||
identities.Remove(source);
|
||||
mergedPairs++;
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
|
||||
audit.Identity.References,
|
||||
audit.TargetName,
|
||||
audit.SourceName,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return new SpeakerIdentityMergeResult(
|
||||
recentIds.Count,
|
||||
identities.Count,
|
||||
attempts,
|
||||
mergedPairs);
|
||||
}
|
||||
|
||||
private sealed record PendingMergeAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string TargetName,
|
||||
string SourceName);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorCandidate(
|
||||
int IdentityId,
|
||||
IReadOnlyList<float[]> Vectors);
|
||||
|
||||
public sealed record ResemblyzerVoiceClusterMatchResult(
|
||||
int? IdentityId,
|
||||
string Reason,
|
||||
double Cohesion,
|
||||
double? BestSimilarity,
|
||||
double? RunnerUpSimilarity)
|
||||
{
|
||||
public bool Accepted => IdentityId.HasValue;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcher
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
|
||||
|
||||
public ResemblyzerVoiceClusterMatcher(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceClusterMatcher> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceClusterMatchResult Match(
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
|
||||
{
|
||||
var requiredCount = options.RequiredVectorsPerSpeaker;
|
||||
if (queryVectors.Count < requiredCount)
|
||||
{
|
||||
return Reject(
|
||||
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
|
||||
cohesion: 0,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var normalizedQuery = queryVectors
|
||||
.Take(requiredCount)
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var cohesion = MeanPairwiseCosine(normalizedQuery);
|
||||
if (cohesion < options.MinimumClusterCohesion)
|
||||
{
|
||||
return Reject(
|
||||
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
|
||||
cohesion,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var scored = candidates
|
||||
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
|
||||
.Where(candidate => candidate is not null)
|
||||
.Select(candidate => candidate!.Value)
|
||||
.OrderByDescending(candidate => candidate.Similarity)
|
||||
.ThenBy(candidate => candidate.IdentityId)
|
||||
.ToList();
|
||||
if (scored.Count == 0)
|
||||
{
|
||||
return Reject("no compatible identity vectors were available", cohesion, null, null);
|
||||
}
|
||||
|
||||
var best = scored[0];
|
||||
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
|
||||
if (best.Similarity < options.MinimumIdentitySimilarity)
|
||||
{
|
||||
return Reject(
|
||||
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
if (runnerUp is { } runnerUpSimilarity &&
|
||||
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
|
||||
{
|
||||
return Reject(
|
||||
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
|
||||
best.IdentityId,
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
best.IdentityId,
|
||||
"accepted",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
private ResemblyzerVoiceClusterMatchResult Reject(
|
||||
string reason,
|
||||
double cohesion,
|
||||
double? bestSimilarity,
|
||||
double? runnerUpSimilarity)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
null,
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
private (int IdentityId, double Similarity)? ScoreCandidate(
|
||||
IReadOnlyList<float[]> normalizedQuery,
|
||||
ResemblyzerVoiceVectorCandidate candidate)
|
||||
{
|
||||
if (candidate.Vectors.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var normalizedCandidate = candidate.Vectors
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
|
||||
var similarities = normalizedQuery
|
||||
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
|
||||
.Order()
|
||||
.ToList();
|
||||
return (candidate.IdentityId, Median(similarities));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
|
||||
candidate.IdentityId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] Centroid(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var centroid = new float[ResemblyzerVectorContract.Dimensions];
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] += vector[index];
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] /= vectors.Count;
|
||||
}
|
||||
|
||||
return centroid;
|
||||
}
|
||||
|
||||
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
if (vectors.Count < 2)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
double total = 0;
|
||||
var pairs = 0;
|
||||
for (var first = 0; first < vectors.Count - 1; first++)
|
||||
{
|
||||
for (var second = first + 1; second < vectors.Count; second++)
|
||||
{
|
||||
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
|
||||
pairs++;
|
||||
}
|
||||
}
|
||||
|
||||
return total / pairs;
|
||||
}
|
||||
|
||||
private static double Median(IReadOnlyList<double> sortedValues)
|
||||
{
|
||||
var middle = sortedValues.Count / 2;
|
||||
return sortedValues.Count % 2 == 0
|
||||
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
|
||||
: sortedValues[middle];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorUpdateResult(
|
||||
int AddedCount,
|
||||
int RemovedCount)
|
||||
{
|
||||
public bool Changed => AddedCount > 0 || RemovedCount > 0;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPruner
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
|
||||
|
||||
public ResemblyzerVoiceVectorOutlierPruner(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public int Prune(SpeakerIdentity identity)
|
||||
{
|
||||
var compatible = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
|
||||
identity,
|
||||
options.ModelId,
|
||||
logger))
|
||||
{
|
||||
try
|
||||
{
|
||||
compatible.Add(new DecodedSpeakerVoiceVector(
|
||||
entry.Stored,
|
||||
SpeakerVoiceVectors.Normalize(
|
||||
entry.Vector,
|
||||
$"Stored voice vector {entry.Stored.Id}")));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
|
||||
entry.Stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
var minimumVectors = options.OutlierPruningMinimumVectors;
|
||||
if (compatible.Count < minimumVectors)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
|
||||
var ranked = clusters
|
||||
.GroupBy(cluster => cluster)
|
||||
.Where(group => group.Key > 0)
|
||||
.Select(group => new { Id = group.Key, Count = group.Count() })
|
||||
.OrderByDescending(group => group.Count)
|
||||
.ThenBy(group => group.Id)
|
||||
.ToList();
|
||||
if (ranked.Count == 0)
|
||||
{
|
||||
return Skip(identity, "no dense cluster was found");
|
||||
}
|
||||
|
||||
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
|
||||
{
|
||||
return Skip(identity, "no uniquely largest dense cluster was found");
|
||||
}
|
||||
|
||||
var dominant = ranked[0];
|
||||
var ratio = (double)dominant.Count / compatible.Count;
|
||||
if (ratio < options.OutlierPruningMinimumClusterRatio)
|
||||
{
|
||||
return Skip(
|
||||
identity,
|
||||
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
|
||||
}
|
||||
|
||||
var removed = compatible
|
||||
.Where((_, index) => clusters[index] != dominant.Id)
|
||||
.Select(item => item.Stored)
|
||||
.ToList();
|
||||
foreach (var vector in removed)
|
||||
{
|
||||
identity.VoiceVectors.Remove(vector);
|
||||
}
|
||||
|
||||
if (removed.Count > 0)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
|
||||
removed.Count,
|
||||
identity.Id,
|
||||
dominant.Count,
|
||||
compatible.Count);
|
||||
return removed.Count;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var removedBeforeAdding = Prune(identity);
|
||||
var added = SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
options.ModelId,
|
||||
options.MaxVectorsPerIdentity,
|
||||
createdAt);
|
||||
var removedAfterAdding = Prune(identity);
|
||||
var result = new ResemblyzerVoiceVectorUpdateResult(
|
||||
added,
|
||||
removedBeforeAdding + removedAfterAdding);
|
||||
if (result.Changed)
|
||||
{
|
||||
identity.UpdatedAt = createdAt;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var neighbors = Enumerable.Range(0, vectors.Count)
|
||||
.Select(index => Enumerable.Range(0, vectors.Count)
|
||||
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
|
||||
.ToList())
|
||||
.ToList();
|
||||
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
|
||||
var labels = new int[vectors.Count];
|
||||
var clusterId = 0;
|
||||
|
||||
for (var point = 0; point < vectors.Count; point++)
|
||||
{
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
labels[point] = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
clusterId++;
|
||||
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
|
||||
}
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
private static void ExpandCluster(
|
||||
int seed,
|
||||
int clusterId,
|
||||
IReadOnlyList<List<int>> neighbors,
|
||||
int[] labels,
|
||||
int minimumNeighbors)
|
||||
{
|
||||
labels[seed] = clusterId;
|
||||
var pending = new Queue<int>(neighbors[seed]);
|
||||
var queued = neighbors[seed].ToHashSet();
|
||||
while (pending.TryDequeue(out var point))
|
||||
{
|
||||
if (labels[point] == -1)
|
||||
{
|
||||
labels[point] = clusterId;
|
||||
}
|
||||
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
labels[point] = clusterId;
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var neighbor in neighbors[point])
|
||||
{
|
||||
if (queued.Add(neighbor))
|
||||
{
|
||||
pending.Enqueue(neighbor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int Skip(
|
||||
SpeakerIdentity identity,
|
||||
string reason)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
|
||||
identity.Id,
|
||||
reason);
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedService : BackgroundService
|
||||
{
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly bool speakerIdentificationEnabled;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
|
||||
public ResemblyzerWarmupHostedService(
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerWarmupHostedService> logger)
|
||||
{
|
||||
this.encoder = encoder;
|
||||
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!speakerIdentificationEnabled || !options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Yield();
|
||||
try
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
await encoder.WarmUpAsync(stoppingToken);
|
||||
logger.LogInformation(
|
||||
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
|
||||
VaultPath.Resolve(options.RuntimeFolder));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
|
||||
|
||||
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
||||
|
||||
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
|
||||
|
||||
public List<SpeakerIdentityReference> References { get; set; } = [];
|
||||
|
||||
[NotMapped]
|
||||
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
|
||||
string.IsNullOrWhiteSpace(reference.TranscriptPath);
|
||||
}
|
||||
|
||||
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerVoiceVector
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public string ModelId { get; set; } = "";
|
||||
|
||||
public int Dimensions { get; set; }
|
||||
|
||||
public byte[] VectorBytes { get; set; } = [];
|
||||
|
||||
public string Fingerprint { get; set; } = "";
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerAlias
|
||||
{
|
||||
public int Id { get; set; }
|
||||
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
|
||||
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
||||
|
||||
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
|
||||
|
||||
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.VoiceVectors)
|
||||
.WithOne(vector => vector.SpeakerIdentity)
|
||||
.HasForeignKey(vector => vector.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.References)
|
||||
.WithOne(reference => reference.SpeakerIdentity)
|
||||
.HasForeignKey(reference => reference.SpeakerIdentityId)
|
||||
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
|
||||
|
||||
modelBuilder.Entity<SpeakerIdentityReference>()
|
||||
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
|
||||
.IsUnique();
|
||||
|
||||
@@ -114,11 +114,7 @@ public sealed class PassthroughSpeakerIdentityAttendeeCanonicalizer : ISpeakerId
|
||||
IReadOnlyList<string> attendees,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var distinctAttendees = attendees
|
||||
.Select(attendee => attendee.Trim())
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
var distinctAttendees = MeetingAttendeeNames.NormalizeDistinct(attendees);
|
||||
return Task.FromResult<IReadOnlyList<string>>(distinctAttendees);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
||||
|
||||
public SpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeInto(
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker);
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
options.Resemblyzer.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
||||
source.Id,
|
||||
|
||||
@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityMerger
|
||||
{
|
||||
public static void MergeIntoAndPrune(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
|
||||
{
|
||||
outlierPruner.Prune(target);
|
||||
outlierPruner.Prune(source);
|
||||
MergeInto(target, source, maxSnippets, maxVoiceVectors);
|
||||
outlierPruner.Prune(target);
|
||||
}
|
||||
|
||||
public static void MergeInto(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets)
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors = int.MaxValue)
|
||||
{
|
||||
AddAlias(target, source.CanonicalName);
|
||||
foreach (var alias in source.Aliases)
|
||||
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
|
||||
.ToList();
|
||||
target.Snippets.Clear();
|
||||
target.Snippets.AddRange(retainedSnippets);
|
||||
|
||||
var retainedVectors = target.VoiceVectors
|
||||
.Concat(source.VoiceVectors)
|
||||
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
|
||||
.OrderByDescending(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Take(Math.Max(1, maxVoiceVectors))
|
||||
.Select(vector => new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentity = target,
|
||||
ModelId = vector.ModelId,
|
||||
Dimensions = vector.Dimensions,
|
||||
VectorBytes = vector.VectorBytes.ToArray(),
|
||||
Fingerprint = vector.Fingerprint,
|
||||
CreatedAt = vector.CreatedAt
|
||||
})
|
||||
.ToList();
|
||||
target.VoiceVectors.Clear();
|
||||
target.VoiceVectors.AddRange(retainedVectors);
|
||||
target.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityNaming
|
||||
{
|
||||
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[] { identity.CanonicalName }
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool MatchesAnyAcceptedName(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
public static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(MeetingAttendeeNames.NormalizeDisplayName)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool UpdateCandidateNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
var resetToAttendees = intersection.Count == 0;
|
||||
SetCandidates(identity, resetToAttendees ? attendees : intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
}
|
||||
|
||||
return resetToAttendees;
|
||||
}
|
||||
|
||||
public static void SetCandidates(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
|
||||
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
|
||||
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
|
||||
"SpeakerIdentityId" INTEGER NOT NULL,
|
||||
"ModelId" TEXT NOT NULL,
|
||||
"Dimensions" INTEGER NOT NULL,
|
||||
"VectorBytes" BLOB NOT NULL,
|
||||
"Fingerprint" TEXT NOT NULL,
|
||||
"CreatedAt" TEXT NOT NULL,
|
||||
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
|
||||
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
|
||||
);
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
||||
|
||||
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
target.UpdatedAt = now;
|
||||
ResetCandidates(target, [targetName]);
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
AddMeetingReference(target, meetingReference);
|
||||
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(speaker.Trim());
|
||||
}
|
||||
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
speakerMappings[speaker] = speakerName;
|
||||
alreadyIdentifiedNames.Add(speakerName);
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(acceptedName);
|
||||
}
|
||||
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
speakerName,
|
||||
GetAcceptedNames(identity).ToList()));
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = MatchesAttendees(identity, attendees),
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration);
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration);
|
||||
logger.LogInformation(
|
||||
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
||||
operation,
|
||||
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Include(identity => identity.References)
|
||||
.ToListAsync(cancellationToken);
|
||||
return identities
|
||||
.Where(identity => GetAcceptedNames(identity).Contains(name))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.ToListAsync(cancellationToken);
|
||||
return candidates
|
||||
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => identity.CandidateNames.Any(candidate =>
|
||||
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
|
||||
identity.Aliases.Any(alias =>
|
||||
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
||||
}
|
||||
|
||||
private static bool IsSameReference(
|
||||
SpeakerIdentityReference first,
|
||||
SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool MatchesAcceptedNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
private static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
|
||||
}
|
||||
|
||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (intersection.Count == 0)
|
||||
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
if (resetToAttendees)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
||||
identity.Id,
|
||||
FormatNames(attendees));
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
|
||||
identity.Id,
|
||||
FormatNames(currentCandidates),
|
||||
FormatNames(attendees),
|
||||
FormatNames(intersection));
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
||||
identity.Id,
|
||||
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
}
|
||||
}
|
||||
|
||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
|
||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
identity.CanonicalName
|
||||
}
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
|
||||
}
|
||||
|
||||
private static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record SpeakerSampleCollectionPolicy(
|
||||
int MinimumRetainedSamples,
|
||||
bool RequireNonOverlappingSamples)
|
||||
{
|
||||
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
|
||||
|
||||
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
|
||||
{
|
||||
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
|
||||
}
|
||||
|
||||
public int ResolveRetainedSampleLimit(int configuredLimit)
|
||||
{
|
||||
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SpeakerSampleDurationConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
SpeakerIdentificationOptions options,
|
||||
string context)
|
||||
{
|
||||
ValidateOrThrow(
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
context);
|
||||
}
|
||||
|
||||
public static void ValidateOrThrow(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(minimumDuration, maximumDuration);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (minimumDuration < TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
|
||||
}
|
||||
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
|
||||
}
|
||||
|
||||
return minimumDuration > maximumDuration
|
||||
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ResemblyzerSpeakerRecognitionConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(options);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
|
||||
{
|
||||
if (options.RequiredVectorsPerSpeaker < 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
|
||||
}
|
||||
|
||||
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumNeighbors < 1 ||
|
||||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
|
||||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumClusterCohesion))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumIdentitySimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
return !double.IsFinite(options.MinimumSimilarityMargin) ||
|
||||
options.MinimumSimilarityMargin is < 0 or > 2
|
||||
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
|
||||
: null;
|
||||
}
|
||||
|
||||
private static bool IsCosine(double value)
|
||||
{
|
||||
return double.IsFinite(value) && value is >= -1 and <= 1;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
|
||||
: IValidateOptions<MeetingAssistantOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
|
||||
{
|
||||
var error = SpeakerSampleDurationConfiguration.GetError(
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleDuration) ??
|
||||
ResemblyzerSpeakerRecognitionConfiguration.GetError(
|
||||
options.SpeakerIdentification.Resemblyzer);
|
||||
return error is null
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record SpeakerSampleSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
TimeSpan SpeechDuration,
|
||||
TimeSpan MaximumDuration,
|
||||
IReadOnlyList<TranscriptionSegment> Segments)
|
||||
{
|
||||
public static SpeakerSampleSpan? Create(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(maximumDuration),
|
||||
maximumDuration,
|
||||
"Speaker sample maximum duration must be greater than zero.");
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
|
||||
return boundedEnd > segment.Start
|
||||
? new SpeakerSampleSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
boundedEnd,
|
||||
boundedEnd - segment.Start,
|
||||
maximumDuration,
|
||||
[segment with { End = boundedEnd }])
|
||||
: null;
|
||||
}
|
||||
|
||||
public bool TryExtend(
|
||||
TranscriptionSegment segment,
|
||||
out SpeakerSampleSpan extended)
|
||||
{
|
||||
extended = this;
|
||||
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
|
||||
segment.Start >= Start + MaximumDuration)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, Start + MaximumDuration);
|
||||
if (boundedEnd <= segment.Start)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var uncoveredStart = segment.Start > End ? segment.Start : End;
|
||||
var additionalSpeechDuration = boundedEnd > uncoveredStart
|
||||
? boundedEnd - uncoveredStart
|
||||
: TimeSpan.Zero;
|
||||
extended = this with
|
||||
{
|
||||
End = boundedEnd > End ? boundedEnd : End,
|
||||
SpeechDuration = SpeechDuration + additionalSpeechDuration,
|
||||
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', Segments
|
||||
.Select(segment => segment.Text)
|
||||
.Where(text => !string.IsNullOrWhiteSpace(text))));
|
||||
}
|
||||
|
||||
private static TimeSpan Min(TimeSpan left, TimeSpan right)
|
||||
{
|
||||
return left < right ? left : right;
|
||||
}
|
||||
}
|
||||
@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerSampleSpanSelector
|
||||
{
|
||||
public static bool CanExtend(
|
||||
string currentSpeaker,
|
||||
TimeSpan currentEnd,
|
||||
TranscriptionSegment nextSegment,
|
||||
TimeSpan maximumSegmentGap)
|
||||
{
|
||||
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
|
||||
nextSegment.Start - currentEnd <= maximumSegmentGap;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan maximumSegmentGap,
|
||||
TimeSpan minimumDuration)
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
var best = new List<TranscriptionSegment>();
|
||||
var current = new List<TranscriptionSegment>();
|
||||
|
||||
SpeakerSampleSpan? best = null;
|
||||
SpeakerSampleSpan? current = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (current.Count == 0)
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
if (IsSpeaker(segment, speaker))
|
||||
{
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is not null &&
|
||||
(best is null || current.SpeechDuration > best.SpeechDuration))
|
||||
{
|
||||
current.Clear();
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
best = current;
|
||||
}
|
||||
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
return SpanDuration(best) >= minimumDuration
|
||||
? best
|
||||
return best is not null && best.SpeechDuration >= minimumDuration
|
||||
? best.Segments
|
||||
: [];
|
||||
}
|
||||
|
||||
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
int maxSpans)
|
||||
{
|
||||
if (maxSpans <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var completed = new List<SpeakerSampleSpan>();
|
||||
SpeakerSampleSpan? current = null;
|
||||
TimeSpan? lastCompletedEnd = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (lastCompletedEnd is { } end && segment.Start < end)
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is null || current.SpeechDuration < minimumDuration)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
completed.Add(current);
|
||||
lastCompletedEnd = current.End;
|
||||
current = null;
|
||||
}
|
||||
|
||||
return completed
|
||||
.OrderByDescending(span => span.SpeechDuration)
|
||||
.ThenBy(span => span.Start)
|
||||
.Take(maxSpans)
|
||||
.OrderBy(span => span.Start)
|
||||
.Select(span => span.Segments)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return segments.Count == 0
|
||||
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
|
||||
: segments[^1].End - segments[0].Start;
|
||||
}
|
||||
|
||||
private static SpeakerSampleSpan? ExtendOrStart(
|
||||
SpeakerSampleSpan? current,
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
return current is not null && current.TryExtend(segment, out var extended)
|
||||
? extended
|
||||
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
}
|
||||
|
||||
private static bool IsSpeaker(
|
||||
TranscriptionSegment segment,
|
||||
string speaker)
|
||||
{
|
||||
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> LongerSpan(
|
||||
List<TranscriptionSegment> current,
|
||||
List<TranscriptionSegment> best)
|
||||
{
|
||||
return SpanDuration(current) > SpanDuration(best)
|
||||
? current.ToList()
|
||||
: best;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record DecodedSpeakerVoiceVector(
|
||||
SpeakerVoiceVector Stored,
|
||||
float[] Vector);
|
||||
|
||||
internal static class ResemblyzerVectorContract
|
||||
{
|
||||
public const int Dimensions = 256;
|
||||
}
|
||||
|
||||
internal static class SpeakerVoiceVectors
|
||||
{
|
||||
public static float[] Decode(SpeakerVoiceVector stored)
|
||||
{
|
||||
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
|
||||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
|
||||
}
|
||||
|
||||
var vector = new float[stored.Dimensions];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
|
||||
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
|
||||
}
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
public static int AddDistinct(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<float[]> vectors,
|
||||
string modelId,
|
||||
int maximumCount,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var limit = Math.Max(1, maximumCount);
|
||||
var fingerprints = identity.VoiceVectors
|
||||
.Select(vector => vector.Fingerprint)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var added = 0;
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
if (identity.VoiceVectors.Count >= limit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var bytes = Encode(vector);
|
||||
var fingerprint = Fingerprint(modelId, bytes);
|
||||
if (!fingerprints.Add(fingerprint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = modelId,
|
||||
Dimensions = vector.Length,
|
||||
VectorBytes = bytes,
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
});
|
||||
added++;
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<float[]> DecodeCompatible(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
return DecodeCompatibleEntries(identity, modelId, logger)
|
||||
.Select(entry => entry.Vector)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
var vectors = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var stored in identity.VoiceVectors
|
||||
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
|
||||
.OrderBy(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Id))
|
||||
{
|
||||
try
|
||||
{
|
||||
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
|
||||
stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
public static double Cosine(float[] first, float[] second)
|
||||
{
|
||||
double dot = 0;
|
||||
for (var index = 0; index < first.Length; index++)
|
||||
{
|
||||
dot += first[index] * second[index];
|
||||
}
|
||||
|
||||
return dot;
|
||||
}
|
||||
|
||||
public static float[] Normalize(float[]? vector, string description)
|
||||
{
|
||||
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
double magnitudeSquared = 0;
|
||||
foreach (var value in vector)
|
||||
{
|
||||
if (!float.IsFinite(value))
|
||||
{
|
||||
throw new InvalidDataException($"{description} contains a non-finite value.");
|
||||
}
|
||||
|
||||
magnitudeSquared += value * value;
|
||||
}
|
||||
|
||||
var magnitude = Math.Sqrt(magnitudeSquared);
|
||||
if (magnitude <= double.Epsilon)
|
||||
{
|
||||
throw new InvalidDataException($"{description} has zero magnitude.");
|
||||
}
|
||||
|
||||
return vector.Select(value => (float)(value / magnitude)).ToArray();
|
||||
}
|
||||
|
||||
private static byte[] Encode(float[] vector)
|
||||
{
|
||||
if (vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException("Voice vector contains a non-finite value.");
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteSingleLittleEndian(
|
||||
bytes.AsSpan(index * sizeof(float), sizeof(float)),
|
||||
vector[index]);
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static string Fingerprint(string modelId, byte[] vectorBytes)
|
||||
{
|
||||
var modelBytes = Encoding.UTF8.GetBytes(modelId);
|
||||
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
|
||||
modelBytes.CopyTo(input, 0);
|
||||
input[modelBytes.Length] = 0;
|
||||
vectorBytes.CopyTo(input, modelBytes.Length + 1);
|
||||
return Convert.ToHexString(SHA256.HashData(input));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public interface IResemblyzerVoiceEncoder
|
||||
{
|
||||
Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task WarmUpAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private const string EnvironmentSchemaVersion = "venv-v2";
|
||||
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
|
||||
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
|
||||
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
|
||||
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
|
||||
private const string LibrosaRequirement = "librosa>=0.9.1";
|
||||
private const string ScipyRequirement = "scipy>=1.2.1";
|
||||
private readonly SemaphoreSlim commandLock = new(1, 1);
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
|
||||
private string? verifiedEnvironmentPythonPath;
|
||||
|
||||
public VenvResemblyzerVoiceEncoder(
|
||||
ICommandRunner commandRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<VenvResemblyzerVoiceEncoder> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (wavSamples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
if (wavSamples.Any(sample => sample.Length == 0))
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
|
||||
}
|
||||
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(inputFolder);
|
||||
try
|
||||
{
|
||||
for (var index = 0; index < wavSamples.Count; index++)
|
||||
{
|
||||
await File.WriteAllBytesAsync(
|
||||
Path.Combine(inputFolder, $"{index:D4}.wav"),
|
||||
wavSamples[index],
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(inputFolder, recursive: true);
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
catch (UnauthorizedAccessException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
|
||||
string inputFolder,
|
||||
int expectedCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
var pythonPath = await EnsureEnvironmentAsync(token);
|
||||
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
|
||||
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
|
||||
var result = await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[scriptPath, Path.GetFullPath(inputFolder)],
|
||||
"encoding",
|
||||
token);
|
||||
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
await EnsureEnvironmentAsync(token);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateDependencySettings();
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
|
||||
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
|
||||
var readyPath = Path.Combine(environmentFolder, ".ready");
|
||||
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
|
||||
&& File.Exists(pythonPath)
|
||||
&& File.Exists(readyPath))
|
||||
{
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
if (File.Exists(pythonPath) && File.Exists(readyPath))
|
||||
{
|
||||
var verification = await commandRunner.RunAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
cancellationToken);
|
||||
if (verification.ExitCode == 0)
|
||||
{
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
|
||||
verification.ExitCode,
|
||||
verification.StandardError);
|
||||
File.Delete(readyPath);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(environmentFolder);
|
||||
await RunRequiredAsync(
|
||||
options.PythonCommand,
|
||||
["-m", "venv", "--clear", environmentFolder],
|
||||
"virtual environment creation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
["-m", "pip", "install", "--upgrade", "pip"],
|
||||
"pip upgrade",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
"--index-url", options.TorchIndexUrl,
|
||||
TorchRequirement
|
||||
],
|
||||
"CPU PyTorch installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
.. RuntimeDependencyRequirements
|
||||
],
|
||||
"Resemblyzer dependency installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install", "--no-deps",
|
||||
ResemblyzerRequirement
|
||||
],
|
||||
"Resemblyzer installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
"environment verification",
|
||||
cancellationToken);
|
||||
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunRequiredAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
|
||||
ThrowIfFailed(result, operation);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string BuildEncodingScript()
|
||||
{
|
||||
return
|
||||
"import json\n"
|
||||
+ "import sys\n"
|
||||
+ "import numpy as np\n"
|
||||
+ "from pathlib import Path\n"
|
||||
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
|
||||
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
|
||||
+ "vectors = []\n"
|
||||
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
|
||||
+ " wav = preprocess_wav(path)\n"
|
||||
+ " vector = encoder.embed_utterance(wav)\n"
|
||||
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
|
||||
+ $"print('{JsonStart}')\n"
|
||||
+ "print(json.dumps(vectors, allow_nan=False))\n"
|
||||
+ $"print('{JsonEnd}')\n";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
|
||||
{
|
||||
var json = ExtractJson(output);
|
||||
float[][]? vectors;
|
||||
try
|
||||
{
|
||||
vectors = JsonSerializer.Deserialize<float[][]>(json);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
|
||||
}
|
||||
|
||||
if (vectors is null || vectors.Length != expectedCount)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
|
||||
}
|
||||
|
||||
return vectors
|
||||
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string ExtractJson(string output)
|
||||
{
|
||||
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||
if (start < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
|
||||
}
|
||||
|
||||
start += JsonStart.Length;
|
||||
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||
if (end < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
|
||||
}
|
||||
|
||||
return output[start..end].Trim();
|
||||
}
|
||||
|
||||
private string BuildEnvironmentKey()
|
||||
{
|
||||
var settings = string.Join('\n', EnvironmentManifest);
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
|
||||
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
|
||||
}
|
||||
|
||||
private string TorchRequirement => $"torch=={options.TorchVersion}";
|
||||
|
||||
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
|
||||
|
||||
private static string[] WarmupArguments =>
|
||||
[
|
||||
"-c",
|
||||
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
|
||||
];
|
||||
|
||||
private string[] RuntimeDependencyRequirements =>
|
||||
[
|
||||
NumpyBeforePython313Requirement,
|
||||
NumpyFromPython313Requirement,
|
||||
LibrosaRequirement,
|
||||
ScipyRequirement,
|
||||
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
|
||||
];
|
||||
|
||||
private IEnumerable<string> EnvironmentManifest =>
|
||||
[
|
||||
EnvironmentSchemaVersion,
|
||||
options.PythonCommand,
|
||||
options.TorchIndexUrl,
|
||||
TorchRequirement,
|
||||
.. RuntimeDependencyRequirements,
|
||||
ResemblyzerRequirement
|
||||
];
|
||||
|
||||
private static string GetEnvironmentPythonPath(string environmentFolder)
|
||||
{
|
||||
return OperatingSystem.IsWindows()
|
||||
? Path.Combine(environmentFolder, "Scripts", "python.exe")
|
||||
: Path.Combine(environmentFolder, "bin", "python");
|
||||
}
|
||||
|
||||
private void ValidateDependencySettings()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.PythonCommand))
|
||||
{
|
||||
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
|
||||
}
|
||||
|
||||
ValidateVersion(options.PackageVersion, "Resemblyzer package");
|
||||
ValidateVersion(options.TorchVersion, "PyTorch");
|
||||
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
|
||||
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|
||||
|| indexUri.Scheme != Uri.UriSchemeHttps)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateVersion(string version, string dependency)
|
||||
{
|
||||
if (!PackageVersionPattern().IsMatch(version))
|
||||
{
|
||||
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<T> RunWithTimeoutAsync<T>(
|
||||
Func<CancellationToken, Task<T>> operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(options.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
try
|
||||
{
|
||||
return await operation(linkedSource?.Token ?? cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (
|
||||
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ThrowIfFailed(CommandResult result, string operation)
|
||||
{
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
}
|
||||
|
||||
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex PackageVersionPattern();
|
||||
}
|
||||
@@ -18,6 +18,12 @@ public sealed class BoundMeetingProjectResolver
|
||||
|
||||
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
|
||||
|
||||
public List<BoundMeetingProject> GetAllProjects() => !Directory.Exists(ProjectsRoot) ? [] :
|
||||
Directory.EnumerateDirectories(ProjectsRoot)
|
||||
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
|
||||
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationToken cancellationToken = default)
|
||||
@@ -33,10 +39,8 @@ public sealed class BoundMeetingProjectResolver
|
||||
return [];
|
||||
}
|
||||
|
||||
return Directory.EnumerateDirectories(ProjectsRoot)
|
||||
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
|
||||
return GetAllProjects()
|
||||
.Where(project => projectNames.Contains(project.Name))
|
||||
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace MeetingAssistant.Summary;
|
||||
|
||||
internal static class FunctionInvocationGuard
|
||||
{
|
||||
public static ValueTask<object?> InvokeAsync(
|
||||
FunctionInvocationContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (context.CallContent.Exception is not null)
|
||||
{
|
||||
return ValueTask.FromResult<object?>(new JsonObject
|
||||
{
|
||||
["error"] = new JsonObject
|
||||
{
|
||||
["code"] = "invalid_tool_arguments",
|
||||
["message"] = "Tool arguments must be a valid JSON object."
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return context.Function.InvokeAsync(context.Arguments, cancellationToken);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user