Public Access
Compare commits
90
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9a0bf1f83 | ||
|
|
7b2bcd3631 | ||
|
|
e011e36ef7 | ||
|
|
e17fb8b225 | ||
|
|
c01213278e | ||
|
|
7814f1c7c4 | ||
|
|
88c3b21b2a | ||
|
|
b249e5d977 | ||
|
|
3e4bf88931 | ||
|
|
5e7c9f64d0 | ||
|
|
fbd5a0852f | ||
|
|
568bedb77a | ||
|
|
f21c56b1c7 | ||
|
|
c56de2b7ab | ||
|
|
40d10d634a | ||
|
|
264a3a823f | ||
|
|
072ebf38a3 | ||
|
|
4d66de943e | ||
|
|
150f11b6a7 | ||
|
|
d7db64bec7 | ||
|
|
cdeef1967e | ||
|
|
aeb0cfe42f | ||
|
|
ef0c0faba8 | ||
|
|
93eb3d4e98 | ||
|
|
d93d758a8f | ||
|
|
48ad4fb930 | ||
|
|
cb5a300b44 | ||
|
|
b6acba8402 | ||
|
|
a82e2fff03 | ||
|
|
336d940acd | ||
|
|
cbf854fb2a | ||
|
|
bacef060fe | ||
|
|
a928382e51 | ||
|
|
e23d04a38c | ||
|
|
c6a50d307e | ||
|
|
436fa6c149 | ||
|
|
5dce5fd350 | ||
|
|
c37654a7d9 | ||
|
|
cc9044e091 | ||
|
|
f86af983e8 | ||
|
|
43fc8aaec0 | ||
|
|
8b543a2161 | ||
|
|
58896073c0 | ||
|
|
945543c078 | ||
|
|
3e1f447b20 | ||
|
|
497390c2ec | ||
|
|
b1365b202d | ||
|
|
70ffaa7357 | ||
|
|
714da456df | ||
|
|
fa67f30c49 | ||
|
|
dc37719d46 | ||
|
|
ee453e170e | ||
|
|
d51463321b | ||
|
|
41ae47fbfe | ||
|
|
f7ad2ce41a | ||
|
|
bc76151926 | ||
|
|
af9f5b34dc | ||
|
|
f3489f8e95 | ||
|
|
39591eef12 | ||
|
|
b3aeef24eb | ||
|
|
c91d956beb | ||
|
|
4c858170d9 | ||
|
|
0b6643d87f | ||
|
|
44ce419c49 | ||
|
|
a087ff0cbf | ||
|
|
8dc1b70f8a | ||
|
|
aaee123b9a | ||
|
|
1a341f1eaa | ||
|
|
e886fac311 | ||
|
|
e35ccf661f | ||
|
|
aa42e8edda | ||
|
|
2f12a96688 | ||
|
|
20eed38477 | ||
|
|
ada4a40f62 | ||
|
|
5d0ae84426 | ||
|
|
b9547ae4c4 | ||
|
|
75250f6041 | ||
|
|
cd2446f620 | ||
|
|
e192ae7cd8 | ||
|
|
3cadf08fa2 | ||
|
|
407db80413 | ||
|
|
f0aac40dfb | ||
|
|
0e0feedad2 | ||
|
|
89d81fa4c6 | ||
|
|
dc49bc3330 | ||
|
|
aff3528406 | ||
|
|
d92da18d08 | ||
|
|
ec59464340 | ||
|
|
5667102272 | ||
|
|
2f62ed467a |
@@ -18,7 +18,7 @@ jobs:
|
|||||||
uses: actions/checkout@v7
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Setup .NET
|
- name: Setup .NET
|
||||||
uses: actions/setup-dotnet@v5
|
uses: actions/setup-dotnet@v6
|
||||||
with:
|
with:
|
||||||
dotnet-version: "10.0.x"
|
dotnet-version: "10.0.x"
|
||||||
|
|
||||||
|
|||||||
@@ -102,6 +102,23 @@ public sealed class AudioMixingTests
|
|||||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm));
|
Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CompositeAudioSourceKeepsSystemAudioWhileMicrophoneIsRecovering()
|
||||||
|
{
|
||||||
|
var microphone = new WaitingAudioSource();
|
||||||
|
var system = new FixedAudioSource(Pcm16(10_000));
|
||||||
|
var source = CreateSource(microphone, system);
|
||||||
|
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
await using var chunks = source
|
||||||
|
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||||
|
.GetAsyncEnumerator(cancellation.Token);
|
||||||
|
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal(10_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
|
||||||
|
await cancellation.CancelAsync();
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal()
|
public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal()
|
||||||
{
|
{
|
||||||
@@ -221,6 +238,16 @@ public sealed class AudioMixingTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class WaitingAudioSource : IMeetingAudioSource
|
||||||
|
{
|
||||||
|
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||||
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class OptionsCapturingAudioSource : IMeetingAudioSource
|
private sealed class OptionsCapturingAudioSource : IMeetingAudioSource
|
||||||
{
|
{
|
||||||
private readonly AudioChunk chunk;
|
private readonly AudioChunk chunk;
|
||||||
|
|||||||
@@ -123,6 +123,49 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
|||||||
Assert.Equal(["stop", "start"], harness.Recorder.Commands);
|
Assert.Equal(["stop", "start"], harness.Recorder.Commands);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ActiveRecordingPromptAttachesTheSelectedAppointmentMetadataWithoutRestarting()
|
||||||
|
{
|
||||||
|
var harness = CreateHarness(isRecording: true, autoAcceptPrompts: false);
|
||||||
|
var firstMetadata = new MeetingMetadata(
|
||||||
|
"First planning",
|
||||||
|
["Ada"],
|
||||||
|
"First agenda",
|
||||||
|
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||||
|
var secondMetadata = new MeetingMetadata(
|
||||||
|
"Second planning",
|
||||||
|
["Grace"],
|
||||||
|
"Second agenda",
|
||||||
|
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
|
||||||
|
var firstMeeting = CreateMeeting(
|
||||||
|
harness.Clock,
|
||||||
|
id: "teams-first",
|
||||||
|
subject: "First planning",
|
||||||
|
metadata: firstMetadata);
|
||||||
|
var secondMeeting = CreateMeeting(
|
||||||
|
harness.Clock,
|
||||||
|
id: "teams-second",
|
||||||
|
subject: "Second planning",
|
||||||
|
metadata: secondMetadata);
|
||||||
|
harness.Provider.Meetings = [firstMeeting, secondMeeting];
|
||||||
|
|
||||||
|
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
|
||||||
|
harness.Clock.Now = firstMeeting.Start;
|
||||||
|
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
|
||||||
|
await harness.PromptService.RespondAsync(
|
||||||
|
secondMeeting,
|
||||||
|
MeetingStartPromptResponse.AttachMetadataToCurrentMeeting);
|
||||||
|
|
||||||
|
Assert.All(
|
||||||
|
harness.PromptService.PromptRequests,
|
||||||
|
request => Assert.True(request.CanAttachToCurrentMeeting));
|
||||||
|
Assert.Equal(["attach-metadata"], harness.Recorder.Commands);
|
||||||
|
Assert.Same(secondMetadata, harness.Recorder.AttachedMetadata.Single());
|
||||||
|
Assert.Equal(0, harness.Recorder.StopCount);
|
||||||
|
Assert.Equal(0, harness.Recorder.StartCount);
|
||||||
|
Assert.True(harness.Recorder.CurrentStatus.IsRecording);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task CanceledCachedMeetingDoesNotPromptRecording()
|
public async Task CanceledCachedMeetingDoesNotPromptRecording()
|
||||||
{
|
{
|
||||||
@@ -302,14 +345,17 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
|||||||
this.autoAccept = autoAccept;
|
this.autoAccept = autoAccept;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<CalendarMeeting> PromptedMeetings { get; } = [];
|
public List<MeetingStartPromptRequest> PromptRequests { get; } = [];
|
||||||
|
|
||||||
|
public IReadOnlyList<CalendarMeeting> PromptedMeetings =>
|
||||||
|
PromptRequests.Select(request => request.Meeting).ToList();
|
||||||
|
|
||||||
public async Task ShowPromptAsync(
|
public async Task ShowPromptAsync(
|
||||||
MeetingStartPromptRequest request,
|
MeetingStartPromptRequest request,
|
||||||
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
|
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
PromptedMeetings.Add(request.Meeting);
|
PromptRequests.Add(request);
|
||||||
pendingPrompts.Add(new PendingPrompt(request.Meeting, handleResponseAsync));
|
pendingPrompts.Add(new PendingPrompt(request.Meeting, handleResponseAsync));
|
||||||
if (autoAccept)
|
if (autoAccept)
|
||||||
{
|
{
|
||||||
@@ -349,6 +395,8 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
|||||||
|
|
||||||
public List<MeetingMetadata?> StartMetadata { get; } = [];
|
public List<MeetingMetadata?> StartMetadata { get; } = [];
|
||||||
|
|
||||||
|
public List<MeetingMetadata> AttachedMetadata { get; } = [];
|
||||||
|
|
||||||
public Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
|
public Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return StartRecordingAsync(null);
|
return StartRecordingAsync(null);
|
||||||
@@ -380,6 +428,15 @@ public sealed class CalendarRecordingPromptSchedulerTests
|
|||||||
return Task.FromResult(CurrentStatus);
|
return Task.FromResult(CurrentStatus);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||||
|
MeetingMetadata metadata,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Commands.Add("attach-metadata");
|
||||||
|
AttachedMetadata.Add(metadata);
|
||||||
|
return Task.FromResult(CurrentStatus);
|
||||||
|
}
|
||||||
|
|
||||||
private static RecordingStatus Status(bool isRecording)
|
private static RecordingStatus Status(bool isRecording)
|
||||||
{
|
{
|
||||||
return new RecordingStatus(
|
return new RecordingStatus(
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
using System.Net;
|
using System.Net;
|
||||||
using System.Net.Http.Json;
|
using System.Net.Http.Json;
|
||||||
using MeetingAssistant.Recording;
|
using MeetingAssistant.Recording;
|
||||||
|
using MeetingAssistant.Workflow;
|
||||||
using Microsoft.AspNetCore.Hosting;
|
using Microsoft.AspNetCore.Hosting;
|
||||||
using Microsoft.AspNetCore.Mvc.Testing;
|
using Microsoft.AspNetCore.Mvc.Testing;
|
||||||
using Microsoft.AspNetCore.TestHost;
|
using Microsoft.AspNetCore.TestHost;
|
||||||
@@ -41,6 +42,17 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
|
|||||||
Assert.Equal("ok", body?.Status);
|
Assert.Equal("ok", body?.Status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
|
||||||
|
{
|
||||||
|
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
|
||||||
|
|
||||||
|
var result = await playback.QueueAsync(1, [1, 2]);
|
||||||
|
|
||||||
|
Assert.Contains("Windows build", result);
|
||||||
|
Assert.DoesNotContain("Queued", result);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
|
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -58,6 +58,64 @@ public sealed class LaunchProfileOptionsProviderTests
|
|||||||
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
|
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
|
||||||
|
{
|
||||||
|
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
|
||||||
|
|
||||||
|
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
|
||||||
|
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
|
||||||
|
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
|
||||||
|
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
|
||||||
|
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
|
||||||
|
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
|
||||||
|
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
|
||||||
|
{
|
||||||
|
var provider = CreateProvider(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
|
||||||
|
});
|
||||||
|
|
||||||
|
var exception = Assert.Throws<InvalidOperationException>(
|
||||||
|
() => provider.GetRequiredProfile(null));
|
||||||
|
|
||||||
|
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
|
||||||
|
{
|
||||||
|
var provider = CreateProvider(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
|
||||||
|
});
|
||||||
|
|
||||||
|
var exception = Assert.Throws<InvalidOperationException>(
|
||||||
|
() => provider.GetRequiredProfile(null));
|
||||||
|
|
||||||
|
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ContradictorySpeakerSampleDurationsAreRejected()
|
||||||
|
{
|
||||||
|
var provider = CreateProvider(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
|
||||||
|
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
|
||||||
|
});
|
||||||
|
|
||||||
|
var exception = Assert.Throws<InvalidOperationException>(
|
||||||
|
() => provider.GetRequiredProfile(null));
|
||||||
|
|
||||||
|
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
|
||||||
|
Assert.Contains("MaximumSampleDuration", exception.Message);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void DuplicateProfileHotkeysAreRejected()
|
public void DuplicateProfileHotkeysAreRejected()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,82 +1,223 @@
|
|||||||
using MeetingAssistant.Summary;
|
using MeetingAssistant.Summary;
|
||||||
using Microsoft.Extensions.AI;
|
using Microsoft.Extensions.AI;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
namespace MeetingAssistant.Tests;
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
public sealed class LiteLlmResponsesChatClientTests
|
public sealed class LiteLlmResponsesChatClientTests
|
||||||
{
|
{
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ParserIgnoresReasoningItemsWithNullStatusAndReadsText()
|
public async Task ClientAssemblesStreamedTextResponseWithMetadataAndUsage()
|
||||||
{
|
{
|
||||||
const string json = """
|
var handler = new SequencedHttpMessageHandler(
|
||||||
|
new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
{
|
{
|
||||||
"id": "resp_test",
|
Content = new StringContent(
|
||||||
"created_at": 1779147100,
|
"""
|
||||||
"model": "gpt-5.5-2026-04-23",
|
data: {"type":"response.created","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"in_progress","store":false},"sequence_number":0}
|
||||||
"output": [
|
|
||||||
|
data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_stream","type":"message","status":"in_progress","content":[],"role":"assistant"},"sequence_number":1}
|
||||||
|
|
||||||
|
data: {"type":"response.content_part.added","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":""},"sequence_number":2}
|
||||||
|
|
||||||
|
data: {"type":"response.output_text.delta","item_id":"msg_stream","output_index":0,"content_index":0,"delta":"Streamed OK","sequence_number":3}
|
||||||
|
|
||||||
|
data: {"type":"response.output_text.done","item_id":"msg_stream","output_index":0,"content_index":0,"text":"Streamed OK","sequence_number":4}
|
||||||
|
|
||||||
|
data: {"type":"response.content_part.done","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":"Streamed OK"},"sequence_number":5}
|
||||||
|
|
||||||
|
data: {"type":"response.output_item.done","output_index":0,"item":{"id":"msg_stream","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Streamed OK"}],"role":"assistant"},"sequence_number":6}
|
||||||
|
|
||||||
|
data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}},"sequence_number":7}
|
||||||
|
|
||||||
|
data: [DONE]
|
||||||
|
|
||||||
|
""",
|
||||||
|
Encoding.UTF8,
|
||||||
|
"text/event-stream")
|
||||||
|
});
|
||||||
|
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||||
|
|
||||||
|
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
|
||||||
|
|
||||||
|
Assert.Equal("Streamed OK", response.Text);
|
||||||
|
Assert.Equal("resp_stream", response.ResponseId);
|
||||||
|
Assert.Equal("gpt-5.5", response.ModelId);
|
||||||
|
Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1779147100), response.CreatedAt);
|
||||||
|
Assert.NotNull(response.Usage);
|
||||||
|
Assert.Equal(12, response.Usage.InputTokenCount);
|
||||||
|
Assert.Equal(3, response.Usage.OutputTokenCount);
|
||||||
|
Assert.Equal(15, response.Usage.TotalTokenCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ClientAssemblesStreamedFunctionCall()
|
||||||
|
{
|
||||||
|
var handler = new SequencedHttpMessageHandler(
|
||||||
|
new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
Content = new StringContent(
|
||||||
|
"""
|
||||||
|
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_stream","name":"write_summary","arguments":"{\"markdown\":\"# Summary\\nDone\"}","status":"completed"}}
|
||||||
|
|
||||||
|
data: {"type":"response.completed","response":{"id":"resp_stream","model":"gpt-5.5","output":[]}}
|
||||||
|
|
||||||
|
data: [DONE]
|
||||||
|
|
||||||
|
""",
|
||||||
|
Encoding.UTF8,
|
||||||
|
"text/event-stream")
|
||||||
|
});
|
||||||
|
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||||
|
|
||||||
|
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
|
||||||
|
var call = Assert.IsType<FunctionCallContent>(Assert.Single(response.Messages[0].Contents));
|
||||||
|
|
||||||
|
Assert.Equal("call_stream", call.CallId);
|
||||||
|
Assert.Equal("write_summary", call.Name);
|
||||||
|
Assert.NotNull(call.Arguments);
|
||||||
|
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]?.ToString());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ClientUsesNonStreamingResponsesWhenConfigured()
|
||||||
|
{
|
||||||
|
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
Content = new StringContent(
|
||||||
|
"""
|
||||||
{
|
{
|
||||||
"type": "reasoning",
|
"id": "resp_nonstream",
|
||||||
"summary": [],
|
"created_at": 1779147100,
|
||||||
"status": null
|
"model": "gpt-5.5",
|
||||||
},
|
"object": "response",
|
||||||
{
|
"output": [
|
||||||
"type": "message",
|
|
||||||
"role": "assistant",
|
|
||||||
"status": "completed",
|
|
||||||
"content": [
|
|
||||||
{
|
{
|
||||||
"type": "output_text",
|
"id": "msg_nonstream",
|
||||||
"text": "OK"
|
"type": "message",
|
||||||
|
"status": "completed",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "output_text",
|
||||||
|
"annotations": [],
|
||||||
|
"text": "Non-streamed OK"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"role": "assistant"
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
|
"parallel_tool_calls": true,
|
||||||
|
"status": "completed",
|
||||||
|
"store": false,
|
||||||
|
"usage": {
|
||||||
|
"input_tokens": 12,
|
||||||
|
"output_tokens": 4,
|
||||||
|
"total_tokens": 16
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
""",
|
||||||
}
|
Encoding.UTF8,
|
||||||
""";
|
"application/json")
|
||||||
|
});
|
||||||
|
using var client = CreateClient(handler, reconnectionAttempts: 0, useStreaming: false);
|
||||||
|
|
||||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
|
||||||
|
|
||||||
Assert.Equal("OK", response.Text);
|
Assert.Equal("Non-streamed OK", response.Text);
|
||||||
Assert.Equal("resp_test", response.ResponseId);
|
Assert.Contains("\"stream\":false", Assert.Single(handler.RequestBodies));
|
||||||
Assert.Equal("gpt-5.5-2026-04-23", response.ModelId);
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ClientSendsChatMessagesAsResponsesMessageItems()
|
||||||
|
{
|
||||||
|
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
|
||||||
|
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||||
|
|
||||||
|
await client.GetResponseAsync(
|
||||||
|
[
|
||||||
|
new ChatMessage(ChatRole.User, "write summary"),
|
||||||
|
new ChatMessage(ChatRole.Assistant, "I will inspect the meeting.")
|
||||||
|
]);
|
||||||
|
|
||||||
|
using var request = JsonDocument.Parse(Assert.Single(handler.RequestBodies));
|
||||||
|
var inputItems = request.RootElement.GetProperty("input").EnumerateArray().ToArray();
|
||||||
|
Assert.Equal(2, inputItems.Length);
|
||||||
|
Assert.All(inputItems, item => Assert.Equal("message", item.GetProperty("type").GetString()));
|
||||||
|
Assert.Equal("user", inputItems[0].GetProperty("role").GetString());
|
||||||
|
Assert.Equal(
|
||||||
|
"input_text",
|
||||||
|
Assert.Single(inputItems[0].GetProperty("content").EnumerateArray())
|
||||||
|
.GetProperty("type")
|
||||||
|
.GetString());
|
||||||
|
Assert.Equal("assistant", inputItems[1].GetProperty("role").GetString());
|
||||||
|
Assert.Equal(
|
||||||
|
"output_text",
|
||||||
|
Assert.Single(inputItems[1].GetProperty("content").EnumerateArray())
|
||||||
|
.GetProperty("type")
|
||||||
|
.GetString());
|
||||||
|
Assert.DoesNotContain("\"type\":\"unknown\"", handler.RequestBodies[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AgentLoopReturnsMalformedFunctionArgumentsWithoutInvokingTool()
|
||||||
|
{
|
||||||
|
var responses = new Queue<HttpResponseMessage>(
|
||||||
|
[
|
||||||
|
new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
Content = new StringContent(
|
||||||
|
"""
|
||||||
|
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_invalid","name":"write_summary","arguments":"{not-json","status":"completed"}}
|
||||||
|
|
||||||
|
data: {"type":"response.completed","response":{"id":"resp_invalid","model":"gpt-5.5","output":[]}}
|
||||||
|
|
||||||
|
data: [DONE]
|
||||||
|
|
||||||
|
""",
|
||||||
|
Encoding.UTF8,
|
||||||
|
"text/event-stream")
|
||||||
|
},
|
||||||
|
CreateStreamedTextResponse("Recovered.")
|
||||||
|
]);
|
||||||
|
var handler = new RecordingHttpMessageHandler(_ => responses.Dequeue());
|
||||||
|
var toolInvoked = false;
|
||||||
|
var tool = AIFunctionFactory.Create(
|
||||||
|
(string markdown) =>
|
||||||
|
{
|
||||||
|
toolInvoked = true;
|
||||||
|
return markdown;
|
||||||
|
},
|
||||||
|
"write_summary",
|
||||||
|
"Writes a summary.");
|
||||||
|
using var innerClient = CreateClient(handler, reconnectionAttempts: 0);
|
||||||
|
using var functionClient = innerClient
|
||||||
|
.AsBuilder()
|
||||||
|
.UseFunctionInvocation(
|
||||||
|
loggerFactory: null,
|
||||||
|
client => client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync)
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
var response = await functionClient.GetResponseAsync(
|
||||||
|
[new ChatMessage(ChatRole.User, "write summary")],
|
||||||
|
new ChatOptions { Tools = [tool] });
|
||||||
|
|
||||||
|
Assert.False(toolInvoked);
|
||||||
|
Assert.Equal("Recovered.", response.Text);
|
||||||
|
Assert.Equal(2, handler.RequestBodies.Count);
|
||||||
|
Assert.Contains("invalid_tool_arguments", handler.RequestBodies[1]);
|
||||||
|
Assert.Contains("call_invalid", handler.RequestBodies[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ClientReportsVisibleReasoningSummariesSeparatelyFromResponseText()
|
public async Task ClientReportsVisibleReasoningSummariesSeparatelyFromResponseText()
|
||||||
{
|
{
|
||||||
var handler = new SequencedHttpMessageHandler(
|
var handler = new SequencedHttpMessageHandler(
|
||||||
new HttpResponseMessage(HttpStatusCode.OK)
|
CreateStreamedTextResponse(
|
||||||
{
|
"Done.",
|
||||||
Content = new StringContent("""
|
"Checked the configured rules file.",
|
||||||
{
|
"Prepared a targeted update."));
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "reasoning",
|
|
||||||
"summary": [
|
|
||||||
{
|
|
||||||
"type": "summary_text",
|
|
||||||
"text": "Checked the configured rules file."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "summary_text",
|
|
||||||
"text": "Prepared a targeted update."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
});
|
|
||||||
var reasoningSummaries = new List<string>();
|
var reasoningSummaries = new List<string>();
|
||||||
using var client = CreateClient(
|
using var client = CreateClient(
|
||||||
handler,
|
handler,
|
||||||
@@ -98,33 +239,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
public async Task ClientDoesNotFailResponseWhenReasoningSummaryCallbackFails()
|
public async Task ClientDoesNotFailResponseWhenReasoningSummaryCallbackFails()
|
||||||
{
|
{
|
||||||
var handler = new SequencedHttpMessageHandler(
|
var handler = new SequencedHttpMessageHandler(
|
||||||
new HttpResponseMessage(HttpStatusCode.OK)
|
CreateStreamedTextResponse("Done.", "Checked the configured rules file."));
|
||||||
{
|
|
||||||
Content = new StringContent("""
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "reasoning",
|
|
||||||
"summary": [
|
|
||||||
{
|
|
||||||
"type": "summary_text",
|
|
||||||
"text": "Checked the configured rules file."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
});
|
|
||||||
using var client = CreateClient(
|
using var client = CreateClient(
|
||||||
handler,
|
handler,
|
||||||
reconnectionAttempts: 0,
|
reconnectionAttempts: 0,
|
||||||
@@ -135,64 +250,6 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
Assert.Equal("Done.", response.Text);
|
Assert.Equal("Done.", response.Text);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void ParserReadsFunctionCalls()
|
|
||||||
{
|
|
||||||
const string json = """
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "function_call",
|
|
||||||
"call_id": "call_1",
|
|
||||||
"name": "write_summary",
|
|
||||||
"arguments": "{\"markdown\":\"# Summary\\nDone\"}",
|
|
||||||
"status": "completed"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""";
|
|
||||||
|
|
||||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
|
||||||
var call = Assert.IsType<FunctionCallContent>(Assert.Single(response.Messages[0].Contents));
|
|
||||||
|
|
||||||
Assert.Equal("call_1", call.CallId);
|
|
||||||
Assert.Equal("write_summary", call.Name);
|
|
||||||
Assert.NotNull(call.Arguments);
|
|
||||||
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void ParserReadsUsage()
|
|
||||||
{
|
|
||||||
const string json = """
|
|
||||||
{
|
|
||||||
"usage": {
|
|
||||||
"input_tokens": 123,
|
|
||||||
"output_tokens": 45,
|
|
||||||
"total_tokens": 168
|
|
||||||
},
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "OK"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""";
|
|
||||||
|
|
||||||
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
|
|
||||||
|
|
||||||
Assert.NotNull(response.Usage);
|
|
||||||
Assert.Equal(123, response.Usage.InputTokenCount);
|
|
||||||
Assert.Equal(45, response.Usage.OutputTokenCount);
|
|
||||||
Assert.Equal(168, response.Usage.TotalTokenCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ClientRetriesTransientServerFailure()
|
public async Task ClientRetriesTransientServerFailure()
|
||||||
{
|
{
|
||||||
@@ -201,24 +258,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
{
|
{
|
||||||
Content = new StringContent("Internal Server Error")
|
Content = new StringContent("Internal Server Error")
|
||||||
},
|
},
|
||||||
new HttpResponseMessage(HttpStatusCode.OK)
|
CreateStreamedTextResponse("Done."));
|
||||||
{
|
|
||||||
Content = new StringContent("""
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
});
|
|
||||||
var retryCount = 0;
|
var retryCount = 0;
|
||||||
using var client = CreateClient(handler, reconnectionAttempts: 1, retrying: () => retryCount++);
|
using var client = CreateClient(handler, reconnectionAttempts: 1, retrying: () => retryCount++);
|
||||||
|
|
||||||
@@ -249,24 +289,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
[Fact]
|
[Fact]
|
||||||
public async Task ClientSendsUserInitiatorOnceThenAgentInitiator()
|
public async Task ClientSendsUserInitiatorOnceThenAgentInitiator()
|
||||||
{
|
{
|
||||||
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
|
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
|
||||||
{
|
|
||||||
Content = new StringContent("""
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
});
|
|
||||||
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
using var client = CreateClient(handler, reconnectionAttempts: 0);
|
||||||
|
|
||||||
await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
|
await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
|
||||||
@@ -298,24 +321,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
return CreateStreamedTextResponse("Done.");
|
||||||
{
|
|
||||||
Content = new StringContent("""
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
using var client = CreateClient(
|
using var client = CreateClient(
|
||||||
handler,
|
handler,
|
||||||
@@ -350,24 +356,7 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
return CreateStreamedTextResponse("Done.");
|
||||||
{
|
|
||||||
Content = new StringContent("""
|
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"type": "message",
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "output_text",
|
|
||||||
"text": "Done."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
using var client = CreateClient(
|
using var client = CreateClient(
|
||||||
handler,
|
handler,
|
||||||
@@ -400,7 +389,8 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
int reconnectionAttempts,
|
int reconnectionAttempts,
|
||||||
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
||||||
Action? retrying = null,
|
Action? retrying = null,
|
||||||
Action<string>? reasoningSummaryChanged = null)
|
Action<string>? reasoningSummaryChanged = null,
|
||||||
|
bool useStreaming = true)
|
||||||
{
|
{
|
||||||
return new LiteLlmResponsesChatClient(
|
return new LiteLlmResponsesChatClient(
|
||||||
new HttpClient(handler)
|
new HttpClient(handler)
|
||||||
@@ -415,7 +405,49 @@ public sealed class LiteLlmResponsesChatClientTests
|
|||||||
TimeSpan.Zero,
|
TimeSpan.Zero,
|
||||||
compactionOptions,
|
compactionOptions,
|
||||||
retrying: retrying,
|
retrying: retrying,
|
||||||
reasoningSummaryChanged: reasoningSummaryChanged);
|
reasoningSummaryChanged: reasoningSummaryChanged,
|
||||||
|
useStreaming: useStreaming);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static HttpResponseMessage CreateStreamedTextResponse(
|
||||||
|
string text,
|
||||||
|
params string[] reasoningSummaries)
|
||||||
|
{
|
||||||
|
var events = new List<string>();
|
||||||
|
for (var index = 0; index < reasoningSummaries.Length; index++)
|
||||||
|
{
|
||||||
|
events.Add("data: " + JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
type = "response.reasoning_summary_text.delta",
|
||||||
|
item_id = "reasoning_stream",
|
||||||
|
output_index = 0,
|
||||||
|
summary_index = index,
|
||||||
|
delta = reasoningSummaries[index]
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
var outputIndex = reasoningSummaries.Length > 0 ? 1 : 0;
|
||||||
|
events.Add("data: " + JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
type = "response.output_text.delta",
|
||||||
|
item_id = "msg_stream",
|
||||||
|
output_index = outputIndex,
|
||||||
|
content_index = 0,
|
||||||
|
delta = text
|
||||||
|
}));
|
||||||
|
events.Add(
|
||||||
|
"""data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}}}""");
|
||||||
|
events.Add("data: [DONE]");
|
||||||
|
|
||||||
|
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
Content = new StringContent(
|
||||||
|
string.Join($"{Environment.NewLine}{Environment.NewLine}", events)
|
||||||
|
+ Environment.NewLine
|
||||||
|
+ Environment.NewLine,
|
||||||
|
Encoding.UTF8,
|
||||||
|
"text/event-stream")
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class SequencedHttpMessageHandler : HttpMessageHandler
|
private sealed class SequencedHttpMessageHandler : HttpMessageHandler
|
||||||
|
|||||||
@@ -13,21 +13,50 @@ namespace MeetingAssistant.Tests;
|
|||||||
|
|
||||||
public sealed class LiteLlmScreenshotOcrClientTests
|
public sealed class LiteLlmScreenshotOcrClientTests
|
||||||
{
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task ExtractUsesAgentStreamingTransportByDefault()
|
||||||
|
{
|
||||||
|
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||||
|
var handler = new RecordingHandler(
|
||||||
|
CreateStreamedTextResponse("Streamed OCR text"),
|
||||||
|
"text/event-stream");
|
||||||
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
|
() => handler,
|
||||||
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
|
var options = new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
Agent =
|
||||||
|
{
|
||||||
|
Endpoint = "https://summary.local",
|
||||||
|
Model = "vision-model",
|
||||||
|
Key = "agent-key",
|
||||||
|
UseStreaming = true
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var result = await client.ExtractAsync(
|
||||||
|
screenshotPath,
|
||||||
|
"Extract screenshot.",
|
||||||
|
options,
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal("Streamed OCR text", result.Text);
|
||||||
|
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||||
|
Assert.True(payload.RootElement.GetProperty("stream").GetBoolean());
|
||||||
|
var message = Assert.Single(payload.RootElement.GetProperty("input").EnumerateArray());
|
||||||
|
Assert.Equal("message", message.GetProperty("type").GetString());
|
||||||
|
var content = message.GetProperty("content").EnumerateArray().ToArray();
|
||||||
|
Assert.Equal("input_text", content[0].GetProperty("type").GetString());
|
||||||
|
Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
|
||||||
|
Assert.Equal("input_image", content[1].GetProperty("type").GetString());
|
||||||
|
Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
|
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
|
||||||
{
|
{
|
||||||
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||||
var handler = new RecordingHandler("""
|
var handler = new RecordingHandler(CreateNonStreamingTextResponse("Visible slide text"));
|
||||||
{
|
|
||||||
"output": [
|
|
||||||
{
|
|
||||||
"content": [
|
|
||||||
{ "type": "output_text", "text": "Visible slide text" }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""");
|
|
||||||
var client = new LiteLlmScreenshotOcrClient(
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
() => handler,
|
() => handler,
|
||||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
@@ -37,7 +66,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
{
|
{
|
||||||
Endpoint = "https://summary.local",
|
Endpoint = "https://summary.local",
|
||||||
Model = "summary-model",
|
Model = "summary-model",
|
||||||
Key = "agent-key"
|
Key = "agent-key",
|
||||||
|
UseStreaming = false
|
||||||
},
|
},
|
||||||
Screenshots =
|
Screenshots =
|
||||||
{
|
{
|
||||||
@@ -62,6 +92,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
Assert.Equal("Bearer", handler.Authorization?.Scheme);
|
Assert.Equal("Bearer", handler.Authorization?.Scheme);
|
||||||
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
|
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
|
||||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||||
|
Assert.False(payload.RootElement.GetProperty("stream").GetBoolean());
|
||||||
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
|
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
|
||||||
var content = payload.RootElement
|
var content = payload.RootElement
|
||||||
.GetProperty("input")[0]
|
.GetProperty("input")[0]
|
||||||
@@ -74,7 +105,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
|
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
|
||||||
{
|
{
|
||||||
var screenshotPath = await CreateScreenshotAsync([4, 5, 6]);
|
var screenshotPath = await CreateScreenshotAsync([4, 5, 6]);
|
||||||
var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
|
var handler = new RecordingHandler(CreateNonStreamingTextResponse("OCR result"));
|
||||||
var client = new LiteLlmScreenshotOcrClient(
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
() => handler,
|
() => handler,
|
||||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
@@ -84,7 +115,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
{
|
{
|
||||||
Endpoint = "https://summary.local",
|
Endpoint = "https://summary.local",
|
||||||
Model = "summary-model",
|
Model = "summary-model",
|
||||||
Key = "agent-key"
|
Key = "agent-key",
|
||||||
|
UseStreaming = false
|
||||||
},
|
},
|
||||||
Screenshots =
|
Screenshots =
|
||||||
{
|
{
|
||||||
@@ -113,11 +145,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
|
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
|
||||||
{
|
{
|
||||||
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
||||||
var handler = new RecordingHandler("""
|
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||||
{
|
"""
|
||||||
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
|
Slide text
|
||||||
}
|
|
||||||
""");
|
```json
|
||||||
|
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 } }
|
||||||
|
```
|
||||||
|
"""));
|
||||||
var client = new LiteLlmScreenshotOcrClient(
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
() => handler,
|
() => handler,
|
||||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
@@ -125,7 +160,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
{
|
{
|
||||||
Agent =
|
Agent =
|
||||||
{
|
{
|
||||||
Key = "agent-key"
|
Key = "agent-key",
|
||||||
|
UseStreaming = false
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -148,11 +184,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText()
|
public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText()
|
||||||
{
|
{
|
||||||
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
|
||||||
var handler = new RecordingHandler("""
|
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||||
{
|
"""
|
||||||
"output_text": "Visible participant tiles: Ada and Grace.\n\n```json\n{ \"crop\": null, \"attendees\": [\"Ada Lovelace\", \"Grace Hopper\"] }\n```"
|
Visible participant tiles: Ada and Grace.
|
||||||
}
|
|
||||||
""");
|
```json
|
||||||
|
{ "crop": null, "attendees": ["Ada Lovelace", "Grace Hopper"] }
|
||||||
|
```
|
||||||
|
"""));
|
||||||
var client = new LiteLlmScreenshotOcrClient(
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
() => handler,
|
() => handler,
|
||||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
@@ -160,7 +199,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
{
|
{
|
||||||
Agent =
|
Agent =
|
||||||
{
|
{
|
||||||
Key = "agent-key"
|
Key = "agent-key",
|
||||||
|
UseStreaming = false
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -178,11 +218,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop()
|
public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop()
|
||||||
{
|
{
|
||||||
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
|
||||||
var handler = new RecordingHandler("""
|
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
|
||||||
{
|
"""
|
||||||
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 }, \"attendees\": \"Ada\" }\n```"
|
Slide text
|
||||||
}
|
|
||||||
""");
|
```json
|
||||||
|
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 }, "attendees": "Ada" }
|
||||||
|
```
|
||||||
|
"""));
|
||||||
var client = new LiteLlmScreenshotOcrClient(
|
var client = new LiteLlmScreenshotOcrClient(
|
||||||
() => handler,
|
() => handler,
|
||||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||||
@@ -190,7 +233,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
{
|
{
|
||||||
Agent =
|
Agent =
|
||||||
{
|
{
|
||||||
Key = "agent-key"
|
Key = "agent-key",
|
||||||
|
UseStreaming = false
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -208,10 +252,12 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
private sealed class RecordingHandler : HttpMessageHandler
|
private sealed class RecordingHandler : HttpMessageHandler
|
||||||
{
|
{
|
||||||
private readonly string responseBody;
|
private readonly string responseBody;
|
||||||
|
private readonly string mediaType;
|
||||||
|
|
||||||
public RecordingHandler(string responseBody)
|
public RecordingHandler(string responseBody, string mediaType = "application/json")
|
||||||
{
|
{
|
||||||
this.responseBody = responseBody;
|
this.responseBody = responseBody;
|
||||||
|
this.mediaType = mediaType;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Uri? RequestUri { get; private set; }
|
public Uri? RequestUri { get; private set; }
|
||||||
@@ -231,7 +277,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
: await request.Content.ReadAsStringAsync(cancellationToken);
|
: await request.Content.ReadAsStringAsync(cancellationToken);
|
||||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||||
{
|
{
|
||||||
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
|
Content = new StringContent(responseBody, Encoding.UTF8, mediaType)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -249,6 +295,55 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
|||||||
return stream.ToArray();
|
return stream.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static string CreateNonStreamingTextResponse(string text)
|
||||||
|
{
|
||||||
|
return JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
id = "resp_ocr",
|
||||||
|
created_at = 1779147100,
|
||||||
|
model = "vision-model",
|
||||||
|
@object = "response",
|
||||||
|
output = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
id = "msg_ocr",
|
||||||
|
type = "message",
|
||||||
|
status = "completed",
|
||||||
|
content = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
type = "output_text",
|
||||||
|
annotations = Array.Empty<object>(),
|
||||||
|
text
|
||||||
|
}
|
||||||
|
},
|
||||||
|
role = "assistant"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
parallel_tool_calls = true,
|
||||||
|
status = "completed",
|
||||||
|
store = false
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string CreateStreamedTextResponse(string text)
|
||||||
|
{
|
||||||
|
var delta = JsonSerializer.Serialize(new
|
||||||
|
{
|
||||||
|
type = "response.output_text.delta",
|
||||||
|
item_id = "msg_ocr",
|
||||||
|
output_index = 0,
|
||||||
|
content_index = 0,
|
||||||
|
delta = text
|
||||||
|
});
|
||||||
|
return
|
||||||
|
$"data: {delta}{Environment.NewLine}{Environment.NewLine}" +
|
||||||
|
"""data: {"type":"response.completed","response":{"id":"resp_ocr","created_at":1779147100,"model":"vision-model","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false}}""" +
|
||||||
|
$"{Environment.NewLine}{Environment.NewLine}data: [DONE]{Environment.NewLine}{Environment.NewLine}";
|
||||||
|
}
|
||||||
|
|
||||||
private static async Task<string> CreateScreenshotAsync(byte[] bytes)
|
private static async Task<string> CreateScreenshotAsync(byte[] bytes)
|
||||||
{
|
{
|
||||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||||
|
|||||||
@@ -9,10 +9,10 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="coverlet.collector" Version="10.0.1" />
|
<PackageReference Include="coverlet.collector" Version="10.0.1" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.9" />
|
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
|
||||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.7.0" />
|
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
|
||||||
<PackageReference Include="xunit" Version="2.9.3" />
|
<PackageReference Include="xunit" Version="2.9.3" />
|
||||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
|
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -135,6 +135,72 @@ public sealed class MeetingNoteStoreTests
|
|||||||
Assert.Equal("User notes.", loaded.UserNotes);
|
Assert.Equal("User notes.", loaded.UserNotes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task StoreEscapesApostrophePrefixedAttendeesBeforeWritingFrontmatter()
|
||||||
|
{
|
||||||
|
var (store, saved) = await SaveNoteAsync(
|
||||||
|
title: "Escaped Attendees",
|
||||||
|
attendees: ["'Ada Lovelace"],
|
||||||
|
projects: [],
|
||||||
|
userNotes: "Discuss attendee import.");
|
||||||
|
var content = await File.ReadAllTextAsync(saved.Path);
|
||||||
|
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Contains("- \"'Ada Lovelace\"", content);
|
||||||
|
Assert.Equal(["'Ada Lovelace"], loaded.Frontmatter.Attendees);
|
||||||
|
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("Ada # platform lead")]
|
||||||
|
[InlineData("- Ada Lovelace")]
|
||||||
|
[InlineData("? Ada Lovelace")]
|
||||||
|
[InlineData("{Ada: Platform}")]
|
||||||
|
[InlineData("*Ada")]
|
||||||
|
[InlineData("&Ada")]
|
||||||
|
[InlineData("!Ada")]
|
||||||
|
[InlineData("| Ada")]
|
||||||
|
[InlineData("> Ada")]
|
||||||
|
[InlineData("@Ada")]
|
||||||
|
[InlineData("`Ada")]
|
||||||
|
[InlineData("true")]
|
||||||
|
[InlineData("null")]
|
||||||
|
[InlineData("2026-07-08")]
|
||||||
|
[InlineData("Ada \"The Architect\" Lovelace")]
|
||||||
|
[InlineData("C:\\People\\Ada")]
|
||||||
|
public async Task StoreEscapesYamlSensitiveAttendeesBeforeWritingFrontmatter(string attendee)
|
||||||
|
{
|
||||||
|
var (store, saved) = await SaveNoteAsync(
|
||||||
|
title: "Escaped Attendees",
|
||||||
|
attendees: [attendee],
|
||||||
|
projects: [],
|
||||||
|
userNotes: "Discuss attendee import.");
|
||||||
|
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal([attendee], loaded.Frontmatter.Attendees);
|
||||||
|
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("Planning # Q3")]
|
||||||
|
[InlineData("- Planning")]
|
||||||
|
[InlineData("{Planning: Q3}")]
|
||||||
|
[InlineData("true")]
|
||||||
|
[InlineData("2026-07-08")]
|
||||||
|
public async Task StoreEscapesYamlSensitiveScalarAndProjectFrontmatterValues(string value)
|
||||||
|
{
|
||||||
|
var (store, saved) = await SaveNoteAsync(
|
||||||
|
title: value,
|
||||||
|
attendees: [],
|
||||||
|
projects: [value],
|
||||||
|
userNotes: "Discuss YAML escaping.");
|
||||||
|
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(value, loaded.Frontmatter.Title);
|
||||||
|
Assert.Equal([value], loaded.Frontmatter.Projects);
|
||||||
|
Assert.Equal("Discuss YAML escaping.", loaded.UserNotes);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ActionLinkEscapesSummaryFileName()
|
public void ActionLinkEscapesSummaryFileName()
|
||||||
{
|
{
|
||||||
@@ -147,6 +213,26 @@ public sealed class MeetingNoteStoreTests
|
|||||||
link);
|
link);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async Task<(MarkdownMeetingNoteStore Store, MeetingNote Saved)> SaveNoteAsync(
|
||||||
|
string title,
|
||||||
|
IReadOnlyList<string> attendees,
|
||||||
|
IReadOnlyList<string> projects,
|
||||||
|
string userNotes)
|
||||||
|
{
|
||||||
|
var (vaultRoot, store) = CreateStore();
|
||||||
|
var note = MeetingNoteTemplate.Create(
|
||||||
|
title: title,
|
||||||
|
attendees: attendees,
|
||||||
|
projects: projects,
|
||||||
|
transcriptPath: Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"),
|
||||||
|
assistantContextPath: Path.Combine(vaultRoot, "Meetings", "Assistant Context", "20260519-context.md"),
|
||||||
|
summaryPath: Path.Combine(vaultRoot, "Meetings", "Summaries", "20260519-summary.md"),
|
||||||
|
userNotes: userNotes);
|
||||||
|
|
||||||
|
var saved = await store.SaveAsync(note, CancellationToken.None);
|
||||||
|
return (store, saved);
|
||||||
|
}
|
||||||
|
|
||||||
private static (string VaultRoot, MarkdownMeetingNoteStore Store) CreateStore()
|
private static (string VaultRoot, MarkdownMeetingNoteStore Store) CreateStore()
|
||||||
{
|
{
|
||||||
var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
|||||||
@@ -7,6 +7,37 @@ namespace MeetingAssistant.Tests;
|
|||||||
|
|
||||||
public sealed class MeetingSummaryInstructionBuilderTests
|
public sealed class MeetingSummaryInstructionBuilderTests
|
||||||
{
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var projectsRoot = Path.Combine(root, "Projects");
|
||||||
|
var artifacts = CreateArtifacts(root);
|
||||||
|
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
|
||||||
|
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||||
|
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
|
||||||
|
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
|
||||||
|
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
Agent = new AgentOptions { InitialPrompt = "Custom base." },
|
||||||
|
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
|
||||||
|
}));
|
||||||
|
|
||||||
|
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.StartsWith("Custom base.", instructions);
|
||||||
|
Assert.Contains("# Alpha", instructions);
|
||||||
|
Assert.Contains("Alpha Platform", instructions);
|
||||||
|
Assert.Contains("Builds meeting automation", instructions);
|
||||||
|
Assert.Contains("PROJECT.md", instructions);
|
||||||
|
Assert.Contains("256", instructions);
|
||||||
|
Assert.Contains("request_project_association", instructions);
|
||||||
|
Assert.Contains("request_workflow_change", instructions);
|
||||||
|
Assert.DoesNotContain("Internal notes", instructions);
|
||||||
|
Assert.DoesNotContain("Do not expose", instructions);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
|
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
|
||||||
{
|
{
|
||||||
@@ -24,6 +55,7 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
|||||||
Assert.Contains("You are the Meeting Assistant summary agent.", instructions);
|
Assert.Contains("You are the Meeting Assistant summary agent.", instructions);
|
||||||
Assert.Contains("include only the most relevant cropped screenshots", instructions);
|
Assert.Contains("include only the most relevant cropped screenshots", instructions);
|
||||||
Assert.Contains("Do not include every cropped screenshot", instructions);
|
Assert.Contains("Do not include every cropped screenshot", instructions);
|
||||||
|
Assert.Contains("encode spaces in image-link targets as `%20`", instructions);
|
||||||
Assert.Contains("Use add_attendee and remove_attendee", instructions);
|
Assert.Contains("Use add_attendee and remove_attendee", instructions);
|
||||||
Assert.Contains("partial screenshot", instructions);
|
Assert.Contains("partial screenshot", instructions);
|
||||||
Assert.Contains("override_speaker", instructions);
|
Assert.Contains("override_speaker", instructions);
|
||||||
@@ -39,6 +71,27 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
|||||||
Assert.Contains("title parameter", instructions);
|
Assert.Contains("title parameter", instructions);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DefaultPromptTreatsAssistantContextAsMeetingMemoryForUncertainty()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var artifacts = CreateArtifacts(root);
|
||||||
|
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, []);
|
||||||
|
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
Agent = new AgentOptions { InitialPrompt = " " },
|
||||||
|
Vault = new VaultOptions { ProjectsFolder = Path.Combine(root, "Projects") }
|
||||||
|
}));
|
||||||
|
|
||||||
|
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Contains("meeting-specific memory", instructions);
|
||||||
|
Assert.Contains("unexpected problems", instructions);
|
||||||
|
Assert.Contains("missing information", instructions);
|
||||||
|
Assert.Contains("assumptions", instructions);
|
||||||
|
Assert.Contains("write_context", instructions);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BuilderUsesConfiguredPrompt()
|
public async Task BuilderUsesConfiguredPrompt()
|
||||||
{
|
{
|
||||||
@@ -53,7 +106,8 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
|||||||
|
|
||||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Equal("Custom summarizer instructions.", instructions);
|
Assert.StartsWith("Custom summarizer instructions.", instructions);
|
||||||
|
Assert.Contains("request_project_association", instructions);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -74,21 +128,12 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
|||||||
|
|
||||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||||
|
|
||||||
var expected = """
|
Assert.StartsWith("Base.", instructions);
|
||||||
Base.
|
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
|
||||||
|
Assert.Contains("# Alpha", instructions);
|
||||||
---
|
Assert.Contains("Alpha instructions.", instructions);
|
||||||
projects:
|
Assert.Contains("# Beta", instructions);
|
||||||
|
Assert.Contains("Beta instructions.", instructions);
|
||||||
# Alpha
|
|
||||||
|
|
||||||
Alpha instructions.
|
|
||||||
|
|
||||||
# Beta
|
|
||||||
|
|
||||||
Beta instructions.
|
|
||||||
""";
|
|
||||||
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static MeetingSessionArtifacts CreateArtifacts(string root)
|
private static MeetingSessionArtifacts CreateArtifacts(string root)
|
||||||
|
|||||||
@@ -189,7 +189,9 @@ public sealed class MeetingSummaryRetryRunnerTests
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await stateChanged.Task.WaitAsync(TimeSpan.FromMilliseconds(100));
|
await Task.WhenAny(
|
||||||
|
stateChanged.Task,
|
||||||
|
Task.Delay(TimeSpan.FromMilliseconds(100)));
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new TimeoutException($"Expected {count} state changes.");
|
throw new TimeoutException($"Expected {count} state changes.");
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
using MeetingAssistant.Recording;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class MicrophoneAudioSourceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task CaptureMovesToNewlyResolvedMicrophoneWhenCurrentCaptureFails()
|
||||||
|
{
|
||||||
|
var captureSources = new SequenceMicrophoneCaptureSourceFactory(
|
||||||
|
new FailingAfterChunkAudioSource(Pcm16(1_000)),
|
||||||
|
new ActiveAudioSource(Pcm16(2_000)));
|
||||||
|
var source = new MicrophoneAudioSource(
|
||||||
|
captureSources,
|
||||||
|
NullLogger<MicrophoneAudioSource>.Instance,
|
||||||
|
TimeSpan.Zero);
|
||||||
|
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
await using var chunks = source
|
||||||
|
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||||
|
.GetAsyncEnumerator(cancellation.Token);
|
||||||
|
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
Assert.Equal(2, captureSources.CaptureCreationCount);
|
||||||
|
|
||||||
|
await cancellation.CancelAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CaptureWaitsForMicrophoneToBecomeAvailable()
|
||||||
|
{
|
||||||
|
var captureSources = new InitiallyUnavailableMicrophoneCaptureSourceFactory(
|
||||||
|
new ActiveAudioSource(Pcm16(3_000)));
|
||||||
|
var source = new MicrophoneAudioSource(
|
||||||
|
captureSources,
|
||||||
|
NullLogger<MicrophoneAudioSource>.Instance,
|
||||||
|
TimeSpan.Zero);
|
||||||
|
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
await using var chunks = source
|
||||||
|
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
|
||||||
|
.GetAsyncEnumerator(cancellation.Token);
|
||||||
|
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal(3_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
Assert.Equal(2, captureSources.CaptureCreationCount);
|
||||||
|
|
||||||
|
await cancellation.CancelAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
|
||||||
|
{
|
||||||
|
var selection = new MicrophoneDeviceSelection();
|
||||||
|
var devices = new SelectableCaptureDevices(selection);
|
||||||
|
var prompts = new AcceptingPrompt();
|
||||||
|
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||||
|
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||||
|
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||||
|
TimeSpan.Zero, monitor, selection);
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
|
||||||
|
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
|
||||||
|
var nextChunk = chunks.MoveNextAsync().AsTask();
|
||||||
|
Assert.False(nextChunk.IsCompleted);
|
||||||
|
Assert.Null(selection.SelectedDeviceId);
|
||||||
|
|
||||||
|
prompts.Response.SetResult(true);
|
||||||
|
|
||||||
|
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||||
|
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
Assert.Equal("new", selection.SelectedDeviceId);
|
||||||
|
Assert.False(lifetime.IsCancellationRequested);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await lifetime.CancelAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
|
||||||
|
{
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
var abandonedCapture = new DisposableAudioSource();
|
||||||
|
var factory = new CallbackCaptureFactory(() =>
|
||||||
|
{
|
||||||
|
lifetime.Cancel();
|
||||||
|
return abandonedCapture;
|
||||||
|
});
|
||||||
|
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
|
||||||
|
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
|
||||||
|
|
||||||
|
Assert.False(await chunks.MoveNextAsync());
|
||||||
|
Assert.True(abandonedCapture.Disposed);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
|
||||||
|
{
|
||||||
|
var selection = new MicrophoneDeviceSelection();
|
||||||
|
var devices = new SwitchDuringCreationDevices(selection);
|
||||||
|
var prompts = new AcceptingPrompt();
|
||||||
|
prompts.Response.SetResult(true);
|
||||||
|
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||||
|
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||||
|
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||||
|
TimeSpan.Zero, monitor, selection);
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Assert.True(await chunks.MoveNextAsync());
|
||||||
|
Assert.Equal("new", selection.SelectedDeviceId);
|
||||||
|
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||||
|
Assert.True(devices.AbandonedCapture.Disposed);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await lifetime.CancelAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
|
||||||
|
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
private MicrophoneDevice[] available = [new("old", "Built-in")];
|
||||||
|
private int creations;
|
||||||
|
private volatile bool switchObserved;
|
||||||
|
public DisposableAudioSource AbandonedCapture { get; } = new();
|
||||||
|
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||||
|
{
|
||||||
|
if (selection.SelectedDeviceId == "new") switchObserved = true;
|
||||||
|
return Volatile.Read(ref available);
|
||||||
|
}
|
||||||
|
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||||
|
=> new(GetAvailableMicrophones(), available[0]);
|
||||||
|
|
||||||
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
|
||||||
|
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
|
||||||
|
// Let the acceptance callback finish before returning the already-opened source.
|
||||||
|
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
|
||||||
|
throw new TimeoutException("The new microphone was not selected during capture creation.");
|
||||||
|
return AbandonedCapture;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
|
||||||
|
{
|
||||||
|
public bool Disposed { get; private set; }
|
||||||
|
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||||
|
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
|
||||||
|
public void Dispose() => Disposed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
|
||||||
|
{
|
||||||
|
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Shown.TrySetResult(microphone);
|
||||||
|
return Response.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
|
||||||
|
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
|
||||||
|
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
|
||||||
|
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||||
|
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
|
||||||
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||||
|
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] Pcm16(short sample)
|
||||||
|
{
|
||||||
|
return BitConverter.GetBytes(sample);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class SequenceMicrophoneCaptureSourceFactory(params IMeetingAudioSource[] sources)
|
||||||
|
: IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
private readonly Queue<IMeetingAudioSource> sources = new(sources);
|
||||||
|
|
||||||
|
public int CaptureCreationCount { get; private set; }
|
||||||
|
|
||||||
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
CaptureCreationCount++;
|
||||||
|
return sources.Dequeue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class InitiallyUnavailableMicrophoneCaptureSourceFactory(IMeetingAudioSource availableSource)
|
||||||
|
: IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
public int CaptureCreationCount { get; private set; }
|
||||||
|
|
||||||
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
CaptureCreationCount++;
|
||||||
|
if (CaptureCreationCount == 1)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("No microphone is currently available.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return availableSource;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class FailingAfterChunkAudioSource(byte[] pcm) : IMeetingAudioSource
|
||||||
|
{
|
||||||
|
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||||
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
yield return new AudioChunk(pcm, 16000, 1);
|
||||||
|
throw new InvalidOperationException("The active microphone was disconnected.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class ActiveAudioSource(byte[] pcm) : IMeetingAudioSource
|
||||||
|
{
|
||||||
|
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||||
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
yield return new AudioChunk(pcm, 16000, 1);
|
||||||
|
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -51,4 +51,17 @@ public sealed class MicrophoneSelectionTests
|
|||||||
|
|
||||||
Assert.Equal("runtime-id", selected?.Id);
|
Assert.Equal("runtime-id", selected?.Id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void UnavailableDefaultMicrophoneFallsBackToAnotherActiveDevice()
|
||||||
|
{
|
||||||
|
var selection = new MicrophoneDeviceSelection();
|
||||||
|
|
||||||
|
var selected = selection.Resolve(
|
||||||
|
configuredDeviceId: null,
|
||||||
|
new MicrophoneDevice("disconnected-id", "disconnected microphone"),
|
||||||
|
[new MicrophoneDevice("backup-id", "backup microphone")]);
|
||||||
|
|
||||||
|
Assert.Equal("backup-id", selected?.Id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
using System.Threading.Channels;
|
||||||
|
using MeetingAssistant.Recording;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class MicrophoneSwitchMonitorTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
|
||||||
|
{
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
var devices = new MutableDevices();
|
||||||
|
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
|
||||||
|
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||||
|
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||||
|
var switched = false;
|
||||||
|
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||||
|
devices.Available = [new("usb", "USB microphone")];
|
||||||
|
|
||||||
|
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
Assert.True(prompts.WasShown);
|
||||||
|
Assert.False(switched);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
|
||||||
|
{
|
||||||
|
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
|
||||||
|
var newDevice = new MicrophoneDevice("new", "USB headset");
|
||||||
|
var devices = new MutableDevices { Available = [oldDevice] };
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var monitor = CreateMonitor(devices, prompts);
|
||||||
|
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
devices.Available = [oldDevice, newDevice];
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
Assert.Equal(newDevice, prompt.Device);
|
||||||
|
Assert.False(switched.Task.IsCompleted);
|
||||||
|
await devices.WaitForPollsAsync();
|
||||||
|
Assert.False(prompts.HasMore);
|
||||||
|
|
||||||
|
prompt.Response.SetResult(true);
|
||||||
|
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||||
|
await devices.WaitForPollsAsync();
|
||||||
|
Assert.False(prompts.HasMore);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await lifetime.CancelAsync();
|
||||||
|
await monitoring;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("declined")]
|
||||||
|
[InlineData("removed")]
|
||||||
|
[InlineData("expired")]
|
||||||
|
[InlineData("ended")]
|
||||||
|
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
|
||||||
|
{
|
||||||
|
var device = new MicrophoneDevice("usb", "USB microphone");
|
||||||
|
var devices = new MutableDevices();
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var monitor = CreateMonitor(devices, prompts,
|
||||||
|
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
|
||||||
|
var switched = false;
|
||||||
|
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||||
|
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
devices.Available = [device];
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
if (outcome == "declined")
|
||||||
|
{
|
||||||
|
prompt.Response.SetResult(false);
|
||||||
|
await devices.WaitForPollsAsync();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (outcome == "removed") devices.Available = [];
|
||||||
|
if (outcome == "ended") await lifetime.CancelAsync();
|
||||||
|
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
|
||||||
|
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
prompt.Response.TrySetResult(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
await lifetime.CancelAsync();
|
||||||
|
await monitoring;
|
||||||
|
Assert.False(switched);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await lifetime.CancelAsync();
|
||||||
|
await monitoring;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
|
||||||
|
TimeSpan? promptLifetime = null) => new(devices, prompts,
|
||||||
|
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
|
||||||
|
promptLifetime ?? TimeSpan.FromMinutes(1));
|
||||||
|
|
||||||
|
private sealed class MutableDevices : IMicrophoneDeviceProvider
|
||||||
|
{
|
||||||
|
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
|
||||||
|
public volatile MicrophoneDevice[] Available = [];
|
||||||
|
public Action? BeforeRead { get; set; }
|
||||||
|
|
||||||
|
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||||
|
{
|
||||||
|
BeforeRead?.Invoke();
|
||||||
|
polls.Writer.TryWrite(true);
|
||||||
|
return Available;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||||
|
=> new(Available, Available.FirstOrDefault());
|
||||||
|
|
||||||
|
public async Task WaitForPollsAsync()
|
||||||
|
{
|
||||||
|
while (polls.Reader.TryRead(out _)) { }
|
||||||
|
for (var i = 0; i < 3; i++)
|
||||||
|
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
|
||||||
|
{
|
||||||
|
public bool WasShown { get; private set; }
|
||||||
|
|
||||||
|
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
WasShown = true;
|
||||||
|
beforeAcceptance();
|
||||||
|
return Task.FromResult(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
|
||||||
|
{
|
||||||
|
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
|
||||||
|
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||||
|
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var prompt = new Prompt(microphone, cancellationToken);
|
||||||
|
requests.Writer.TryWrite(prompt);
|
||||||
|
return await prompt.Response.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
|
||||||
|
{
|
||||||
|
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,4 +25,16 @@ public sealed class NotificationActivationArgumentsTests
|
|||||||
Assert.Equal("abc123", arguments["promptId"]);
|
Assert.Equal("abc123", arguments["promptId"]);
|
||||||
Assert.Equal("continue", arguments["response"]);
|
Assert.Equal("continue", arguments["response"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
|
||||||
|
{
|
||||||
|
Assert.Contains(
|
||||||
|
MeetingInactivityPromptActions.All,
|
||||||
|
action => action.Content == "Pause transcription" &&
|
||||||
|
action.ResponseArgument == "pause");
|
||||||
|
Assert.Equal(
|
||||||
|
MeetingInactivityPromptResponse.Pause,
|
||||||
|
MeetingInactivityPromptActions.ParseResponse("pause"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ public sealed class OutlookMeetingCandidateSelectorTests
|
|||||||
{
|
{
|
||||||
var now = new DateTime(2026, 5, 20, 10, 0, 0);
|
var now = new DateTime(2026, 5, 20, 10, 0, 0);
|
||||||
var endingOverlap = new Candidate(now.AddMinutes(-25), now.AddMinutes(2));
|
var endingOverlap = new Candidate(now.AddMinutes(-25), now.AddMinutes(2));
|
||||||
var upcoming = new Candidate(now.AddMinutes(5), now.AddMinutes(35));
|
var upcoming = new Candidate(now.AddMinutes(1), now.AddMinutes(31));
|
||||||
|
|
||||||
var selected = OutlookMeetingCandidateSelector.Select(
|
var selected = OutlookMeetingCandidateSelector.Select(
|
||||||
[endingOverlap, upcoming],
|
[endingOverlap, upcoming],
|
||||||
|
|||||||
@@ -1,10 +1,90 @@
|
|||||||
using MeetingAssistant.MeetingNotes;
|
using MeetingAssistant.MeetingNotes;
|
||||||
using MeetingAssistant.Summary;
|
using MeetingAssistant.Summary;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
namespace MeetingAssistant.Tests;
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
public sealed class ProjectKnowledgeToolTests
|
public sealed class ProjectKnowledgeToolTests
|
||||||
{
|
{
|
||||||
|
[Theory]
|
||||||
|
[InlineData("name: Incomplete project")]
|
||||||
|
[InlineData("description: Missing name")]
|
||||||
|
[InlineData("name: ' '\ndescription: Missing name")]
|
||||||
|
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||||
|
Directory.CreateDirectory(projectRoot);
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
|
||||||
|
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||||
|
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||||
|
|
||||||
|
using var catalog = JsonDocument.Parse(await tools.ListProjects());
|
||||||
|
|
||||||
|
var project = Assert.Single(catalog.RootElement.EnumerateArray());
|
||||||
|
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
|
||||||
|
Assert.Equal("", project.GetProperty("description").GetString());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||||
|
Directory.CreateDirectory(projectRoot);
|
||||||
|
var artifacts = CreateArtifacts(root);
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||||
|
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
|
||||||
|
var tools = new MeetingSummaryTools(artifacts,
|
||||||
|
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||||
|
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
|
||||||
|
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
|
||||||
|
|
||||||
|
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||||
|
$"description: {new string('b', 257)}", from: 3, to: 3);
|
||||||
|
|
||||||
|
Assert.StartsWith("Refused:", rejected);
|
||||||
|
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
|
||||||
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
|
||||||
|
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||||
|
"description: Updated project purpose", from: 3, to: 3));
|
||||||
|
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
|
||||||
|
Assert.Contains("owner: Ada", updated);
|
||||||
|
Assert.Contains("User-authored project notes.", updated);
|
||||||
|
Assert.Contains("Updated project purpose", await tools.ListProjects());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var projectsRoot = Path.Combine(root, "Projects");
|
||||||
|
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||||
|
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
|
||||||
|
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||||
|
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
|
||||||
|
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||||
|
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
|
||||||
|
|
||||||
|
var catalog = await tools.ListProjects();
|
||||||
|
|
||||||
|
using var json = JsonDocument.Parse(catalog);
|
||||||
|
var projects = json.RootElement.EnumerateArray().ToArray();
|
||||||
|
Assert.Equal(3, projects.Length);
|
||||||
|
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
|
||||||
|
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
|
||||||
|
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
|
||||||
|
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
|
||||||
|
Assert.Equal("", projects[1].GetProperty("description").GetString());
|
||||||
|
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
|
||||||
|
Assert.Equal("", projects[2].GetProperty("description").GetString());
|
||||||
|
Assert.DoesNotContain("Private", catalog);
|
||||||
|
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
|
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
|
||||||
{
|
{
|
||||||
@@ -48,7 +128,7 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
Assert.Contains("MeetingAssistant", await tools.ListProjects());
|
||||||
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
|
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
|
||||||
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
|
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
|
||||||
|
|
||||||
@@ -56,15 +136,15 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
|
|
||||||
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
|
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
|
||||||
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
|
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
|
||||||
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
||||||
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
||||||
Assert.Equal(
|
Assert.Equal(
|
||||||
"README.md:2 Second alpha line",
|
"README.md:2 Second alpha line",
|
||||||
await tools.Search("alpha"));
|
await tools.Search("alpha"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
|
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
|
||||||
{
|
{
|
||||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
var projectsRoot = Path.Combine(root, "Projects");
|
var projectsRoot = Path.Combine(root, "Projects");
|
||||||
@@ -88,7 +168,8 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
|
||||||
|
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -100,8 +181,11 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
Directory.CreateDirectory(projectRoot);
|
Directory.CreateDirectory(projectRoot);
|
||||||
var projectFile = Path.Combine(projectRoot, "notes.md");
|
var projectFile = Path.Combine(projectRoot, "notes.md");
|
||||||
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
|
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
|
||||||
|
var artifacts = CreateArtifacts(root);
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||||
|
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||||
var tools = new MeetingSummaryTools(
|
var tools = new MeetingSummaryTools(
|
||||||
CreateArtifacts(root),
|
artifacts,
|
||||||
new MeetingAssistantOptions
|
new MeetingAssistantOptions
|
||||||
{
|
{
|
||||||
Vault =
|
Vault =
|
||||||
@@ -132,8 +216,13 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
var projectsRoot = Path.Combine(root, "Projects");
|
var projectsRoot = Path.Combine(root, "Projects");
|
||||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
|
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
|
||||||
|
var artifacts = CreateArtifacts(root);
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||||
|
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||||
|
var outsidePath = Path.Combine(projectsRoot, "outside.md");
|
||||||
|
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
|
||||||
var tools = new MeetingSummaryTools(
|
var tools = new MeetingSummaryTools(
|
||||||
CreateArtifacts(root),
|
artifacts,
|
||||||
new MeetingAssistantOptions
|
new MeetingAssistantOptions
|
||||||
{
|
{
|
||||||
Vault =
|
Vault =
|
||||||
@@ -144,6 +233,7 @@ public sealed class ProjectKnowledgeToolTests
|
|||||||
|
|
||||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
|
||||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
|
||||||
|
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
|
||||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
|
||||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
|
||||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
|
||||||
|
|||||||
@@ -18,31 +18,8 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
|||||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||||
var service = new PyannoteDiarizationWarmupHostedService(
|
var service = new PyannoteDiarizationWarmupHostedService(
|
||||||
finalizer,
|
finalizer,
|
||||||
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
|
new FakeLaunchProfileOptionsProvider(
|
||||||
{
|
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
|
||||||
Recording = { TranscriptionProvider = "azure-speech" },
|
|
||||||
SpeakerIdentification =
|
|
||||||
{
|
|
||||||
PyannoteValidation =
|
|
||||||
{
|
|
||||||
Enabled = true,
|
|
||||||
Diarization =
|
|
||||||
{
|
|
||||||
Enabled = true,
|
|
||||||
DockerCommand = "docker",
|
|
||||||
Image = "meeting-assistant-pyannote-validation:local",
|
|
||||||
ModelsFolder = Path.Combine(
|
|
||||||
Path.GetTempPath(),
|
|
||||||
"meeting-assistant-tests",
|
|
||||||
Guid.NewGuid().ToString("N"),
|
|
||||||
"models"),
|
|
||||||
Token = "hf_test",
|
|
||||||
TokenEnv = "",
|
|
||||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||||
|
|
||||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||||
@@ -54,23 +31,109 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
|||||||
Assert.True(commandRunner.RunCancellationWasObserved);
|
Assert.True(commandRunner.RunCancellationWasObserved);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
|
||||||
|
{
|
||||||
|
var commandRunner = new BlockingCommandRunner();
|
||||||
|
var finalizer = new PyannoteTranscriptFinalizer(
|
||||||
|
commandRunner,
|
||||||
|
Options.Create(new MeetingAssistantOptions()),
|
||||||
|
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||||
|
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
|
||||||
|
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
|
||||||
|
var service = new PyannoteDiarizationWarmupHostedService(
|
||||||
|
finalizer,
|
||||||
|
new FakeLaunchProfileOptionsProvider(
|
||||||
|
[
|
||||||
|
new LaunchProfile("english", namedOptions),
|
||||||
|
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
|
||||||
|
]),
|
||||||
|
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||||
|
await commandRunner.WaitForRunAsync();
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
|
||||||
|
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
|
||||||
|
{
|
||||||
|
var commandRunner = new BlockingCommandRunner();
|
||||||
|
var finalizer = new PyannoteTranscriptFinalizer(
|
||||||
|
commandRunner,
|
||||||
|
Options.Create(new MeetingAssistantOptions()),
|
||||||
|
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||||
|
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
|
||||||
|
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||||
|
var service = new PyannoteDiarizationWarmupHostedService(
|
||||||
|
finalizer,
|
||||||
|
new FakeLaunchProfileOptionsProvider(options),
|
||||||
|
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None);
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(50));
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Empty(commandRunner.Commands);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static MeetingAssistantOptions CreateValidationOptions(string image)
|
||||||
|
{
|
||||||
|
return new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
Recording = { TranscriptionProvider = "azure-speech" },
|
||||||
|
SpeakerIdentification =
|
||||||
|
{
|
||||||
|
PyannoteValidation =
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
Diarization =
|
||||||
|
{
|
||||||
|
DockerCommand = "docker",
|
||||||
|
Image = image,
|
||||||
|
ModelsFolder = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"),
|
||||||
|
"models"),
|
||||||
|
Token = "hf_test",
|
||||||
|
TokenEnv = "",
|
||||||
|
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
||||||
{
|
{
|
||||||
private readonly MeetingAssistantOptions options;
|
private readonly IReadOnlyList<LaunchProfile> profiles;
|
||||||
|
|
||||||
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
|
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
|
||||||
|
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
|
||||||
{
|
{
|
||||||
this.options = options;
|
}
|
||||||
|
|
||||||
|
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
|
||||||
|
{
|
||||||
|
this.profiles = profiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
public LaunchProfile GetRequiredProfile(string? name)
|
public LaunchProfile GetRequiredProfile(string? name)
|
||||||
{
|
{
|
||||||
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
|
var profileName = string.IsNullOrWhiteSpace(name)
|
||||||
|
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
|
||||||
|
: name;
|
||||||
|
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
|
||||||
}
|
}
|
||||||
|
|
||||||
public IReadOnlyList<LaunchProfile> GetProfiles()
|
public IReadOnlyList<LaunchProfile> GetProfiles()
|
||||||
{
|
{
|
||||||
return [GetRequiredProfile(null)];
|
return profiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ using MeetingAssistant;
|
|||||||
using MeetingAssistant.Recording;
|
using MeetingAssistant.Recording;
|
||||||
using MeetingAssistant.Speakers;
|
using MeetingAssistant.Speakers;
|
||||||
using MeetingAssistant.Transcription;
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.Logging.Abstractions;
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using NAudio.Wave;
|
using NAudio.Wave;
|
||||||
@@ -57,17 +58,49 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
|||||||
Assert.Empty(commandRunner.Commands);
|
Assert.Empty(commandRunner.Commands);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
|
||||||
|
{
|
||||||
|
var commandRunner = new CapturingCommandRunner(
|
||||||
|
"""
|
||||||
|
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
|
||||||
|
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
|
||||||
|
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
|
||||||
|
""");
|
||||||
|
var modelsFolder = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"),
|
||||||
|
"models");
|
||||||
|
var configuration = new ConfigurationBuilder()
|
||||||
|
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
|
||||||
|
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
|
||||||
|
})
|
||||||
|
.Build();
|
||||||
|
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
|
||||||
|
var validator = CreateValidator(commandRunner, configuredOptions);
|
||||||
|
|
||||||
|
var valid = await validator.ValidateSampleAsync(
|
||||||
|
CreateWav(TimeSpan.FromSeconds(20)),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.True(valid);
|
||||||
|
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
|
||||||
|
}
|
||||||
|
|
||||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||||
CapturingCommandRunner commandRunner,
|
CapturingCommandRunner commandRunner,
|
||||||
bool enabled = true)
|
bool enabled = true)
|
||||||
{
|
{
|
||||||
var finalizer = new PyannoteTranscriptFinalizer(
|
return CreateValidator(
|
||||||
commandRunner,
|
commandRunner,
|
||||||
Options.Create(new MeetingAssistantOptions()),
|
new MeetingAssistantOptions
|
||||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
|
||||||
return new PyannoteSpeakerIdentityMatchValidator(
|
|
||||||
finalizer,
|
|
||||||
Options.Create(new MeetingAssistantOptions
|
|
||||||
{
|
{
|
||||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||||
{
|
{
|
||||||
@@ -76,9 +109,8 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
|||||||
Enabled = enabled,
|
Enabled = enabled,
|
||||||
MinimumSingleSpeakerCoverage = 0.90,
|
MinimumSingleSpeakerCoverage = 0.90,
|
||||||
MinimumMatchingKnownSnippetRatio = 1,
|
MinimumMatchingKnownSnippetRatio = 1,
|
||||||
Diarization = new PyannoteDiarizationOptions
|
Diarization = new PyannoteRuntimeOptions
|
||||||
{
|
{
|
||||||
Enabled = true,
|
|
||||||
BuildImage = false,
|
BuildImage = false,
|
||||||
DockerCommand = "docker",
|
DockerCommand = "docker",
|
||||||
Image = "meeting-assistant-pyannote:local",
|
Image = "meeting-assistant-pyannote:local",
|
||||||
@@ -94,7 +126,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}),
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||||
|
CapturingCommandRunner commandRunner,
|
||||||
|
MeetingAssistantOptions configuredOptions)
|
||||||
|
{
|
||||||
|
var finalizer = new PyannoteTranscriptFinalizer(
|
||||||
|
commandRunner,
|
||||||
|
Options.Create(new MeetingAssistantOptions()),
|
||||||
|
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||||
|
return new PyannoteSpeakerIdentityMatchValidator(
|
||||||
|
finalizer,
|
||||||
|
Options.Create(configuredOptions),
|
||||||
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
|
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -156,9 +156,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
|||||||
commandRunner,
|
commandRunner,
|
||||||
Options.Create(new MeetingAssistantOptions()),
|
Options.Create(new MeetingAssistantOptions()),
|
||||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||||
var explicitDiarization = new PyannoteDiarizationOptions
|
var explicitDiarization = new PyannoteRuntimeOptions
|
||||||
{
|
{
|
||||||
Enabled = true,
|
|
||||||
DockerCommand = "docker",
|
DockerCommand = "docker",
|
||||||
BaseImage = "python:3.11-slim",
|
BaseImage = "python:3.11-slim",
|
||||||
Image = "meeting-assistant-pyannote-azure:local",
|
Image = "meeting-assistant-pyannote-azure:local",
|
||||||
@@ -169,7 +168,7 @@ public sealed class PyannoteTranscriptFinalizerTests
|
|||||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||||
};
|
};
|
||||||
|
|
||||||
await finalizer.FinalizeAsync(
|
await finalizer.FinalizeEnabledAsync(
|
||||||
audioPath,
|
audioPath,
|
||||||
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
|
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
|
||||||
explicitDiarization,
|
explicitDiarization,
|
||||||
@@ -186,9 +185,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
|||||||
{
|
{
|
||||||
var commandRunner = new CapturingCommandRunner("");
|
var commandRunner = new CapturingCommandRunner("");
|
||||||
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
|
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
|
||||||
var diarization = new PyannoteDiarizationOptions
|
var diarization = new PyannoteRuntimeOptions
|
||||||
{
|
{
|
||||||
Enabled = true,
|
|
||||||
DockerCommand = "docker",
|
DockerCommand = "docker",
|
||||||
BaseImage = "python:3.11-slim",
|
BaseImage = "python:3.11-slim",
|
||||||
Image = "meeting-assistant-pyannote-warmup:local",
|
Image = "meeting-assistant-pyannote-warmup:local",
|
||||||
@@ -218,9 +216,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
|||||||
var commandRunner = new CapturingCommandRunner("");
|
var commandRunner = new CapturingCommandRunner("");
|
||||||
var finalizer = CreateFinalizer(commandRunner, token: null);
|
var finalizer = CreateFinalizer(commandRunner, token: null);
|
||||||
|
|
||||||
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
|
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
|
||||||
{
|
{
|
||||||
Enabled = true,
|
|
||||||
Token = "",
|
Token = "",
|
||||||
TokenEnv = "",
|
TokenEnv = "",
|
||||||
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
|
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,596 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerSpeakerIdentificationServiceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
var result = await service.IdentifyKnownSpeakersAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||||
|
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
|
||||||
|
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||||
|
Assert.Equal(7, saved.VoiceVectors.Count);
|
||||||
|
Assert.Empty(saved.Snippets);
|
||||||
|
Assert.Single(saved.References);
|
||||||
|
Assert.Single(fixture.Encoder.Requests);
|
||||||
|
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
|
||||||
|
|
||||||
|
await service.ApplySpeakerOverrideAsync(
|
||||||
|
request,
|
||||||
|
"Guest-01",
|
||||||
|
"Sabrina",
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||||
|
Assert.Equal("Sabrina", saved.CanonicalName);
|
||||||
|
Assert.Equal(3, saved.VoiceVectors.Count);
|
||||||
|
Assert.Empty(saved.Snippets);
|
||||||
|
Assert.Single(saved.References);
|
||||||
|
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||||
|
Assert.Null(saved.CanonicalName);
|
||||||
|
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||||
|
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||||
|
Assert.Empty(saved.Snippets);
|
||||||
|
Assert.Single(saved.References);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||||
|
Assert.Equal(8, saved.VoiceVectors.Count);
|
||||||
|
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync(options =>
|
||||||
|
{
|
||||||
|
options.OutlierPruningMinimumVectors = 20;
|
||||||
|
options.OutlierPruningNeighborSimilarity = 0.90;
|
||||||
|
options.OutlierPruningMinimumNeighbors = 3;
|
||||||
|
options.OutlierPruningMinimumClusterRatio = 0.60;
|
||||||
|
});
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||||
|
.Concat(Enumerable.Range(0, 4)
|
||||||
|
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||||
|
Assert.Equal(16, saved.VoiceVectors.Count);
|
||||||
|
Assert.All(
|
||||||
|
saved.VoiceVectors,
|
||||||
|
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData(false)]
|
||||||
|
[InlineData(true)]
|
||||||
|
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
var identity = await fixture.AddIdentityAsync(
|
||||||
|
"Chris",
|
||||||
|
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||||
|
var currentRunVectors = Enumerable.Range(0, 8)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
var liveResult = await service.IdentifyKnownSpeakersAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||||
|
CancellationToken.None);
|
||||||
|
fixture.Encoder.Vectors = currentRunVectors;
|
||||||
|
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
|
||||||
|
{
|
||||||
|
KnownSpeakerMappings = liveResult.SpeakerMappings
|
||||||
|
};
|
||||||
|
if (transcriptAlreadyRelabeled)
|
||||||
|
{
|
||||||
|
finalRequest = finalRequest with { Segments = liveResult.Segments };
|
||||||
|
}
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||||
|
Assert.Equal(10, saved.VoiceVectors.Count);
|
||||||
|
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
|
||||||
|
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||||
|
fixture.Encoder.Vectors =
|
||||||
|
[
|
||||||
|
UnitVector(0),
|
||||||
|
UnitVector(0, 2, 0.01f),
|
||||||
|
UnitVector(0, 3, 0.02f),
|
||||||
|
UnitVector(0, 4, 0.03f),
|
||||||
|
UnitVector(0, 5, 0.04f)
|
||||||
|
];
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
|
||||||
|
|
||||||
|
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||||
|
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||||
|
Assert.Equal(6, saved.VoiceVectors.Count);
|
||||||
|
Assert.Single(saved.References);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
var result = await service.IdentifyFinishedSpeakersAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal("Chris", result.Segments[0].Speaker);
|
||||||
|
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
|
||||||
|
Assert.All(
|
||||||
|
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
|
||||||
|
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||||
|
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
var identity = await fixture.AddIdentityAsync(
|
||||||
|
null,
|
||||||
|
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||||
|
candidates: ["John", "Mike"]);
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
var result = await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||||
|
Assert.Equal("John", saved.CanonicalName);
|
||||||
|
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
|
||||||
|
Assert.Equal("John", result.Segments.Single().Speaker);
|
||||||
|
Assert.All(saved.References, reference =>
|
||||||
|
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
var identity = await fixture.AddIdentityAsync(
|
||||||
|
null,
|
||||||
|
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||||
|
candidates: ["John", "Mike"]);
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||||
|
Assert.Null(saved.CanonicalName);
|
||||||
|
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ProcessFinishedTranscriptAsync(
|
||||||
|
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||||
|
Assert.Equal("Manuel", saved.CanonicalName);
|
||||||
|
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync(
|
||||||
|
configureSpeaker: options => options.MaxMatchCandidates = 1);
|
||||||
|
await fixture.AddIdentityAsync("Legacy WAV identity", []);
|
||||||
|
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
var result = await service.IdentifyKnownSpeakersAsync(
|
||||||
|
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
|
||||||
|
{
|
||||||
|
await using var fixture = await Fixture.CreateAsync();
|
||||||
|
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
|
||||||
|
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
|
||||||
|
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
|
||||||
|
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||||
|
.ToList();
|
||||||
|
var service = fixture.CreateService();
|
||||||
|
|
||||||
|
await service.ApplySpeakerOverrideAsync(
|
||||||
|
request,
|
||||||
|
"Guest-01",
|
||||||
|
"Sabrina",
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
var saved = await fixture.LoadIdentityAsync(target.Id);
|
||||||
|
Assert.Single(saved.Snippets);
|
||||||
|
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||||
|
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||||
|
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
|
||||||
|
Assert.Single(await context.SpeakerIdentities.ToListAsync());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] UnitVector(
|
||||||
|
int primaryDimension,
|
||||||
|
int? secondaryDimension = null,
|
||||||
|
float secondaryValue = 0)
|
||||||
|
{
|
||||||
|
var vector = new float[256];
|
||||||
|
vector[primaryDimension] = 1;
|
||||||
|
if (secondaryDimension is { } dimension)
|
||||||
|
{
|
||||||
|
vector[dimension] = secondaryValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] ToBytes(float[] vector)
|
||||||
|
{
|
||||||
|
var bytes = new byte[vector.Length * sizeof(float)];
|
||||||
|
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class Fixture : IAsyncDisposable
|
||||||
|
{
|
||||||
|
private readonly string directory;
|
||||||
|
private readonly string databasePath;
|
||||||
|
private readonly SpeakerIdentificationOptions speakerOptions;
|
||||||
|
|
||||||
|
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
|
||||||
|
{
|
||||||
|
this.directory = directory;
|
||||||
|
this.databasePath = databasePath;
|
||||||
|
this.speakerOptions = speakerOptions;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FakeEncoder Encoder { get; } = new();
|
||||||
|
|
||||||
|
public FakeSnippetExtractor SnippetExtractor { get; } = new();
|
||||||
|
|
||||||
|
public string DatabasePath => databasePath;
|
||||||
|
|
||||||
|
public static async Task<Fixture> CreateAsync(
|
||||||
|
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
|
||||||
|
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
var databasePath = Path.Combine(directory, "speaker-identities.db");
|
||||||
|
var speakerOptions = new SpeakerIdentificationOptions
|
||||||
|
{
|
||||||
|
DatabasePath = databasePath,
|
||||||
|
MatchBatchSize = 6,
|
||||||
|
MaxMatchCandidates = 100,
|
||||||
|
MatchIdentityActiveAge = TimeSpan.FromDays(365),
|
||||||
|
MinimumSampleSpeechDuration = TimeSpan.Zero,
|
||||||
|
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
RequiredVectorsPerSpeaker = 5,
|
||||||
|
MaxVectorsPerIdentity = 1000,
|
||||||
|
MinimumClusterCohesion = 0.75,
|
||||||
|
MinimumIdentitySimilarity = 0.75,
|
||||||
|
MinimumSimilarityMargin = 0.05,
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
configure?.Invoke(speakerOptions.Resemblyzer);
|
||||||
|
configureSpeaker?.Invoke(speakerOptions);
|
||||||
|
await using var context = new SpeakerIdentityDbContext(
|
||||||
|
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||||
|
.Options);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||||
|
return new Fixture(directory, databasePath, speakerOptions);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResemblyzerSpeakerIdentificationService CreateService()
|
||||||
|
{
|
||||||
|
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
|
||||||
|
return new ResemblyzerSpeakerIdentificationService(
|
||||||
|
new TestDbContextFactory(databasePath),
|
||||||
|
SnippetExtractor,
|
||||||
|
Encoder,
|
||||||
|
new ResemblyzerVoiceClusterMatcher(
|
||||||
|
speakerOptions.Resemblyzer,
|
||||||
|
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||||
|
new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
speakerOptions.Resemblyzer,
|
||||||
|
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||||
|
Options.Create(appOptions),
|
||||||
|
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SpeakerIdentificationRequest CreateRequest(
|
||||||
|
string speaker,
|
||||||
|
int sampleCount,
|
||||||
|
IReadOnlyList<string>? attendees = null,
|
||||||
|
int segmentCount = 1)
|
||||||
|
{
|
||||||
|
var transcriptPath = Path.Combine(directory, "transcript.md");
|
||||||
|
File.WriteAllText(transcriptPath, "Transcript");
|
||||||
|
var segments = Enumerable.Range(0, segmentCount)
|
||||||
|
.Select(index => new TranscriptionSegment(
|
||||||
|
TimeSpan.FromSeconds(index * 30),
|
||||||
|
TimeSpan.FromSeconds((index + 1) * 30),
|
||||||
|
speaker,
|
||||||
|
"enough useful words for a speaker sample"))
|
||||||
|
.ToList();
|
||||||
|
var segment = segments[0];
|
||||||
|
return new SpeakerIdentificationRequest(
|
||||||
|
Path.Combine(directory, "meeting.wav"),
|
||||||
|
new MeetingNote(
|
||||||
|
Path.Combine(directory, "meeting.md"),
|
||||||
|
new MeetingNoteFrontmatter
|
||||||
|
{
|
||||||
|
Title = "Test",
|
||||||
|
Attendees = attendees?.ToList() ?? ["Chris"],
|
||||||
|
Transcript = transcriptPath,
|
||||||
|
AssistantContext = Path.Combine(directory, "context.md"),
|
||||||
|
Summary = Path.Combine(directory, "summary.md")
|
||||||
|
},
|
||||||
|
""),
|
||||||
|
segments,
|
||||||
|
Enumerable.Range(0, sampleCount)
|
||||||
|
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
|
||||||
|
.ToList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<SpeakerIdentity> AddIdentityAsync(
|
||||||
|
string? name,
|
||||||
|
IReadOnlyList<float[]> vectors,
|
||||||
|
IReadOnlyList<string>? candidates = null,
|
||||||
|
IReadOnlyList<string>? aliases = null)
|
||||||
|
{
|
||||||
|
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var identity = new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = name,
|
||||||
|
CreatedAt = now,
|
||||||
|
UpdatedAt = now,
|
||||||
|
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
|
||||||
|
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
|
||||||
|
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = speakerOptions.Resemblyzer.ModelId,
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = ToBytes(vector),
|
||||||
|
Fingerprint = $"known-{index}",
|
||||||
|
CreatedAt = now.AddMinutes(index)
|
||||||
|
}).ToList()
|
||||||
|
};
|
||||||
|
context.SpeakerIdentities.Add(identity);
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
return identity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task AddCurrentRunCandidateAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
string candidateName)
|
||||||
|
{
|
||||||
|
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CreatedAt = now,
|
||||||
|
UpdatedAt = now,
|
||||||
|
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
|
||||||
|
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
|
||||||
|
VoiceVectors =
|
||||||
|
[
|
||||||
|
new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "older-model",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = ToBytes(UnitVector(1)),
|
||||||
|
Fingerprint = "older-model-vector",
|
||||||
|
CreatedAt = now
|
||||||
|
}
|
||||||
|
],
|
||||||
|
References =
|
||||||
|
[
|
||||||
|
SpeakerIdentityReferences.Create(
|
||||||
|
request.MeetingNote.Path,
|
||||||
|
request.MeetingNote.Frontmatter.Transcript,
|
||||||
|
now)
|
||||||
|
]
|
||||||
|
});
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
|
||||||
|
{
|
||||||
|
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||||
|
return await context.SpeakerIdentities
|
||||||
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
|
.Include(identity => identity.CandidateNames)
|
||||||
|
.Include(identity => identity.Aliases)
|
||||||
|
.Include(identity => identity.References)
|
||||||
|
.SingleAsync(identity => identity.Id == id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
|
||||||
|
{
|
||||||
|
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||||
|
return await context.SpeakerIdentities
|
||||||
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
|
.Include(identity => identity.CandidateNames)
|
||||||
|
.Include(identity => identity.References)
|
||||||
|
.SingleAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(directory))
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
|
||||||
|
{
|
||||||
|
public IReadOnlyList<float[]> Vectors { get; set; } = [];
|
||||||
|
|
||||||
|
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||||
|
IReadOnlyList<byte[]> wavSamples,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
|
||||||
|
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
|
||||||
|
{
|
||||||
|
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
|
||||||
|
|
||||||
|
public Task<byte[]> ExtractSnippetAsync(
|
||||||
|
string audioPath,
|
||||||
|
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Requests.Add(speakerSegments.ToList());
|
||||||
|
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||||
|
{
|
||||||
|
private readonly string databasePath;
|
||||||
|
|
||||||
|
public TestDbContextFactory(string databasePath)
|
||||||
|
{
|
||||||
|
this.databasePath = databasePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SpeakerIdentityDbContext CreateDbContext()
|
||||||
|
{
|
||||||
|
return new SpeakerIdentityDbContext(
|
||||||
|
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||||
|
.Options);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(directory, "identities.db");
|
||||||
|
var factory = new TestDbContextFactory(databasePath);
|
||||||
|
await using (var context = factory.CreateDbContext())
|
||||||
|
{
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||||
|
context.SpeakerIdentities.Add(CreateIdentity(
|
||||||
|
"Chris",
|
||||||
|
DateTimeOffset.UtcNow.AddMonths(-2),
|
||||||
|
Enumerable.Range(0, 10)
|
||||||
|
.Select(index => UnitVector(0, index + 20, 0.02f))
|
||||||
|
.ToList(),
|
||||||
|
directory));
|
||||||
|
context.SpeakerIdentities.Add(CreateIdentity(
|
||||||
|
"Chris duplicate",
|
||||||
|
DateTimeOffset.UtcNow.AddDays(-1),
|
||||||
|
Enumerable.Range(0, 10)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||||
|
.Concat(Enumerable.Range(0, 4)
|
||||||
|
.Select(index => UnitVector(1, index + 40, 0.02f)))
|
||||||
|
.ToList(),
|
||||||
|
directory));
|
||||||
|
context.SpeakerIdentities.Add(CreateIdentity(
|
||||||
|
"Decoy",
|
||||||
|
DateTimeOffset.UtcNow.AddMonths(-1),
|
||||||
|
[UnitVector(0)],
|
||||||
|
directory));
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
RequiredVectorsPerSpeaker = 5,
|
||||||
|
MaxVectorsPerIdentity = 1000,
|
||||||
|
MinimumClusterCohesion = 0.75,
|
||||||
|
MinimumIdentitySimilarity = 0.75,
|
||||||
|
MinimumSimilarityMargin = 0.05,
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||||
|
};
|
||||||
|
var service = new ResemblyzerSpeakerIdentityMergeService(
|
||||||
|
factory,
|
||||||
|
new ResemblyzerVoiceClusterMatcher(
|
||||||
|
resemblyzerOptions,
|
||||||
|
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||||
|
new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
resemblyzerOptions,
|
||||||
|
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||||
|
Options.Create(new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||||
|
{
|
||||||
|
MergeRecentIdentityAge = TimeSpan.FromDays(14),
|
||||||
|
MaxMatchCandidates = 1,
|
||||||
|
MaxSnippetsPerSpeaker = 3,
|
||||||
|
Resemblyzer = resemblyzerOptions
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
|
||||||
|
|
||||||
|
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(2, result.MatchAttempts);
|
||||||
|
Assert.Equal(1, result.MergedPairs);
|
||||||
|
await using var verification = factory.CreateDbContext();
|
||||||
|
var saved = await verification.SpeakerIdentities
|
||||||
|
.Include(identity => identity.Aliases)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
|
.SingleAsync(identity => identity.CanonicalName == "Chris");
|
||||||
|
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
|
||||||
|
Assert.Equal("Chris", saved.CanonicalName);
|
||||||
|
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
|
||||||
|
Assert.Equal(20, saved.VoiceVectors.Count);
|
||||||
|
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerIdentity CreateIdentity(
|
||||||
|
string name,
|
||||||
|
DateTimeOffset createdAt,
|
||||||
|
IReadOnlyList<float[]> vectors,
|
||||||
|
string directory)
|
||||||
|
{
|
||||||
|
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
|
||||||
|
File.WriteAllText(transcriptPath, "Transcript");
|
||||||
|
return new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = name,
|
||||||
|
CreatedAt = createdAt,
|
||||||
|
UpdatedAt = createdAt,
|
||||||
|
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = ToBytes(vector),
|
||||||
|
Fingerprint = $"{name}-{index}",
|
||||||
|
CreatedAt = createdAt.AddMinutes(index)
|
||||||
|
}).ToList(),
|
||||||
|
References =
|
||||||
|
[
|
||||||
|
new SpeakerIdentityReference
|
||||||
|
{
|
||||||
|
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
|
||||||
|
TranscriptPath = transcriptPath,
|
||||||
|
CreatedAt = createdAt
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
|
||||||
|
{
|
||||||
|
var vector = new float[256];
|
||||||
|
vector[primary] = 1;
|
||||||
|
if (secondary is { } index)
|
||||||
|
{
|
||||||
|
vector[index] = secondaryValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] ToBytes(float[] vector)
|
||||||
|
{
|
||||||
|
var bytes = new byte[vector.Length * sizeof(float)];
|
||||||
|
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] ToVector(byte[] bytes)
|
||||||
|
{
|
||||||
|
var vector = new float[bytes.Length / sizeof(float)];
|
||||||
|
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||||
|
{
|
||||||
|
private readonly string databasePath;
|
||||||
|
|
||||||
|
public TestDbContextFactory(string databasePath)
|
||||||
|
{
|
||||||
|
this.databasePath = databasePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SpeakerIdentityDbContext CreateDbContext()
|
||||||
|
{
|
||||||
|
return new SpeakerIdentityDbContext(
|
||||||
|
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||||
|
.Options);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerVoiceClusterMatcherTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
|
||||||
|
{
|
||||||
|
var matcher = CreateMatcher();
|
||||||
|
var query = Enumerable.Range(0, 5)
|
||||||
|
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
|
||||||
|
.ToList();
|
||||||
|
var candidates = new[]
|
||||||
|
{
|
||||||
|
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
|
||||||
|
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
|
||||||
|
};
|
||||||
|
|
||||||
|
var result = matcher.Match(query, candidates);
|
||||||
|
|
||||||
|
Assert.True(result.Accepted);
|
||||||
|
Assert.Equal(42, result.IdentityId);
|
||||||
|
Assert.True(result.Cohesion >= 0.99);
|
||||||
|
Assert.True(result.BestSimilarity >= 0.99);
|
||||||
|
Assert.True(result.RunnerUpSimilarity < 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void FourVectorsDoNotTriggerAutomaticMatching()
|
||||||
|
{
|
||||||
|
var result = CreateMatcher().Match(
|
||||||
|
Enumerable.Repeat(UnitVector(0), 4).ToList(),
|
||||||
|
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||||
|
|
||||||
|
Assert.False(result.Accepted);
|
||||||
|
Assert.Contains("4/5", result.Reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
|
||||||
|
{
|
||||||
|
var result = CreateMatcher().Match(
|
||||||
|
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
|
||||||
|
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||||
|
|
||||||
|
Assert.False(result.Accepted);
|
||||||
|
Assert.Contains("cohesion", result.Reason);
|
||||||
|
Assert.Null(result.BestSimilarity);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SimilarityBelowThresholdIsRejected()
|
||||||
|
{
|
||||||
|
var result = CreateMatcher().Match(
|
||||||
|
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||||
|
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
|
||||||
|
|
||||||
|
Assert.False(result.Accepted);
|
||||||
|
Assert.Contains("best similarity", result.Reason);
|
||||||
|
Assert.Equal(0, result.BestSimilarity);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SimilarCandidatesWithinMarginAreRejected()
|
||||||
|
{
|
||||||
|
var result = CreateMatcher().Match(
|
||||||
|
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||||
|
[
|
||||||
|
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
|
||||||
|
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
|
||||||
|
]);
|
||||||
|
|
||||||
|
Assert.False(result.Accepted);
|
||||||
|
Assert.Contains("margin", result.Reason);
|
||||||
|
Assert.NotNull(result.RunnerUpSimilarity);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
|
||||||
|
{
|
||||||
|
var result = CreateMatcher().Match(
|
||||||
|
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||||
|
[
|
||||||
|
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
|
||||||
|
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
|
||||||
|
]);
|
||||||
|
|
||||||
|
Assert.True(result.Accepted);
|
||||||
|
Assert.Equal(42, result.IdentityId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
|
||||||
|
{
|
||||||
|
return new ResemblyzerVoiceClusterMatcher(
|
||||||
|
new ResemblyzerSpeakerRecognitionOptions
|
||||||
|
{
|
||||||
|
RequiredVectorsPerSpeaker = 5,
|
||||||
|
MinimumClusterCohesion = 0.75,
|
||||||
|
MinimumIdentitySimilarity = 0.75,
|
||||||
|
MinimumSimilarityMargin = 0.05
|
||||||
|
},
|
||||||
|
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] UnitVector(
|
||||||
|
int primaryDimension,
|
||||||
|
int? secondaryDimension = null,
|
||||||
|
float secondaryValue = 0)
|
||||||
|
{
|
||||||
|
var vector = new float[256];
|
||||||
|
vector[primaryDimension] = 1;
|
||||||
|
if (secondaryDimension is { } dimension)
|
||||||
|
{
|
||||||
|
vector[dimension] = secondaryValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
|
||||||
|
{
|
||||||
|
var identity = IdentityWithVectors(
|
||||||
|
Enumerable.Range(0, 16)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||||
|
.Concat(Enumerable.Range(0, 4)
|
||||||
|
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||||
|
.ToList());
|
||||||
|
var pruner = CreatePruner();
|
||||||
|
|
||||||
|
var result = pruner.Prune(identity);
|
||||||
|
|
||||||
|
Assert.Equal(4, result);
|
||||||
|
Assert.Equal(16, identity.VoiceVectors.Count);
|
||||||
|
Assert.All(
|
||||||
|
identity.VoiceVectors,
|
||||||
|
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void AmbiguousDenseClustersPreserveAllVectors()
|
||||||
|
{
|
||||||
|
var identity = IdentityWithVectors(
|
||||||
|
Enumerable.Range(0, 10)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||||
|
.Concat(Enumerable.Range(0, 10)
|
||||||
|
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||||
|
.ToList());
|
||||||
|
var pruner = CreatePruner();
|
||||||
|
|
||||||
|
var result = pruner.Prune(identity);
|
||||||
|
|
||||||
|
Assert.Equal(0, result);
|
||||||
|
Assert.Equal(20, identity.VoiceVectors.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
|
||||||
|
{
|
||||||
|
var identity = IdentityWithVectors(
|
||||||
|
Enumerable.Range(0, 15)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||||
|
.Concat(Enumerable.Range(0, 4)
|
||||||
|
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||||
|
.ToList());
|
||||||
|
var pruner = CreatePruner();
|
||||||
|
|
||||||
|
var result = pruner.Prune(identity);
|
||||||
|
|
||||||
|
Assert.Equal(0, result);
|
||||||
|
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void PruningPreservesOtherModelsAndMalformedStoredRows()
|
||||||
|
{
|
||||||
|
var identity = IdentityWithVectors(
|
||||||
|
Enumerable.Range(0, 16)
|
||||||
|
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||||
|
.Concat(Enumerable.Range(0, 4)
|
||||||
|
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||||
|
.ToList());
|
||||||
|
SpeakerVoiceVectors.AddDistinct(
|
||||||
|
identity,
|
||||||
|
[UnitVector(2, 60, 0.04f)],
|
||||||
|
"older-model",
|
||||||
|
1000,
|
||||||
|
DateTimeOffset.UtcNow);
|
||||||
|
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = [1],
|
||||||
|
Fingerprint = "malformed",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = new byte[256 * sizeof(float)],
|
||||||
|
Fingerprint = "zero-magnitude",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
var pruner = CreatePruner();
|
||||||
|
|
||||||
|
var result = pruner.Prune(identity);
|
||||||
|
|
||||||
|
Assert.Equal(4, result);
|
||||||
|
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||||
|
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||||
|
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
|
||||||
|
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
|
||||||
|
{
|
||||||
|
return new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
new ResemblyzerSpeakerRecognitionOptions
|
||||||
|
{
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
OutlierPruningMinimumVectors = 20,
|
||||||
|
OutlierPruningNeighborSimilarity = 0.90,
|
||||||
|
OutlierPruningMinimumNeighbors = 3,
|
||||||
|
OutlierPruningMinimumClusterRatio = 0.60
|
||||||
|
},
|
||||||
|
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
|
||||||
|
{
|
||||||
|
var identity = new SpeakerIdentity();
|
||||||
|
SpeakerVoiceVectors.AddDistinct(
|
||||||
|
identity,
|
||||||
|
vectors,
|
||||||
|
"resemblyzer-0.1.4-pretrained",
|
||||||
|
1000,
|
||||||
|
DateTimeOffset.UtcNow);
|
||||||
|
return identity;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] UnitVector(
|
||||||
|
int primaryDimension,
|
||||||
|
int secondaryDimension,
|
||||||
|
float secondaryValue)
|
||||||
|
{
|
||||||
|
var vector = new float[256];
|
||||||
|
vector[primaryDimension] = 1;
|
||||||
|
vector[secondaryDimension] = secondaryValue;
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerWarmupHostedServiceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
|
||||||
|
{
|
||||||
|
var encoder = new BlockingEncoder();
|
||||||
|
var options = new MeetingAssistantOptions();
|
||||||
|
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||||
|
var service = new ResemblyzerWarmupHostedService(
|
||||||
|
encoder,
|
||||||
|
Options.Create(options),
|
||||||
|
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||||
|
await encoder.WaitForWarmupAsync();
|
||||||
|
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.True(encoder.CancellationObserved);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DisabledWarmupDoesNotInvokeEncoder()
|
||||||
|
{
|
||||||
|
var encoder = new BlockingEncoder();
|
||||||
|
var service = new ResemblyzerWarmupHostedService(
|
||||||
|
encoder,
|
||||||
|
Options.Create(new MeetingAssistantOptions()),
|
||||||
|
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||||
|
|
||||||
|
await service.StartAsync(CancellationToken.None);
|
||||||
|
await service.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.False(encoder.WarmupStarted);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
|
||||||
|
{
|
||||||
|
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
public bool WarmupStarted { get; private set; }
|
||||||
|
|
||||||
|
public bool CancellationObserved { get; private set; }
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||||
|
IReadOnlyList<byte[]> wavSamples,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
WarmupStarted = true;
|
||||||
|
started.TrySetResult();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
CancellationObserved = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WaitForWarmupAsync()
|
||||||
|
{
|
||||||
|
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,13 +8,22 @@ namespace MeetingAssistant.Tests;
|
|||||||
public sealed class SpeakerAudioSampleCollectorTests
|
public sealed class SpeakerAudioSampleCollectorTests
|
||||||
{
|
{
|
||||||
[Fact]
|
[Fact]
|
||||||
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
|
public void DefaultSampleDurationsAreTenAndSixtySeconds()
|
||||||
|
{
|
||||||
|
var options = new SpeakerIdentificationOptions();
|
||||||
|
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
|
||||||
{
|
{
|
||||||
var collector = new SpeakerAudioSampleCollector(
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
TimeSpan.FromMinutes(2),
|
TimeSpan.FromMinutes(2),
|
||||||
maxSamplesPerSpeaker: 3,
|
maxSamplesPerSpeaker: 3,
|
||||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||||
|
|
||||||
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
|
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
|
||||||
@@ -37,8 +46,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
|||||||
var collector = new SpeakerAudioSampleCollector(
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
TimeSpan.FromMinutes(2),
|
TimeSpan.FromMinutes(2),
|
||||||
maxSamplesPerSpeaker: 3,
|
maxSamplesPerSpeaker: 3,
|
||||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
|
||||||
|
|
||||||
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
|
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
|
||||||
@@ -48,6 +57,61 @@ public sealed class SpeakerAudioSampleCollectorTests
|
|||||||
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
|
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
|
||||||
|
{
|
||||||
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
|
TimeSpan.FromMinutes(2),
|
||||||
|
maxSamplesPerSpeaker: 3,
|
||||||
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||||
|
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||||
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
|
||||||
|
|
||||||
|
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||||
|
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
|
||||||
|
|
||||||
|
var sample = Assert.Single(collector.Snapshot());
|
||||||
|
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
|
||||||
|
{
|
||||||
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
|
TimeSpan.FromMinutes(2),
|
||||||
|
maxSamplesPerSpeaker: 3,
|
||||||
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||||
|
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||||
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
|
||||||
|
|
||||||
|
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||||
|
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
|
||||||
|
|
||||||
|
Assert.Empty(collector.Snapshot());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
|
||||||
|
{
|
||||||
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
|
TimeSpan.FromMinutes(2),
|
||||||
|
maxSamplesPerSpeaker: 3,
|
||||||
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||||
|
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||||
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
|
||||||
|
|
||||||
|
var sample = collector.TryAdd(Segment(
|
||||||
|
0,
|
||||||
|
70,
|
||||||
|
"Guest01",
|
||||||
|
"one two three four five six seven eight nine ten."));
|
||||||
|
|
||||||
|
Assert.NotNull(sample);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
|
||||||
|
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
|
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
|
||||||
{
|
{
|
||||||
@@ -55,8 +119,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
|||||||
var collector = new SpeakerAudioSampleCollector(
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
TimeSpan.FromMinutes(2),
|
TimeSpan.FromMinutes(2),
|
||||||
maxSamplesPerSpeaker: 3,
|
maxSamplesPerSpeaker: 3,
|
||||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||||
maximumSegmentGap: TimeSpan.FromSeconds(1),
|
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||||
logger: logger);
|
logger: logger);
|
||||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||||
|
|
||||||
@@ -69,6 +133,47 @@ public sealed class SpeakerAudioSampleCollectorTests
|
|||||||
Assert.Contains("minimum duration", message);
|
Assert.Contains("minimum duration", message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
|
||||||
|
{
|
||||||
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
|
TimeSpan.FromMinutes(2),
|
||||||
|
maxSamplesPerSpeaker: 5,
|
||||||
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||||
|
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||||
|
requireNonOverlappingSamples: true);
|
||||||
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
|
||||||
|
|
||||||
|
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||||
|
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
|
||||||
|
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
|
||||||
|
|
||||||
|
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
|
||||||
|
|
||||||
|
Assert.Collection(
|
||||||
|
samples,
|
||||||
|
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
|
||||||
|
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
|
||||||
|
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
|
||||||
|
{
|
||||||
|
var collector = new SpeakerAudioSampleCollector(
|
||||||
|
TimeSpan.FromMinutes(2),
|
||||||
|
maxSamplesPerSpeaker: 5,
|
||||||
|
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||||
|
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||||
|
requireNonOverlappingSamples: true);
|
||||||
|
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
|
||||||
|
|
||||||
|
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||||
|
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
|
||||||
|
|
||||||
|
Assert.Single(collector.Snapshot());
|
||||||
|
}
|
||||||
|
|
||||||
private static TranscriptionSegment Segment(
|
private static TranscriptionSegment Segment(
|
||||||
double start,
|
double start,
|
||||||
double end,
|
double end,
|
||||||
|
|||||||
@@ -107,6 +107,9 @@ public sealed class SpeakerIdentityMergeServiceTests
|
|||||||
return new SpeakerIdentityMergeService(
|
return new SpeakerIdentityMergeService(
|
||||||
new TestSpeakerIdentityDbContextFactory(dbPath),
|
new TestSpeakerIdentityDbContextFactory(dbPath),
|
||||||
Matcher,
|
Matcher,
|
||||||
|
new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
new ResemblyzerSpeakerRecognitionOptions(),
|
||||||
|
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||||
Options.Create(new MeetingAssistantOptions
|
Options.Create(new MeetingAssistantOptions
|
||||||
{
|
{
|
||||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class SpeakerIdentityMergerTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
|
||||||
|
{
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var target = new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = "Chris",
|
||||||
|
VoiceVectors =
|
||||||
|
[
|
||||||
|
Vector("oldest", 1, now.AddMinutes(-4)),
|
||||||
|
Vector("duplicate", 2, now.AddMinutes(-3))
|
||||||
|
]
|
||||||
|
};
|
||||||
|
var source = new SpeakerIdentity
|
||||||
|
{
|
||||||
|
VoiceVectors =
|
||||||
|
[
|
||||||
|
Vector("duplicate", 2, now.AddMinutes(-2)),
|
||||||
|
Vector("newer", 3, now.AddMinutes(-1)),
|
||||||
|
Vector("newest", 4, now)
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
|
||||||
|
|
||||||
|
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
|
||||||
|
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
|
||||||
|
{
|
||||||
|
return new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "model",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
|
||||||
|
Fingerprint = fingerprint,
|
||||||
|
CreatedAt = createdAt
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Mvc.Testing;
|
||||||
|
using Microsoft.AspNetCore.TestHost;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class SpeakerRecognitionFeatureSelectionTests
|
||||||
|
{
|
||||||
|
[Theory]
|
||||||
|
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
|
||||||
|
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
|
||||||
|
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
|
||||||
|
bool enabled,
|
||||||
|
Type expectedIdentificationType,
|
||||||
|
Type expectedMergeType,
|
||||||
|
int expectedMinimumSamples,
|
||||||
|
bool expectedNonOverlappingSamples)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var factory = new WebApplicationFactory<Program>()
|
||||||
|
.WithWebHostBuilder(builder =>
|
||||||
|
{
|
||||||
|
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||||
|
{
|
||||||
|
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
|
||||||
|
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
|
||||||
|
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
|
||||||
|
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
|
||||||
|
});
|
||||||
|
});
|
||||||
|
builder.ConfigureTestServices(services =>
|
||||||
|
{
|
||||||
|
services.RemoveAll<IResemblyzerVoiceEncoder>();
|
||||||
|
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
|
||||||
|
|
||||||
|
Assert.IsType(expectedIdentificationType, service);
|
||||||
|
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
|
||||||
|
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
|
||||||
|
Assert.IsType(expectedMergeType, mergeService);
|
||||||
|
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
|
||||||
|
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
|
||||||
|
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
|
||||||
|
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
SqliteConnection.ClearAllPools();
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
|
||||||
|
{
|
||||||
|
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||||
|
IReadOnlyList<byte[]> wavSamples,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class SpeakerSampleSpanSelectorTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
|
||||||
|
{
|
||||||
|
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||||
|
[
|
||||||
|
Segment(0, 4),
|
||||||
|
Segment(7, 13),
|
||||||
|
Segment(20, 70)
|
||||||
|
],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
Assert.Equal(3, span.Count);
|
||||||
|
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
|
||||||
|
{
|
||||||
|
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||||
|
[
|
||||||
|
Segment(0, 4),
|
||||||
|
Segment(20, 25)
|
||||||
|
],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60));
|
||||||
|
|
||||||
|
Assert.Empty(span);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
|
||||||
|
{
|
||||||
|
var segments = Enumerable.Range(0, 6)
|
||||||
|
.Select(index => new TranscriptionSegment(
|
||||||
|
TimeSpan.FromSeconds(index * 10),
|
||||||
|
TimeSpan.FromSeconds((index + 1) * 10),
|
||||||
|
"Guest01",
|
||||||
|
$"segment {index} has enough words"))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
segments,
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60),
|
||||||
|
maxSpans: 5);
|
||||||
|
|
||||||
|
Assert.Equal(5, spans.Count);
|
||||||
|
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
|
||||||
|
Assert.All(
|
||||||
|
spans.Zip(spans.Skip(1)),
|
||||||
|
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
|
||||||
|
{
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
[
|
||||||
|
Segment(0, 10),
|
||||||
|
Segment(9, 19),
|
||||||
|
Segment(20, 30)
|
||||||
|
],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60),
|
||||||
|
maxSpans: 5);
|
||||||
|
|
||||||
|
Assert.Equal(2, spans.Count);
|
||||||
|
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
|
||||||
|
{
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
[
|
||||||
|
Segment(0, 4),
|
||||||
|
Segment(7, 13)
|
||||||
|
],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60),
|
||||||
|
maxSpans: 5);
|
||||||
|
|
||||||
|
var span = Assert.Single(spans);
|
||||||
|
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
|
||||||
|
{
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
[
|
||||||
|
Segment(0, 4),
|
||||||
|
Segment(20, 25)
|
||||||
|
],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60),
|
||||||
|
maxSpans: 5);
|
||||||
|
|
||||||
|
Assert.Empty(spans);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
|
||||||
|
{
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
[Segment(0, 70)],
|
||||||
|
"Guest01",
|
||||||
|
TimeSpan.FromSeconds(10),
|
||||||
|
TimeSpan.FromSeconds(60),
|
||||||
|
maxSpans: 5);
|
||||||
|
|
||||||
|
var span = Assert.Single(spans);
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TranscriptionSegment Segment(double start, double end)
|
||||||
|
{
|
||||||
|
return new TranscriptionSegment(
|
||||||
|
TimeSpan.FromSeconds(start),
|
||||||
|
TimeSpan.FromSeconds(end),
|
||||||
|
"Guest01",
|
||||||
|
"enough words for this segment");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class SpeakerVoiceVectorPersistenceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"),
|
||||||
|
"speaker-identities.db");
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||||
|
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath}")
|
||||||
|
.Options;
|
||||||
|
|
||||||
|
await using (var context = new SpeakerIdentityDbContext(options))
|
||||||
|
{
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||||
|
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = "Chris",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow,
|
||||||
|
UpdatedAt = DateTimeOffset.UtcNow,
|
||||||
|
VoiceVectors =
|
||||||
|
[
|
||||||
|
new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = new byte[256 * sizeof(float)],
|
||||||
|
Fingerprint = "vector-a",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (var context = new SpeakerIdentityDbContext(options))
|
||||||
|
{
|
||||||
|
var identity = await context.SpeakerIdentities
|
||||||
|
.Include(candidate => candidate.VoiceVectors)
|
||||||
|
.SingleAsync();
|
||||||
|
var vector = Assert.Single(identity.VoiceVectors);
|
||||||
|
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
|
||||||
|
Assert.Equal(256, vector.Dimensions);
|
||||||
|
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
|
||||||
|
|
||||||
|
context.SpeakerIdentities.Remove(identity);
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (var context = new SpeakerIdentityDbContext(options))
|
||||||
|
{
|
||||||
|
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"),
|
||||||
|
"speaker-identities.db");
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||||
|
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath}")
|
||||||
|
.Options;
|
||||||
|
await using var context = new SpeakerIdentityDbContext(options);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||||
|
var identity = new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = "Chris",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow,
|
||||||
|
UpdatedAt = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
context.SpeakerIdentities.Add(identity);
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
|
||||||
|
context.SpeakerVoiceVectors.AddRange(
|
||||||
|
CreateVector(identity.Id, "same-vector"),
|
||||||
|
CreateVector(identity.Id, "same-vector"));
|
||||||
|
|
||||||
|
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
|
||||||
|
{
|
||||||
|
return new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
SpeakerIdentityId = identityId,
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = new byte[256 * sizeof(float)],
|
||||||
|
Fingerprint = fingerprint,
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
using System.Threading.Channels;
|
||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Notifications;
|
||||||
|
using MeetingAssistant.Summary;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using MeetingAssistant.Workflow;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class SummaryAgentRequestTests
|
||||||
|
{
|
||||||
|
[Theory]
|
||||||
|
[InlineData("denied")]
|
||||||
|
[InlineData("expired")]
|
||||||
|
[InlineData("shutdown")]
|
||||||
|
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
|
||||||
|
{
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var editor = new RecordingEditor();
|
||||||
|
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||||
|
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||||
|
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||||
|
await requests.StartAsync(CancellationToken.None);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||||
|
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
|
||||||
|
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
prompt.Response.TrySetResult(true);
|
||||||
|
}
|
||||||
|
finally { await requests.StopAsync(CancellationToken.None); }
|
||||||
|
Assert.Empty(editor.Prompts);
|
||||||
|
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task InvalidRequestsAreRefusedBeforeNotification()
|
||||||
|
{
|
||||||
|
var fixture = await CreateAsync();
|
||||||
|
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||||
|
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
|
||||||
|
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
|
||||||
|
|
||||||
|
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
|
||||||
|
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
|
||||||
|
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
|
||||||
|
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
|
||||||
|
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
|
||||||
|
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
|
||||||
|
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
|
||||||
|
Assert.False(prompts.HasMore);
|
||||||
|
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
|
||||||
|
{
|
||||||
|
var fixture = await CreateAsync();
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var editor = new RecordingEditor();
|
||||||
|
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||||
|
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||||
|
await requests.StartAsync(CancellationToken.None);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
|
||||||
|
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
|
||||||
|
|
||||||
|
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
|
||||||
|
|
||||||
|
Assert.StartsWith("Requested", result);
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
|
||||||
|
Assert.Empty(editor.Prompts);
|
||||||
|
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
|
||||||
|
Assert.True(tools.SummaryWasWritten);
|
||||||
|
|
||||||
|
prompt.Response.SetResult(true);
|
||||||
|
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||||
|
Assert.False(prompt.Response.TrySetResult(true));
|
||||||
|
Assert.Single(editor.Prompts);
|
||||||
|
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
}
|
||||||
|
finally { await requests.StopAsync(CancellationToken.None); }
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("denied")]
|
||||||
|
[InlineData("expired")]
|
||||||
|
[InlineData("canceled")]
|
||||||
|
[InlineData("removed")]
|
||||||
|
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
|
||||||
|
{
|
||||||
|
var fixture = await CreateAsync();
|
||||||
|
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||||
|
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||||
|
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||||
|
using var cancellation = new CancellationTokenSource();
|
||||||
|
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
|
||||||
|
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||||
|
if (outcome == "canceled") await cancellation.CancelAsync();
|
||||||
|
if (outcome == "removed")
|
||||||
|
{
|
||||||
|
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
|
||||||
|
prompt.Response.SetResult(true);
|
||||||
|
}
|
||||||
|
if (outcome == "canceled")
|
||||||
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
|
||||||
|
else
|
||||||
|
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||||
|
|
||||||
|
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
prompt.Response.TrySetResult(true);
|
||||||
|
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||||
|
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
|
||||||
|
{
|
||||||
|
var fixture = await CreateAsync();
|
||||||
|
var prompts = new ControlledPrompts();
|
||||||
|
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||||
|
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||||
|
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
|
||||||
|
var prompt = await prompts.NextAsync();
|
||||||
|
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
|
||||||
|
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
|
||||||
|
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
|
||||||
|
Assert.False(request.IsCompleted);
|
||||||
|
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||||
|
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
|
||||||
|
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
|
||||||
|
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
|
||||||
|
|
||||||
|
prompt.Response.SetResult(true);
|
||||||
|
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
Assert.StartsWith("Approved", result);
|
||||||
|
Assert.Contains("Alpha instructions", result);
|
||||||
|
Assert.Contains("Alpha Platform", result);
|
||||||
|
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||||
|
Assert.Contains("Edited by user", note);
|
||||||
|
Assert.Contains("custom: keep-me", note);
|
||||||
|
Assert.Contains("User's latest notes.", note);
|
||||||
|
Assert.Contains("Existing", note);
|
||||||
|
Assert.Contains("Alpha", note);
|
||||||
|
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
|
||||||
|
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||||
|
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
|
||||||
|
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
|
||||||
|
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
|
||||||
|
Assert.False(prompts.HasMore);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||||
|
var project = Path.Combine(root, "Projects", "Alpha");
|
||||||
|
Directory.CreateDirectory(project);
|
||||||
|
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
|
||||||
|
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
|
||||||
|
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
|
||||||
|
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
|
||||||
|
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
|
||||||
|
{
|
||||||
|
public List<string> Prompts { get; } = [];
|
||||||
|
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public void Show() { }
|
||||||
|
public void ShowWithPrompt(string initialPrompt)
|
||||||
|
{
|
||||||
|
Prompts.Add(initialPrompt);
|
||||||
|
Started.TrySetResult(initialPrompt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class ControlledPrompts : IAgentApprovalPromptService
|
||||||
|
{
|
||||||
|
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
|
||||||
|
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||||
|
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var request = new PendingPrompt(prompt, cancellationToken);
|
||||||
|
requests.Writer.TryWrite(request);
|
||||||
|
try { return await request.Response.Task.WaitAsync(cancellationToken); }
|
||||||
|
finally { request.Removed.TrySetResult(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
|
||||||
|
{
|
||||||
|
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,22 +13,23 @@ public sealed class TaskbarIconTests
|
|||||||
{
|
{
|
||||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||||
Status(),
|
Status(),
|
||||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")]);
|
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
|
||||||
|
[new MicrophoneDevice("integrated", "integrated microphone")],
|
||||||
|
"integrated");
|
||||||
|
|
||||||
Assert.Equal(RecordingProcessState.Idle, menu.State);
|
Assert.Equal(RecordingProcessState.Idle, menu.State);
|
||||||
Assert.Contains(menu.Items, item =>
|
AssertMenuLayout(
|
||||||
item.Action == MeetingTaskbarAction.EditRules &&
|
menu,
|
||||||
item.Text == "Open agent");
|
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||||
Assert.Contains(menu.Items, item =>
|
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
|
||||||
item.Action == MeetingTaskbarAction.StartRecording &&
|
("Start meeting recording (default)\tCtrl+Alt+M", MeetingTaskbarAction.StartRecording, false),
|
||||||
item.ProfileName == "default" &&
|
("Start meeting recording (english)\tCtrl+Alt+L", MeetingTaskbarAction.StartRecording, false),
|
||||||
item.Text == "Start meeting recording (default)\tCtrl+Alt+M");
|
("Exit", MeetingTaskbarAction.Exit, true));
|
||||||
Assert.Contains(menu.Items, item =>
|
Assert.Equal(
|
||||||
item.Action == MeetingTaskbarAction.StartRecording &&
|
["default", "english"],
|
||||||
item.ProfileName == "english" &&
|
menu.Items
|
||||||
item.Text == "Start meeting recording (english)\tCtrl+Alt+L");
|
.Where(item => item.Action == MeetingTaskbarAction.StartRecording)
|
||||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
|
.Select(item => item.ProfileName));
|
||||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -60,27 +61,64 @@ public sealed class TaskbarIconTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void RecordingMenuOffersStopAbortAndOtherProfileSwitches()
|
public void RecordingMenuPrioritizesFinishMeetingInDedicatedSection()
|
||||||
{
|
{
|
||||||
var menu = MeetingTaskbarMenuBuilder.Build(
|
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||||
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L"), Profile("french", "Ctrl+Alt+F")]);
|
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
|
||||||
|
[new MicrophoneDevice("integrated", "integrated microphone")],
|
||||||
|
"integrated");
|
||||||
|
|
||||||
Assert.Equal(RecordingProcessState.Recording, menu.State);
|
Assert.Equal(RecordingProcessState.Recording, menu.State);
|
||||||
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
|
AssertMenuLayout(
|
||||||
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
|
menu,
|
||||||
Assert.Contains(menu.Items, item =>
|
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||||
item.ProfileName == "english" &&
|
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||||
item.Text == "Switch to english\tCtrl+Alt+L");
|
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
|
||||||
Assert.Contains(menu.Items, item =>
|
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
|
||||||
item.ProfileName == "french" &&
|
("Exit", MeetingTaskbarAction.Exit, true));
|
||||||
item.Text == "Switch to french\tCtrl+Alt+F");
|
Assert.Equal(
|
||||||
Assert.DoesNotContain(menu.Items, item =>
|
"english",
|
||||||
item.Action == MeetingTaskbarAction.SwitchProfile &&
|
Assert.Single(menu.Items, item => item.Action == MeetingTaskbarAction.SwitchProfile).ProfileName);
|
||||||
item.ProfileName == "default");
|
}
|
||||||
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StartRecording);
|
|
||||||
|
[Fact]
|
||||||
|
public void RecordingMenuKeepsFinishMeetingIsolatedWithoutMicrophones()
|
||||||
|
{
|
||||||
|
var menu = MeetingTaskbarMenuBuilder.Build(
|
||||||
|
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||||
|
[Profile("default")]);
|
||||||
|
|
||||||
|
AssertMenuLayout(
|
||||||
|
menu,
|
||||||
|
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||||
|
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||||
|
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||||
|
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||||
|
("Exit", MeetingTaskbarAction.Exit, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
|
||||||
|
{
|
||||||
|
var runningMenu = MeetingTaskbarMenuBuilder.Build(
|
||||||
|
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||||
|
[Profile("default")]);
|
||||||
|
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
|
||||||
|
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
|
||||||
|
[Profile("default")]);
|
||||||
|
|
||||||
|
Assert.Contains(runningMenu.Items, item =>
|
||||||
|
item.Action == MeetingTaskbarAction.PauseTranscription &&
|
||||||
|
item.Text == "Pause transcription");
|
||||||
|
Assert.Contains(pausedMenu.Items, item =>
|
||||||
|
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
|
||||||
|
item.Text == "Unpause transcription");
|
||||||
|
Assert.Contains(pausedMenu.Items, item =>
|
||||||
|
item.Action == MeetingTaskbarAction.StopRecording &&
|
||||||
|
item.Text == "Finish meeting");
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -173,10 +211,20 @@ public sealed class TaskbarIconTests
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void AssertMenuLayout(
|
||||||
|
MeetingTaskbarMenu menu,
|
||||||
|
params (string Text, MeetingTaskbarAction Action, bool StartsSection)[] expected)
|
||||||
|
{
|
||||||
|
Assert.Equal(
|
||||||
|
expected,
|
||||||
|
menu.Items.Select(item => (item.Text, item.Action, item.StartsSection)));
|
||||||
|
}
|
||||||
|
|
||||||
private static RecordingStatus Status(
|
private static RecordingStatus Status(
|
||||||
bool isRecording = false,
|
bool isRecording = false,
|
||||||
RecordingProcessState state = RecordingProcessState.Idle,
|
RecordingProcessState state = RecordingProcessState.Idle,
|
||||||
string? profile = null)
|
string? profile = null,
|
||||||
|
bool isPaused = false)
|
||||||
{
|
{
|
||||||
return new RecordingStatus(
|
return new RecordingStatus(
|
||||||
isRecording,
|
isRecording,
|
||||||
@@ -185,7 +233,8 @@ public sealed class TaskbarIconTests
|
|||||||
state == RecordingProcessState.Idle ? null : "context.md",
|
state == RecordingProcessState.Idle ? null : "context.md",
|
||||||
state == RecordingProcessState.Idle ? null : "summary.md",
|
state == RecordingProcessState.Idle ? null : "summary.md",
|
||||||
state,
|
state,
|
||||||
profile);
|
profile,
|
||||||
|
isPaused);
|
||||||
}
|
}
|
||||||
|
|
||||||
[SupportedOSPlatform("windows")]
|
[SupportedOSPlatform("windows")]
|
||||||
|
|||||||
@@ -0,0 +1,249 @@
|
|||||||
|
using MeetingAssistant;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class VenvResemblyzerVoiceEncoderTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
|
||||||
|
{
|
||||||
|
using var fixture = new EncoderFixture();
|
||||||
|
|
||||||
|
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.DoesNotContain(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
|
||||||
|
Assert.Contains(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.FileName == "python-test"
|
||||||
|
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
|
||||||
|
Assert.Contains(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.Arguments.Contains("torch==2.14.0+cpu")
|
||||||
|
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
|
||||||
|
Assert.Contains(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
|
||||||
|
Assert.Contains(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.Arguments.Contains("Resemblyzer==0.1.4")
|
||||||
|
&& command.Arguments.Contains("--no-deps"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
|
||||||
|
{
|
||||||
|
using var fixture = new EncoderFixture(
|
||||||
|
warmupExitCode: 21,
|
||||||
|
warmupError: "incompatible environment");
|
||||||
|
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||||
|
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Empty(Directory.EnumerateFiles(
|
||||||
|
fixture.RuntimeFolder,
|
||||||
|
".ready",
|
||||||
|
SearchOption.AllDirectories));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
|
||||||
|
{
|
||||||
|
var first = new float[256];
|
||||||
|
first[0] = 2;
|
||||||
|
var second = new float[256];
|
||||||
|
second[1] = 3;
|
||||||
|
using var fixture = new EncoderFixture(VectorOutput([first, second]));
|
||||||
|
|
||||||
|
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(2, vectors.Count);
|
||||||
|
Assert.Equal(1f, vectors[0][0], 5);
|
||||||
|
Assert.Equal(1f, vectors[1][1], 5);
|
||||||
|
var encodingCommand = Assert.Single(
|
||||||
|
fixture.Runner.Commands,
|
||||||
|
command => command.Arguments.Any(
|
||||||
|
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
|
||||||
|
Assert.EndsWith(
|
||||||
|
Path.Combine("Scripts", "python.exe"),
|
||||||
|
encodingCommand.FileName,
|
||||||
|
StringComparison.OrdinalIgnoreCase);
|
||||||
|
Assert.Equal(2, encodingCommand.Arguments.Count);
|
||||||
|
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRejectsWrongVectorDimension()
|
||||||
|
{
|
||||||
|
var vector = new float[255];
|
||||||
|
vector[0] = 1;
|
||||||
|
using var fixture = new EncoderFixture(VectorOutput([vector]));
|
||||||
|
|
||||||
|
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Contains("255 dimensions", error.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRejectsMalformedVectorJson()
|
||||||
|
{
|
||||||
|
using var fixture = new EncoderFixture(
|
||||||
|
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||||
|
|
||||||
|
await Assert.ThrowsAsync<InvalidDataException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRejectsNonFiniteVector()
|
||||||
|
{
|
||||||
|
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
|
||||||
|
using var fixture = new EncoderFixture(
|
||||||
|
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||||
|
|
||||||
|
await Assert.ThrowsAsync<InvalidDataException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRejectsWrongResultCount()
|
||||||
|
{
|
||||||
|
var vector = new float[256];
|
||||||
|
vector[0] = 1;
|
||||||
|
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
|
||||||
|
|
||||||
|
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
|
||||||
|
{
|
||||||
|
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
|
||||||
|
|
||||||
|
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Contains("zero magnitude", error.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EncodeAsyncReportsFailedLocalCommand()
|
||||||
|
{
|
||||||
|
using var fixture = new EncoderFixture("", 17, "runtime failed");
|
||||||
|
|
||||||
|
var error = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||||
|
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Contains("exit code 17", error.Message);
|
||||||
|
Assert.Contains("runtime failed", error.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string VectorOutput(IReadOnlyList<float[]> vectors)
|
||||||
|
{
|
||||||
|
return $"""
|
||||||
|
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
|
||||||
|
{JsonSerializer.Serialize(vectors)}
|
||||||
|
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
|
||||||
|
""";
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class EncoderFixture : IDisposable
|
||||||
|
{
|
||||||
|
public EncoderFixture(
|
||||||
|
string encodingOutput = "",
|
||||||
|
int encodingExitCode = 0,
|
||||||
|
string encodingError = "",
|
||||||
|
int warmupExitCode = 0,
|
||||||
|
string warmupError = "")
|
||||||
|
{
|
||||||
|
RuntimeFolder = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
"meeting-assistant-tests",
|
||||||
|
Guid.NewGuid().ToString("N"),
|
||||||
|
"resemblyzer");
|
||||||
|
Runner = new CapturingCommandRunner(
|
||||||
|
encodingOutput,
|
||||||
|
encodingExitCode,
|
||||||
|
encodingError,
|
||||||
|
warmupExitCode,
|
||||||
|
warmupError);
|
||||||
|
var options = new MeetingAssistantOptions();
|
||||||
|
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
|
||||||
|
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
|
||||||
|
Encoder = new VenvResemblyzerVoiceEncoder(
|
||||||
|
Runner,
|
||||||
|
Options.Create(options),
|
||||||
|
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CapturingCommandRunner Runner { get; }
|
||||||
|
|
||||||
|
public VenvResemblyzerVoiceEncoder Encoder { get; }
|
||||||
|
|
||||||
|
public string RuntimeFolder { get; }
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(RuntimeFolder))
|
||||||
|
{
|
||||||
|
Directory.Delete(RuntimeFolder, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CapturingCommandRunner : ICommandRunner
|
||||||
|
{
|
||||||
|
private readonly string encodingOutput;
|
||||||
|
private readonly int encodingExitCode;
|
||||||
|
private readonly string encodingError;
|
||||||
|
private readonly int warmupExitCode;
|
||||||
|
private readonly string warmupError;
|
||||||
|
|
||||||
|
public CapturingCommandRunner(
|
||||||
|
string encodingOutput = "",
|
||||||
|
int encodingExitCode = 0,
|
||||||
|
string encodingError = "",
|
||||||
|
int warmupExitCode = 0,
|
||||||
|
string warmupError = "")
|
||||||
|
{
|
||||||
|
this.encodingOutput = encodingOutput;
|
||||||
|
this.encodingExitCode = encodingExitCode;
|
||||||
|
this.encodingError = encodingError;
|
||||||
|
this.warmupExitCode = warmupExitCode;
|
||||||
|
this.warmupError = warmupError;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<CapturedCommand> Commands { get; } = [];
|
||||||
|
|
||||||
|
public Task<CommandResult> RunAsync(
|
||||||
|
string fileName,
|
||||||
|
IReadOnlyList<string> arguments,
|
||||||
|
CancellationToken cancellationToken,
|
||||||
|
IReadOnlyDictionary<string, string>? environment = null)
|
||||||
|
{
|
||||||
|
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
|
||||||
|
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
|
||||||
|
if (isEncoding)
|
||||||
|
{
|
||||||
|
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
|
||||||
|
}
|
||||||
|
|
||||||
|
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
|
||||||
|
return Task.FromResult(isWarmup
|
||||||
|
? new CommandResult(warmupExitCode, string.Empty, warmupError)
|
||||||
|
: new CommandResult(0, string.Empty, string.Empty));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
|
||||||
|
}
|
||||||
@@ -20,6 +20,7 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
Key = "summary-key",
|
Key = "summary-key",
|
||||||
KeyEnv = "SUMMARY_KEY",
|
KeyEnv = "SUMMARY_KEY",
|
||||||
Model = "summary-model",
|
Model = "summary-model",
|
||||||
|
UseStreaming = true,
|
||||||
EnableThinking = true,
|
EnableThinking = true,
|
||||||
ReasoningEffort = ReasoningEffortOption.High,
|
ReasoningEffort = ReasoningEffortOption.High,
|
||||||
ReconnectionAttempts = 7,
|
ReconnectionAttempts = 7,
|
||||||
@@ -33,6 +34,7 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
var editor = new WorkflowRulesEditorOptions
|
var editor = new WorkflowRulesEditorOptions
|
||||||
{
|
{
|
||||||
Model = "editor-model",
|
Model = "editor-model",
|
||||||
|
UseStreaming = false,
|
||||||
EnableThinking = false,
|
EnableThinking = false,
|
||||||
MaxOutputTokens = 50
|
MaxOutputTokens = 50
|
||||||
};
|
};
|
||||||
@@ -43,6 +45,7 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
Assert.Equal("summary-key", effective.Key);
|
Assert.Equal("summary-key", effective.Key);
|
||||||
Assert.Equal("SUMMARY_KEY", effective.KeyEnv);
|
Assert.Equal("SUMMARY_KEY", effective.KeyEnv);
|
||||||
Assert.Equal("editor-model", effective.Model);
|
Assert.Equal("editor-model", effective.Model);
|
||||||
|
Assert.False(effective.UseStreaming);
|
||||||
Assert.False(effective.EnableThinking);
|
Assert.False(effective.EnableThinking);
|
||||||
Assert.Equal(ReasoningEffortOption.High, effective.ReasoningEffort);
|
Assert.Equal(ReasoningEffortOption.High, effective.ReasoningEffort);
|
||||||
Assert.Equal(7, effective.ReconnectionAttempts);
|
Assert.Equal(7, effective.ReconnectionAttempts);
|
||||||
@@ -760,6 +763,29 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
Assert.Contains("matching correction", instructions);
|
Assert.Contains("matching correction", instructions);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
|
||||||
|
{
|
||||||
|
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||||
|
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||||
|
var options = new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
WorkflowRulesEditor = new WorkflowRulesEditorOptions
|
||||||
|
{
|
||||||
|
InitialPrompt = "Custom interactive agent instructions."
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var instructions = await builder.BuildAsync(options, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Contains("Custom interactive agent instructions.", instructions);
|
||||||
|
Assert.Contains("meeting-specific memory", instructions);
|
||||||
|
Assert.Contains("fix or investigate a meeting or summary", instructions);
|
||||||
|
Assert.Contains("read the matching assistant context", instructions);
|
||||||
|
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
|
||||||
|
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task RulesEditorToolsCrudAndSearchSpeakerIdentities()
|
public async Task RulesEditorToolsCrudAndSearchSpeakerIdentities()
|
||||||
{
|
{
|
||||||
@@ -783,6 +809,32 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
|
||||||
|
{
|
||||||
|
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
|
||||||
|
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
|
||||||
|
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
SpeakerIdentityId = identity.Id,
|
||||||
|
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||||
|
Dimensions = 256,
|
||||||
|
VectorBytes = new byte[256 * sizeof(float)],
|
||||||
|
Fingerprint = "vector-1",
|
||||||
|
CreatedAt = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
await fixture.Context.SaveChangesAsync();
|
||||||
|
var tools = fixture.CreateTools();
|
||||||
|
|
||||||
|
var searchResult = await tools.SearchIdentities("Sabrina");
|
||||||
|
var readResult = await tools.ReadIdentity(identity.Id);
|
||||||
|
|
||||||
|
Assert.Contains("\"sampleCount\": 1", searchResult);
|
||||||
|
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
|
||||||
|
Assert.Contains("\"voiceVectorCount\": 1", readResult);
|
||||||
|
Assert.DoesNotContain("vector-1", readResult);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
|
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
|
||||||
{
|
{
|
||||||
@@ -791,7 +843,7 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
|
|
||||||
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
|
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
|
||||||
|
|
||||||
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
|
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
|
||||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -888,6 +940,30 @@ public sealed class WorkflowRulesEditorTests
|
|||||||
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
|
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
|
||||||
|
{
|
||||||
|
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
|
||||||
|
{
|
||||||
|
Draft = "My unfinished draft"
|
||||||
|
};
|
||||||
|
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
|
||||||
|
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
|
||||||
|
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
|
||||||
|
|
||||||
|
var task = requested.SendPromptAsync(prompt);
|
||||||
|
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
Assert.True(requested.IsThinking);
|
||||||
|
Assert.Empty(pipeline.LastConversation);
|
||||||
|
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
|
||||||
|
Assert.Equal("My unfinished draft", existing.Draft);
|
||||||
|
Assert.Empty(existing.Messages);
|
||||||
|
pipeline.Release.SetResult();
|
||||||
|
await task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||||
|
Assert.False(requested.IsThinking);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ViewModelShowsAgentErrorWhenRequestTimesOut()
|
public async Task ViewModelShowsAgentErrorWhenRequestTimesOut()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -98,7 +98,9 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
|||||||
meeting.Subject,
|
meeting.Subject,
|
||||||
meeting.Start);
|
meeting.Start);
|
||||||
await promptService.ShowPromptAsync(
|
await promptService.ShowPromptAsync(
|
||||||
new MeetingStartPromptRequest(meeting),
|
new MeetingStartPromptRequest(
|
||||||
|
meeting,
|
||||||
|
recordingController.CurrentStatus.IsRecording && meeting.Metadata is not null),
|
||||||
(response, token) => HandlePromptResponseAsync(meeting, response, token),
|
(response, token) => HandlePromptResponseAsync(meeting, response, token),
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
@@ -139,6 +141,18 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
|||||||
MeetingStartPromptResponse response,
|
MeetingStartPromptResponse response,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
|
if (response == MeetingStartPromptResponse.AttachMetadataToCurrentMeeting)
|
||||||
|
{
|
||||||
|
if (meeting.Metadata is not null)
|
||||||
|
{
|
||||||
|
await recordingController.AttachMetadataToCurrentMeetingAsync(
|
||||||
|
meeting.Metadata,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (response != MeetingStartPromptResponse.Record)
|
if (response != MeetingStartPromptResponse.Record)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
@@ -244,12 +258,15 @@ public interface IMeetingStartPromptService
|
|||||||
CancellationToken cancellationToken);
|
CancellationToken cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record MeetingStartPromptRequest(CalendarMeeting Meeting);
|
public sealed record MeetingStartPromptRequest(
|
||||||
|
CalendarMeeting Meeting,
|
||||||
|
bool CanAttachToCurrentMeeting = false);
|
||||||
|
|
||||||
public enum MeetingStartPromptResponse
|
public enum MeetingStartPromptResponse
|
||||||
{
|
{
|
||||||
Record,
|
Record,
|
||||||
Skip
|
Skip,
|
||||||
|
AttachMetadataToCurrentMeeting
|
||||||
}
|
}
|
||||||
|
|
||||||
public interface IMeetingPromptRecordingController
|
public interface IMeetingPromptRecordingController
|
||||||
@@ -262,6 +279,10 @@ public interface IMeetingPromptRecordingController
|
|||||||
MeetingMetadata? metadata,
|
MeetingMetadata? metadata,
|
||||||
CancellationToken cancellationToken);
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||||
|
MeetingMetadata metadata,
|
||||||
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
|
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,6 +309,13 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo
|
|||||||
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
|
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||||
|
MeetingMetadata metadata,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return coordinator.AttachMetadataToCurrentMeetingAsync(metadata, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return coordinator.StopAsync(cancellationToken);
|
return coordinator.StopAsync(cancellationToken);
|
||||||
|
|||||||
@@ -118,21 +118,10 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
|||||||
string promptId,
|
string promptId,
|
||||||
MeetingStartPromptRequest request)
|
MeetingStartPromptRequest request)
|
||||||
{
|
{
|
||||||
var yesButton = new ToastButton()
|
var yesButton = BuildResponseButton("Yes", promptId, "record");
|
||||||
.SetContent("Yes")
|
var noButton = BuildResponseButton("No", promptId, "skip");
|
||||||
.AddArgument("source", NotificationSource)
|
|
||||||
.AddArgument("promptId", promptId)
|
|
||||||
.AddArgument("response", "record")
|
|
||||||
.SetBackgroundActivation();
|
|
||||||
|
|
||||||
var noButton = new ToastButton()
|
var notification = new ToastContentBuilder()
|
||||||
.SetContent("No")
|
|
||||||
.AddArgument("source", NotificationSource)
|
|
||||||
.AddArgument("promptId", promptId)
|
|
||||||
.AddArgument("response", "skip")
|
|
||||||
.SetBackgroundActivation();
|
|
||||||
|
|
||||||
return new ToastContentBuilder()
|
|
||||||
.AddArgument("source", NotificationSource)
|
.AddArgument("source", NotificationSource)
|
||||||
.AddArgument("promptId", promptId)
|
.AddArgument("promptId", promptId)
|
||||||
.SetToastScenario(ToastScenario.Reminder)
|
.SetToastScenario(ToastScenario.Reminder)
|
||||||
@@ -141,6 +130,30 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
|||||||
.AddText($"{request.Meeting.Subject} starts at {request.Meeting.Start.LocalDateTime:t}.")
|
.AddText($"{request.Meeting.Subject} starts at {request.Meeting.Start.LocalDateTime:t}.")
|
||||||
.AddButton(yesButton)
|
.AddButton(yesButton)
|
||||||
.AddButton(noButton);
|
.AddButton(noButton);
|
||||||
|
|
||||||
|
if (request.CanAttachToCurrentMeeting)
|
||||||
|
{
|
||||||
|
notification.AddButton(
|
||||||
|
BuildResponseButton(
|
||||||
|
"Add metadata to current meeting",
|
||||||
|
promptId,
|
||||||
|
"attach-metadata"));
|
||||||
|
}
|
||||||
|
|
||||||
|
return notification;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ToastButton BuildResponseButton(
|
||||||
|
string content,
|
||||||
|
string promptId,
|
||||||
|
string response)
|
||||||
|
{
|
||||||
|
return new ToastButton()
|
||||||
|
.SetContent(content)
|
||||||
|
.AddArgument("source", NotificationSource)
|
||||||
|
.AddArgument("promptId", promptId)
|
||||||
|
.AddArgument("response", response)
|
||||||
|
.SetBackgroundActivation();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||||
@@ -166,10 +179,14 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||||
string.Equals(responseValue, "record", StringComparison.OrdinalIgnoreCase)
|
? responseValue.ToLowerInvariant() switch
|
||||||
? MeetingStartPromptResponse.Record
|
{
|
||||||
: MeetingStartPromptResponse.Skip;
|
"record" => MeetingStartPromptResponse.Record,
|
||||||
|
"attach-metadata" => MeetingStartPromptResponse.AttachMetadataToCurrentMeeting,
|
||||||
|
_ => MeetingStartPromptResponse.Skip
|
||||||
|
}
|
||||||
|
: MeetingStartPromptResponse.Skip;
|
||||||
_ = Task.Run(async () =>
|
_ = Task.Run(async () =>
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
|||||||
@@ -32,7 +32,14 @@ Calendar-Feature is refined, Golem feature needs more work.
|
|||||||
|
|
||||||
PROJECT.md should use this structure:
|
PROJECT.md should use this structure:
|
||||||
|
|
||||||
|
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
|
---
|
||||||
|
name: Project display name
|
||||||
|
description: Concise project purpose and distinguishing context.
|
||||||
|
---
|
||||||
|
|
||||||
# Executive Summary
|
# Executive Summary
|
||||||
|
|
||||||
<Elevator Pitch>
|
<Elevator Pitch>
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using MeetingAssistant.Speakers;
|
||||||
|
|
||||||
namespace MeetingAssistant.LaunchProfiles;
|
namespace MeetingAssistant.LaunchProfiles;
|
||||||
|
|
||||||
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
|||||||
var options = BindDefaultOptions();
|
var options = BindDefaultOptions();
|
||||||
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
|
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
|
ValidateSpeakerSampleDurations(options, DefaultProfileName);
|
||||||
return new LaunchProfile(DefaultProfileName, options);
|
return new LaunchProfile(DefaultProfileName, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
|||||||
|
|
||||||
profileSection.Bind(options);
|
profileSection.Bind(options);
|
||||||
ApplyArrayOverrides(profileSection, options);
|
ApplyArrayOverrides(profileSection, options);
|
||||||
|
ValidateSpeakerSampleDurations(options, profileName);
|
||||||
return new LaunchProfile(profileName, options);
|
return new LaunchProfile(profileName, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
|||||||
: name.Trim();
|
: name.Trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void ValidateSpeakerSampleDurations(
|
||||||
|
MeetingAssistantOptions options,
|
||||||
|
string profileName)
|
||||||
|
{
|
||||||
|
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||||
|
options.SpeakerIdentification,
|
||||||
|
$"Launch profile '{profileName}' has invalid speaker sample durations");
|
||||||
|
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
|
||||||
|
options.SpeakerIdentification.Resemblyzer,
|
||||||
|
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
|
||||||
|
}
|
||||||
|
|
||||||
private static void ApplyArrayOverrides(
|
private static void ApplyArrayOverrides(
|
||||||
IConfigurationSection profileSection,
|
IConfigurationSection profileSection,
|
||||||
MeetingAssistantOptions options)
|
MeetingAssistantOptions options)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||||
<MicrosoftSpeechVersion>1.50.0</MicrosoftSpeechVersion>
|
<MicrosoftSpeechVersion>1.52.0</MicrosoftSpeechVersion>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||||
@@ -20,16 +20,16 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.13.0" />
|
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
|
||||||
<PackageReference Include="DiffPlex" Version="1.9.0" />
|
<PackageReference Include="DiffPlex" Version="1.9.0" />
|
||||||
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
|
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
|
||||||
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
|
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.9" />
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
|
||||||
<PackageReference Include="NAudio" Version="2.3.0" />
|
<PackageReference Include="NAudio" Version="3.1.0" />
|
||||||
<PackageReference Include="NCalcSync" Version="6.4.0" />
|
<PackageReference Include="NCalcSync" Version="7.2.0" />
|
||||||
<PackageReference Include="RazorLight" Version="2.3.1" />
|
<PackageReference Include="RazorLight" Version="2.3.1" />
|
||||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.3" />
|
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
|
||||||
<PackageReference Include="System.Drawing.Common" Version="10.0.9" />
|
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
|
||||||
<PackageReference Include="Whisper.net" Version="1.9.1" />
|
<PackageReference Include="Whisper.net" Version="1.9.1" />
|
||||||
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
|
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
|
||||||
<PackageReference Include="YamlDotNet" Version="18.1.0" />
|
<PackageReference Include="YamlDotNet" Version="18.1.0" />
|
||||||
@@ -50,8 +50,7 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\aec_v1.fpie" Link="runtimes\win-x64\native\MASmodels\aec_v1.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\*.fpie" Link="runtimes\win-x64\native\MASmodels\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\pns_avg4.fpie" Link="runtimes\win-x64\native\MASmodels\pns_avg4.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@@ -65,6 +64,7 @@
|
|||||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||||
|
<Compile Remove="Recording\WindowsMicrophoneDeviceProvider.cs" />
|
||||||
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
|
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
|
||||||
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
|
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
|
||||||
<Compile Remove="Taskbar\UnoTaskbarIconService.Windows.cs" />
|
<Compile Remove="Taskbar\UnoTaskbarIconService.Windows.cs" />
|
||||||
|
|||||||
@@ -165,6 +165,8 @@ public sealed class RecordingInactivitySafeguardOptions
|
|||||||
|
|
||||||
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
|
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
|
||||||
|
|
||||||
|
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
|
||||||
|
|
||||||
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
|
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
|
||||||
|
|
||||||
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
|
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
|
||||||
@@ -181,10 +183,8 @@ public sealed class WhisperLocalOptions
|
|||||||
public PyannoteDiarizationOptions Diarization { get; set; } = new();
|
public PyannoteDiarizationOptions Diarization { get; set; } = new();
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class PyannoteDiarizationOptions
|
public class PyannoteRuntimeOptions
|
||||||
{
|
{
|
||||||
public bool Enabled { get; set; } = true;
|
|
||||||
|
|
||||||
public string DockerCommand { get; set; } = "docker";
|
public string DockerCommand { get; set; } = "docker";
|
||||||
|
|
||||||
public string BaseImage { get; set; } = "python:3.11-slim";
|
public string BaseImage { get; set; } = "python:3.11-slim";
|
||||||
@@ -208,6 +208,11 @@ public sealed class PyannoteDiarizationOptions
|
|||||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
|
||||||
|
{
|
||||||
|
public bool Enabled { get; set; } = true;
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class AzureSpeechOptions
|
public sealed class AzureSpeechOptions
|
||||||
{
|
{
|
||||||
public string Endpoint { get; set; } = "";
|
public string Endpoint { get; set; } = "";
|
||||||
@@ -340,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
|
|||||||
|
|
||||||
public int MaxSnippetsPerSpeaker { get; set; } = 3;
|
public int MaxSnippetsPerSpeaker { get; set; } = 3;
|
||||||
|
|
||||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
|
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
|
||||||
|
|
||||||
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
|
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
|
||||||
|
|
||||||
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
|
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
|
||||||
|
|
||||||
@@ -355,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
|
|||||||
public AzureSpeechOptions AzureSpeech { get; set; } = new();
|
public AzureSpeechOptions AzureSpeech { get; set; } = new();
|
||||||
|
|
||||||
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
|
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
|
||||||
|
|
||||||
|
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ResemblyzerSpeakerRecognitionOptions
|
||||||
|
{
|
||||||
|
public bool Enabled { get; set; }
|
||||||
|
|
||||||
|
public int RequiredVectorsPerSpeaker { get; set; } = 5;
|
||||||
|
|
||||||
|
public int MaxVectorsPerIdentity { get; set; } = 1000;
|
||||||
|
|
||||||
|
public int OutlierPruningMinimumVectors { get; set; } = 20;
|
||||||
|
|
||||||
|
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
|
||||||
|
|
||||||
|
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
|
||||||
|
|
||||||
|
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
|
||||||
|
|
||||||
|
public double MinimumClusterCohesion { get; set; } = 0.75;
|
||||||
|
|
||||||
|
public double MinimumIdentitySimilarity { get; set; } = 0.75;
|
||||||
|
|
||||||
|
public double MinimumSimilarityMargin { get; set; } = 0.05;
|
||||||
|
|
||||||
|
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
|
||||||
|
|
||||||
|
public string PackageVersion { get; set; } = "0.1.4";
|
||||||
|
|
||||||
|
public string PythonCommand { get; set; } = "python";
|
||||||
|
|
||||||
|
public string TorchVersion { get; set; } = "2.14.0+cpu";
|
||||||
|
|
||||||
|
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
|
||||||
|
|
||||||
|
public string WebRtcVadVersion { get; set; } = "2.0.14";
|
||||||
|
|
||||||
|
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
|
||||||
|
|
||||||
|
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class SpeakerIdentityPyannoteValidationOptions
|
public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||||
@@ -365,9 +411,8 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
|
|||||||
|
|
||||||
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
|
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
|
||||||
|
|
||||||
public PyannoteDiarizationOptions Diarization { get; set; } = new()
|
public PyannoteRuntimeOptions Diarization { get; set; } = new()
|
||||||
{
|
{
|
||||||
Enabled = true,
|
|
||||||
CommandTimeout = TimeSpan.FromHours(1),
|
CommandTimeout = TimeSpan.FromHours(1),
|
||||||
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
|
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
|
||||||
};
|
};
|
||||||
@@ -383,6 +428,8 @@ public sealed class AgentOptions
|
|||||||
|
|
||||||
public string Model { get; set; } = "chatgpt/gpt-5.5";
|
public string Model { get; set; } = "chatgpt/gpt-5.5";
|
||||||
|
|
||||||
|
public bool UseStreaming { get; set; } = true;
|
||||||
|
|
||||||
public bool EnableThinking { get; set; } = true;
|
public bool EnableThinking { get; set; } = true;
|
||||||
|
|
||||||
public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium;
|
public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium;
|
||||||
@@ -414,6 +461,8 @@ public sealed class WorkflowRulesEditorOptions
|
|||||||
|
|
||||||
public string? Model { get; set; }
|
public string? Model { get; set; }
|
||||||
|
|
||||||
|
public bool? UseStreaming { get; set; }
|
||||||
|
|
||||||
public bool? EnableThinking { get; set; }
|
public bool? EnableThinking { get; set; }
|
||||||
|
|
||||||
public ReasoningEffortOption? ReasoningEffort { get; set; }
|
public ReasoningEffortOption? ReasoningEffort { get; set; }
|
||||||
@@ -442,6 +491,7 @@ public sealed class WorkflowRulesEditorOptions
|
|||||||
Key = string.IsNullOrWhiteSpace(Key) ? defaults.Key : Key,
|
Key = string.IsNullOrWhiteSpace(Key) ? defaults.Key : Key,
|
||||||
KeyEnv = string.IsNullOrWhiteSpace(KeyEnv) ? defaults.KeyEnv : KeyEnv!,
|
KeyEnv = string.IsNullOrWhiteSpace(KeyEnv) ? defaults.KeyEnv : KeyEnv!,
|
||||||
Model = string.IsNullOrWhiteSpace(Model) ? defaults.Model : Model!,
|
Model = string.IsNullOrWhiteSpace(Model) ? defaults.Model : Model!,
|
||||||
|
UseStreaming = UseStreaming ?? defaults.UseStreaming,
|
||||||
EnableThinking = EnableThinking ?? defaults.EnableThinking,
|
EnableThinking = EnableThinking ?? defaults.EnableThinking,
|
||||||
ReasoningEffort = ReasoningEffort ?? defaults.ReasoningEffort,
|
ReasoningEffort = ReasoningEffort ?? defaults.ReasoningEffort,
|
||||||
ReconnectionAttempts = ReconnectionAttempts ?? defaults.ReconnectionAttempts,
|
ReconnectionAttempts = ReconnectionAttempts ?? defaults.ReconnectionAttempts,
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Text;
|
||||||
using YamlDotNet.Core;
|
using YamlDotNet.Core;
|
||||||
using YamlDotNet.Serialization;
|
using YamlDotNet.Serialization;
|
||||||
|
|
||||||
@@ -113,7 +115,24 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
|||||||
|
|
||||||
private static string EscapeQuoted(string value)
|
private static string EscapeQuoted(string value)
|
||||||
{
|
{
|
||||||
return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
|
var escaped = new StringBuilder(value.Length + 2);
|
||||||
|
escaped.Append('"');
|
||||||
|
foreach (var character in value)
|
||||||
|
{
|
||||||
|
escaped.Append(character switch
|
||||||
|
{
|
||||||
|
'\\' => "\\\\",
|
||||||
|
'"' => "\\\"",
|
||||||
|
'\r' => "\\r",
|
||||||
|
'\n' => "\\n",
|
||||||
|
'\t' => "\\t",
|
||||||
|
< ' ' => "\\x" + ((int)character).ToString("X2", CultureInfo.InvariantCulture),
|
||||||
|
_ => character.ToString()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
escaped.Append('"');
|
||||||
|
return escaped.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string EscapeNullableDateTime(DateTimeOffset? value)
|
private static string EscapeNullableDateTime(DateTimeOffset? value)
|
||||||
@@ -135,7 +154,7 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
|||||||
return "\"\"";
|
return "\"\"";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal))
|
if (RequiresQuotedScalar(value))
|
||||||
{
|
{
|
||||||
return EscapeQuoted(value);
|
return EscapeQuoted(value);
|
||||||
}
|
}
|
||||||
@@ -143,6 +162,49 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool RequiresQuotedScalar(string value)
|
||||||
|
{
|
||||||
|
if (char.IsWhiteSpace(value[0]) || char.IsWhiteSpace(value[^1]))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value.Contains(':', StringComparison.Ordinal) ||
|
||||||
|
value.Contains('\'', StringComparison.Ordinal) ||
|
||||||
|
value.Contains(" #", StringComparison.Ordinal) ||
|
||||||
|
value.Any(char.IsControl))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (IsYamlIndicator(value[0]))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (IsYamlCoreSchemaKeyword(value))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return DateTimeOffset.TryParse(value, out _) ||
|
||||||
|
double.TryParse(value, CultureInfo.InvariantCulture, out _);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsYamlIndicator(char character)
|
||||||
|
{
|
||||||
|
return character is '-' or '?' or ':' or ',' or '[' or ']' or '{' or '}' or
|
||||||
|
'#' or '&' or '*' or '!' or '|' or '>' or '\'' or '"' or '%' or '@' or '`';
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsYamlCoreSchemaKeyword(string value)
|
||||||
|
{
|
||||||
|
return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
string.Equals(value, "false", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
string.Equals(value, "null", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
value == "~";
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class MeetingNoteYaml
|
private sealed class MeetingNoteYaml
|
||||||
{
|
{
|
||||||
[YamlMember(Alias = "title")]
|
[YamlMember(Alias = "title")]
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
namespace MeetingAssistant.Notifications;
|
||||||
|
|
||||||
|
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
|
||||||
|
|
||||||
|
public interface IAgentApprovalPromptService
|
||||||
|
{
|
||||||
|
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
|
||||||
|
{
|
||||||
|
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class AgentApprovalService
|
||||||
|
{
|
||||||
|
private readonly IAgentApprovalPromptService prompts;
|
||||||
|
private readonly ILogger<AgentApprovalService> logger;
|
||||||
|
private readonly TimeSpan timeout;
|
||||||
|
|
||||||
|
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
|
||||||
|
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
|
||||||
|
|
||||||
|
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
|
||||||
|
{
|
||||||
|
this.prompts = prompts;
|
||||||
|
this.logger = logger;
|
||||||
|
this.timeout = timeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
deadline.CancelAfter(timeout);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
|
||||||
|
.WaitAsync(deadline.Token);
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
return accepted && !deadline.IsCancellationRequested;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await deadline.CancelAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
#if WINDOWS
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using CommunityToolkit.WinUI.Notifications;
|
||||||
|
using MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Notifications;
|
||||||
|
|
||||||
|
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
|
||||||
|
: IAgentApprovalPromptService, IDisposable
|
||||||
|
{
|
||||||
|
private const string Source = "summary-agent-approval";
|
||||||
|
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||||
|
private readonly object gate = new();
|
||||||
|
private bool registered;
|
||||||
|
private bool disposed;
|
||||||
|
|
||||||
|
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
var id = Guid.NewGuid().ToString("N");
|
||||||
|
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
if (disposed) return false;
|
||||||
|
if (!registered)
|
||||||
|
{
|
||||||
|
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||||
|
registered = true;
|
||||||
|
}
|
||||||
|
pending[id] = response;
|
||||||
|
new ToastContentBuilder()
|
||||||
|
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||||
|
.SetToastScenario(ToastScenario.Reminder)
|
||||||
|
.SetToastDuration(ToastDuration.Long)
|
||||||
|
.AddText(prompt.Title)
|
||||||
|
.AddText(prompt.Message)
|
||||||
|
.AddButton(new ToastButton().SetContent("Yes")
|
||||||
|
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||||
|
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||||
|
.AddButton(new ToastButton().SetContent("No")
|
||||||
|
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||||
|
.AddArgument("response", "no").SetBackgroundActivation())
|
||||||
|
.Show(toast =>
|
||||||
|
{
|
||||||
|
toast.Tag = id;
|
||||||
|
toast.Group = Source;
|
||||||
|
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return await response.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
pending.TryRemove(id, out _);
|
||||||
|
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
|
||||||
|
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||||
|
{
|
||||||
|
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||||
|
if (arguments.GetValueOrDefault("source") != Source ||
|
||||||
|
!arguments.TryGetValue("requestId", out var id) ||
|
||||||
|
!pending.TryRemove(id, out var response)) return;
|
||||||
|
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
disposed = true;
|
||||||
|
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||||
|
registered = false;
|
||||||
|
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||||
|
pending.Clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
@@ -4,6 +4,7 @@ using MeetingAssistant.Hotkeys;
|
|||||||
using MeetingAssistant.LaunchProfiles;
|
using MeetingAssistant.LaunchProfiles;
|
||||||
using MeetingAssistant.Logging;
|
using MeetingAssistant.Logging;
|
||||||
using MeetingAssistant.MeetingNotes;
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Notifications;
|
||||||
using MeetingAssistant.Recording;
|
using MeetingAssistant.Recording;
|
||||||
using MeetingAssistant.Screenshots;
|
using MeetingAssistant.Screenshots;
|
||||||
using MeetingAssistant.Speakers;
|
using MeetingAssistant.Speakers;
|
||||||
@@ -16,11 +17,21 @@ using Microsoft.Extensions.Options;
|
|||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
|
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
|
||||||
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
|
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
|
||||||
|
builder.Services.AddOptions<MeetingAssistantOptions>()
|
||||||
|
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
|
||||||
|
.ValidateOnStart();
|
||||||
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
|
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
|
||||||
#if WINDOWS
|
#if WINDOWS
|
||||||
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
|
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
|
||||||
builder.Services.AddSingleton<IMicrophoneDeviceProvider, WindowsMicrophoneDeviceProvider>();
|
builder.Services.AddSingleton<WindowsMicrophoneDeviceProvider>();
|
||||||
|
builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
|
||||||
|
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||||
|
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
|
||||||
|
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||||
|
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
|
||||||
|
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
|
||||||
|
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
|
||||||
builder.Services.AddSingleton<MicrophoneAudioSource>();
|
builder.Services.AddSingleton<MicrophoneAudioSource>();
|
||||||
builder.Services.AddSingleton<SystemAudioSource>();
|
builder.Services.AddSingleton<SystemAudioSource>();
|
||||||
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||||
@@ -32,6 +43,8 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
|
|||||||
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
||||||
#else
|
#else
|
||||||
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||||
|
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
|
||||||
|
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
|
||||||
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||||
#endif
|
#endif
|
||||||
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||||
@@ -87,8 +100,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
|
|||||||
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
|
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
|
||||||
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
|
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
|
||||||
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
|
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
|
||||||
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
|
builder.Services.AddSingleton<SpeakerIdentityService>();
|
||||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
|
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
|
||||||
|
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
|
||||||
|
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||||
|
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
|
||||||
|
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||||
|
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
|
||||||
|
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
|
||||||
|
builder.Services.AddSingleton(services =>
|
||||||
|
{
|
||||||
|
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||||
|
.Value.SpeakerIdentification.Resemblyzer;
|
||||||
|
return resemblyzer.Enabled
|
||||||
|
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
|
||||||
|
: SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||||
|
});
|
||||||
|
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
|
||||||
|
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||||
|
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||||
|
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
|
||||||
|
: services.GetRequiredService<SpeakerIdentityService>());
|
||||||
|
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
|
||||||
|
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
|
||||||
|
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
|
||||||
|
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||||
|
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||||
|
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
|
||||||
|
: services.GetRequiredService<SpeakerIdentityMergeService>());
|
||||||
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
|
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
|
||||||
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
|
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
|
||||||
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
|
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
|
||||||
@@ -98,7 +138,11 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
|
|||||||
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
|
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
|
||||||
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
|
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
|
||||||
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
|
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
|
||||||
|
#if WINDOWS
|
||||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
|
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
|
||||||
|
#else
|
||||||
|
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
|
||||||
|
#endif
|
||||||
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
|
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
|
||||||
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
|
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
|
||||||
#if WINDOWS
|
#if WINDOWS
|
||||||
@@ -108,6 +152,9 @@ builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRule
|
|||||||
#else
|
#else
|
||||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
|
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
|
||||||
#endif
|
#endif
|
||||||
|
builder.Services.AddSingleton<AgentApprovalService>();
|
||||||
|
builder.Services.AddSingleton<WorkflowChangeRequestService>();
|
||||||
|
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
|
||||||
builder.Services.AddSingleton<AsrDiagnosticService>();
|
builder.Services.AddSingleton<AsrDiagnosticService>();
|
||||||
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
|
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||||
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
|
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
|
||||||
@@ -130,6 +177,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
|
|||||||
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
||||||
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
||||||
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
||||||
|
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
|
||||||
#if WINDOWS
|
#if WINDOWS
|
||||||
builder.Services.AddHostedService<GlobalHotkeyService>();
|
builder.Services.AddHostedService<GlobalHotkeyService>();
|
||||||
builder.Services.AddHostedService<UnoTaskbarIconService>();
|
builder.Services.AddHostedService<UnoTaskbarIconService>();
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ public interface IMeetingInactivityPromptService
|
|||||||
MeetingInactivityPromptRequest request,
|
MeetingInactivityPromptRequest request,
|
||||||
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
|
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||||
CancellationToken cancellationToken);
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
Task DismissAllAsync(CancellationToken cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record MeetingInactivityPromptRequest(
|
public sealed record MeetingInactivityPromptRequest(
|
||||||
@@ -18,9 +20,32 @@ public enum MeetingInactivityPromptResponse
|
|||||||
{
|
{
|
||||||
Dismissed,
|
Dismissed,
|
||||||
Continue,
|
Continue,
|
||||||
|
Pause,
|
||||||
Stop
|
Stop
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal sealed record MeetingInactivityPromptAction(
|
||||||
|
string Content,
|
||||||
|
string ResponseArgument,
|
||||||
|
MeetingInactivityPromptResponse Response);
|
||||||
|
|
||||||
|
internal static class MeetingInactivityPromptActions
|
||||||
|
{
|
||||||
|
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
|
||||||
|
[
|
||||||
|
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
|
||||||
|
new("No", "continue", MeetingInactivityPromptResponse.Continue),
|
||||||
|
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
|
||||||
|
];
|
||||||
|
|
||||||
|
public static MeetingInactivityPromptResponse ParseResponse(string? value)
|
||||||
|
{
|
||||||
|
return All.FirstOrDefault(action =>
|
||||||
|
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||||
|
?.Response ?? MeetingInactivityPromptResponse.Continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
|
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
|
||||||
{
|
{
|
||||||
public Task ShowStopPromptAsync(
|
public Task ShowStopPromptAsync(
|
||||||
@@ -30,4 +55,9 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
|
|||||||
{
|
{
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
public interface IMicrophoneCaptureSourceFactory
|
||||||
|
{
|
||||||
|
// The caller owns the returned source and disposes it when it implements IDisposable,
|
||||||
|
// including when capture is canceled before enumeration starts.
|
||||||
|
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
|
||||||
|
}
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
using NAudio.Wave;
|
|
||||||
|
|
||||||
namespace MeetingAssistant.Recording;
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
public interface IMicrophoneDeviceProvider
|
public interface IMicrophoneDeviceProvider
|
||||||
@@ -7,6 +5,4 @@ public interface IMicrophoneDeviceProvider
|
|||||||
IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones();
|
IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones();
|
||||||
|
|
||||||
MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options);
|
MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options);
|
||||||
|
|
||||||
IWaveIn CreateCapture(MeetingAssistantOptions options);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
public interface IMicrophoneSwitchPromptService
|
||||||
|
{
|
||||||
|
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
|
||||||
|
{
|
||||||
|
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult(false);
|
||||||
|
}
|
||||||
@@ -33,6 +33,17 @@ public interface ITranscriptStore
|
|||||||
string line,
|
string line,
|
||||||
CancellationToken cancellationToken);
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
async Task AppendLinesAsync(
|
||||||
|
TranscriptSession session,
|
||||||
|
IReadOnlyList<string> lines,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
foreach (var line in lines)
|
||||||
|
{
|
||||||
|
await AppendLineAsync(session, line, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Task ReplaceLineAsync(
|
Task ReplaceLineAsync(
|
||||||
TranscriptSession session,
|
TranscriptSession session,
|
||||||
TranscriptLineReference lineReference,
|
TranscriptLineReference lineReference,
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
private readonly IMeetingInactivityClock inactivityClock;
|
private readonly IMeetingInactivityClock inactivityClock;
|
||||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||||
private readonly MeetingAssistantOptions options;
|
private readonly MeetingAssistantOptions options;
|
||||||
|
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
|
||||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||||
private readonly SemaphoreSlim gate = new(1, 1);
|
private readonly SemaphoreSlim gate = new(1, 1);
|
||||||
private RecordingRun? currentRun;
|
private RecordingRun? currentRun;
|
||||||
@@ -63,7 +64,8 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
||||||
IMeetingInactivityPromptService? inactivityPromptService = null,
|
IMeetingInactivityPromptService? inactivityPromptService = null,
|
||||||
IMeetingInactivityClock? inactivityClock = null,
|
IMeetingInactivityClock? inactivityClock = null,
|
||||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
|
||||||
|
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
|
||||||
{
|
{
|
||||||
this.audioSource = audioSource;
|
this.audioSource = audioSource;
|
||||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||||
@@ -86,6 +88,8 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
||||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||||
this.options = options.Value;
|
this.options = options.Value;
|
||||||
|
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
|
||||||
|
?? SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,7 +100,8 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
currentArtifacts?.AssistantContextPath,
|
currentArtifacts?.AssistantContextPath,
|
||||||
currentArtifacts?.SummaryPath,
|
currentArtifacts?.SummaryPath,
|
||||||
GetProcessState(currentRun),
|
GetProcessState(currentRun),
|
||||||
currentRun?.LaunchProfileName);
|
currentRun?.LaunchProfileName,
|
||||||
|
IsRecording && currentRun?.IsTranscriptionPaused == true);
|
||||||
|
|
||||||
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
|
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
|
||||||
|
|
||||||
@@ -155,6 +160,44 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
|
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
|
||||||
|
MeetingMetadata metadata,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await gate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var run = currentRun;
|
||||||
|
if (run is null || run.IsCaptureStopping)
|
||||||
|
{
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||||
|
run.Artifacts,
|
||||||
|
await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken),
|
||||||
|
metadata,
|
||||||
|
run.Options,
|
||||||
|
cancellationToken);
|
||||||
|
currentMeetingNote = meetingNote;
|
||||||
|
await transcriptStore.UpdateMetadataAsync(
|
||||||
|
run.Session,
|
||||||
|
run.Artifacts,
|
||||||
|
meetingNote,
|
||||||
|
cancellationToken);
|
||||||
|
run.MarkPromptMetadataAttached();
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Attached prompted calendar metadata to active meeting {MeetingNotePath}",
|
||||||
|
run.MeetingNotePath);
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<RecordingStatus> StartAsync(
|
public async Task<RecordingStatus> StartAsync(
|
||||||
string? launchProfileName,
|
string? launchProfileName,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
@@ -213,22 +256,12 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
|
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
|
||||||
if (suppliedMetadata is not null)
|
if (suppliedMetadata is not null)
|
||||||
{
|
{
|
||||||
await ApplyMeetingMetadataAsync(
|
currentMeetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||||
currentArtifacts,
|
currentArtifacts,
|
||||||
currentMeetingNote,
|
currentMeetingNote,
|
||||||
suppliedMetadata,
|
suppliedMetadata,
|
||||||
runOptions,
|
runOptions,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
currentMeetingNote = await meetingNoteStore.SaveAsync(
|
|
||||||
currentMeetingNote,
|
|
||||||
runOptions,
|
|
||||||
cancellationToken);
|
|
||||||
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
|
|
||||||
currentArtifacts,
|
|
||||||
currentMeetingNote,
|
|
||||||
suppliedMetadata.Agenda,
|
|
||||||
suppliedMetadata.ScheduledEnd,
|
|
||||||
cancellationToken);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
|
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
|
||||||
@@ -261,10 +294,12 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
startedAt,
|
startedAt,
|
||||||
launchProfile.Name,
|
launchProfile.Name,
|
||||||
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
|
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
|
||||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
|
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
|
||||||
|
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
|
||||||
|
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
|
||||||
logger);
|
logger);
|
||||||
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
|
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
|
||||||
if (runOptions.Recording.InactivitySafeguard.Enabled)
|
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
|
||||||
{
|
{
|
||||||
_ = Task.Run(
|
_ = Task.Run(
|
||||||
() => RunInactivitySafeguardAsync(run),
|
() => RunInactivitySafeguardAsync(run),
|
||||||
@@ -306,11 +341,14 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
|
|
||||||
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return await StopAsync(null, cancellationToken);
|
return await StopAsync(null, null, null, null, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<RecordingStatus> StopAsync(
|
private async Task<RecordingStatus> StopAsync(
|
||||||
DateTimeOffset? inferredEndTime,
|
DateTimeOffset? inferredEndTime,
|
||||||
|
RecordingRun? expectedRun,
|
||||||
|
long? expectedActivityVersion,
|
||||||
|
DateTimeOffset? expectedPauseStartedAt,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
RecordingRun run;
|
RecordingRun run;
|
||||||
@@ -319,13 +357,30 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
await gate.WaitAsync(cancellationToken);
|
await gate.WaitAsync(cancellationToken);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (currentRun is null || currentRun.IsCaptureStopping)
|
if (currentRun is null ||
|
||||||
|
currentRun.IsCaptureStopping ||
|
||||||
|
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
|
||||||
{
|
{
|
||||||
return CurrentStatus;
|
return CurrentStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
run = currentRun;
|
run = currentRun;
|
||||||
if (inferredEndTime is not null)
|
var inactivityStopClaimed = expectedActivityVersion is not null;
|
||||||
|
if (expectedActivityVersion is not null &&
|
||||||
|
(inferredEndTime is null ||
|
||||||
|
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
|
||||||
|
{
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
|
||||||
|
if (expectedPauseStartedAt is not null &&
|
||||||
|
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
|
||||||
|
{
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
|
||||||
{
|
{
|
||||||
run.SetInferredEndTime(inferredEndTime.Value);
|
run.SetInferredEndTime(inferredEndTime.Value);
|
||||||
}
|
}
|
||||||
@@ -335,7 +390,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
{
|
{
|
||||||
run.Abort();
|
run.Abort();
|
||||||
}
|
}
|
||||||
else
|
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
|
||||||
{
|
{
|
||||||
run.StopCapture();
|
run.StopCapture();
|
||||||
}
|
}
|
||||||
@@ -361,6 +416,12 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
}
|
}
|
||||||
catch (TimeoutException)
|
catch (TimeoutException)
|
||||||
{
|
{
|
||||||
|
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
|
||||||
if (IsAzureSpeechRun(run))
|
if (IsAzureSpeechRun(run))
|
||||||
{
|
{
|
||||||
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
|
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
|
||||||
@@ -381,6 +442,65 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return CurrentStatus;
|
return CurrentStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||||
|
bool isPaused,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||||
|
bool isPaused,
|
||||||
|
RecordingRun? expectedRun,
|
||||||
|
long? expectedActivityVersion,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await gate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var run = currentRun;
|
||||||
|
if (run is null ||
|
||||||
|
run.IsCaptureStopping ||
|
||||||
|
expectedRun is not null && !ReferenceEquals(run, expectedRun))
|
||||||
|
{
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
var changed = false;
|
||||||
|
await run.PipelineGate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var changedAt = inactivityClock.Now;
|
||||||
|
changed = run.TrySetTranscriptionPaused(
|
||||||
|
isPaused,
|
||||||
|
changedAt,
|
||||||
|
expectedActivityVersion);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
run.PipelineGate.Release();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (changed && isPaused)
|
||||||
|
{
|
||||||
|
await DismissInactivityPromptsAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (changed)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Meeting transcription {PauseState}",
|
||||||
|
isPaused ? "paused" : "unpaused");
|
||||||
|
}
|
||||||
|
|
||||||
|
return CurrentStatus;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
RecordingRun run;
|
RecordingRun run;
|
||||||
@@ -633,8 +753,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
chunk.Channels);
|
chunk.Channels);
|
||||||
}
|
}
|
||||||
|
|
||||||
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
|
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
|
||||||
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
@@ -673,7 +792,6 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
run.ResetSpeakerIdentification();
|
run.ResetSpeakerIdentification();
|
||||||
}
|
}
|
||||||
|
|
||||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
|
||||||
var sample = run.TryAddSpeakerSample(segment);
|
var sample = run.TryAddSpeakerSample(segment);
|
||||||
if (sample is not null)
|
if (sample is not null)
|
||||||
{
|
{
|
||||||
@@ -690,6 +808,56 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
var relabeledSegment = run.Relabel(segment);
|
var relabeledSegment = run.Relabel(segment);
|
||||||
run.AddLiveSegment(relabeledSegment);
|
run.AddLiveSegment(relabeledSegment);
|
||||||
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
|
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
|
||||||
|
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||||
|
if (!string.IsNullOrWhiteSpace(segment.Text))
|
||||||
|
{
|
||||||
|
// Switching profiles holds the coordinator gate while draining this reader.
|
||||||
|
// Notification cleanup must not make transcript draining wait for that gate.
|
||||||
|
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task DismissInactivityPromptsForRunAsync(
|
||||||
|
RecordingRun run,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await gate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await DismissInactivityPromptsAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
// Pending cleanup is no longer needed after this run stops capturing.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await inactivityPromptService.DismissAllAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -817,7 +985,10 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
||||||
{
|
{
|
||||||
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
|
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
|
||||||
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
|
var promptThresholds = safeguardOptions.Enabled
|
||||||
|
? GetInactivityPromptThresholds(safeguardOptions)
|
||||||
|
: [];
|
||||||
|
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
|
||||||
var promptedThresholds = new HashSet<TimeSpan>();
|
var promptedThresholds = new HashSet<TimeSpan>();
|
||||||
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
|
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
|
||||||
|
|
||||||
@@ -825,7 +996,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
|
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
|
||||||
string.Join(", ", promptThresholds),
|
string.Join(", ", promptThresholds),
|
||||||
safeguardOptions.AutoStopAfter,
|
safeguardOptions.AutoStopAfter,
|
||||||
safeguardOptions.CheckInterval);
|
checkInterval);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -841,9 +1012,38 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
snapshot.LastTranscriptActivityAt);
|
snapshot.LastTranscriptActivityAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var pauseStartedAt = snapshot.TranscriptionPausedAt;
|
||||||
|
if (pauseStartedAt is not null)
|
||||||
|
{
|
||||||
|
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
|
||||||
|
if (maximumPauseDuration > TimeSpan.Zero &&
|
||||||
|
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
|
||||||
|
maximumPauseDuration);
|
||||||
|
var stopStatus = await StopAsync(
|
||||||
|
null,
|
||||||
|
run,
|
||||||
|
null,
|
||||||
|
pauseStartedAt,
|
||||||
|
CancellationToken.None);
|
||||||
|
if (!stopStatus.IsRecording)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await inactivityClock.DelayAsync(
|
||||||
|
checkInterval,
|
||||||
|
run.CaptureCancellation);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
var now = inactivityClock.Now;
|
var now = inactivityClock.Now;
|
||||||
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
|
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
|
||||||
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
if (safeguardOptions.Enabled &&
|
||||||
|
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||||
inactivityDuration >= safeguardOptions.AutoStopAfter)
|
inactivityDuration >= safeguardOptions.AutoStopAfter)
|
||||||
{
|
{
|
||||||
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
|
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
|
||||||
@@ -851,8 +1051,16 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
|
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
|
||||||
inactivityDuration,
|
inactivityDuration,
|
||||||
inferredEndTime);
|
inferredEndTime);
|
||||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
var stopStatus = await StopAsync(
|
||||||
return;
|
inferredEndTime,
|
||||||
|
run,
|
||||||
|
snapshot.ActivityVersion,
|
||||||
|
null,
|
||||||
|
CancellationToken.None);
|
||||||
|
if (!stopStatus.IsRecording)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var threshold in promptThresholds)
|
foreach (var threshold in promptThresholds)
|
||||||
@@ -869,6 +1077,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
inactivityDuration,
|
inactivityDuration,
|
||||||
threshold,
|
threshold,
|
||||||
inferredEndTime);
|
inferredEndTime);
|
||||||
|
var promptActivityVersion = snapshot.ActivityVersion;
|
||||||
await inactivityPromptService.ShowStopPromptAsync(
|
await inactivityPromptService.ShowStopPromptAsync(
|
||||||
new MeetingInactivityPromptRequest(
|
new MeetingInactivityPromptRequest(
|
||||||
inactivityDuration,
|
inactivityDuration,
|
||||||
@@ -884,16 +1093,36 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
|
|
||||||
if (response == MeetingInactivityPromptResponse.Stop)
|
if (response == MeetingInactivityPromptResponse.Stop)
|
||||||
{
|
{
|
||||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
await StopAsync(
|
||||||
|
inferredEndTime,
|
||||||
|
run,
|
||||||
|
promptActivityVersion,
|
||||||
|
null,
|
||||||
|
CancellationToken.None);
|
||||||
|
}
|
||||||
|
else if (response == MeetingInactivityPromptResponse.Pause)
|
||||||
|
{
|
||||||
|
await SetTranscriptionPausedAsync(
|
||||||
|
true,
|
||||||
|
run,
|
||||||
|
promptActivityVersion,
|
||||||
|
CancellationToken.None);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
run.CaptureCancellation);
|
run.CaptureCancellation);
|
||||||
|
|
||||||
|
var currentActivity = run.GetTranscriptActivitySnapshot();
|
||||||
|
if (currentActivity.IsTranscriptionPaused ||
|
||||||
|
currentActivity.ActivityVersion != promptActivityVersion)
|
||||||
|
{
|
||||||
|
await DismissInactivityPromptsAsync(run.CaptureCancellation);
|
||||||
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
await inactivityClock.DelayAsync(
|
await inactivityClock.DelayAsync(
|
||||||
GetInactivitySafeguardCheckInterval(safeguardOptions),
|
checkInterval,
|
||||||
run.CaptureCancellation);
|
run.CaptureCancellation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -961,11 +1190,11 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
|
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
|
||||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
|
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
|
||||||
if (checkpoint is null)
|
if (checkpoint is null)
|
||||||
{
|
{
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
|
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1021,7 +1250,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (run.IsAborted)
|
if (run.IsAborted || run.HasAttachedPromptMetadata)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1029,25 +1258,21 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
await gate.WaitAsync(CancellationToken.None);
|
await gate.WaitAsync(CancellationToken.None);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (run.IsAborted)
|
if (run.IsAborted || run.HasAttachedPromptMetadata)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
|
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
|
||||||
await ApplyMeetingMetadataAsync(run.Artifacts, meetingNote, metadata, run.Options, CancellationToken.None);
|
run.Artifacts,
|
||||||
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
|
await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None),
|
||||||
|
metadata,
|
||||||
|
run.Options,
|
||||||
|
CancellationToken.None);
|
||||||
if (currentMeetingNote?.Path == meetingNote.Path)
|
if (currentMeetingNote?.Path == meetingNote.Path)
|
||||||
{
|
{
|
||||||
currentMeetingNote = meetingNote;
|
currentMeetingNote = meetingNote;
|
||||||
}
|
}
|
||||||
|
|
||||||
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
|
|
||||||
run.Artifacts,
|
|
||||||
meetingNote,
|
|
||||||
metadata.Agenda,
|
|
||||||
metadata.ScheduledEnd,
|
|
||||||
CancellationToken.None);
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Applied Outlook meeting metadata to {MeetingNotePath}",
|
"Applied Outlook meeting metadata to {MeetingNotePath}",
|
||||||
run.MeetingNotePath);
|
run.MeetingNotePath);
|
||||||
@@ -1119,6 +1344,32 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<MeetingNote> ApplyAndPersistMeetingMetadataAsync(
|
||||||
|
MeetingSessionArtifacts artifacts,
|
||||||
|
MeetingNote meetingNote,
|
||||||
|
MeetingMetadata metadata,
|
||||||
|
MeetingAssistantOptions options,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await ApplyMeetingMetadataAsync(
|
||||||
|
artifacts,
|
||||||
|
meetingNote,
|
||||||
|
metadata,
|
||||||
|
options,
|
||||||
|
cancellationToken);
|
||||||
|
var savedMeetingNote = await meetingNoteStore.SaveAsync(
|
||||||
|
meetingNote,
|
||||||
|
options,
|
||||||
|
cancellationToken);
|
||||||
|
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
|
||||||
|
artifacts,
|
||||||
|
savedMeetingNote,
|
||||||
|
metadata.Agenda,
|
||||||
|
metadata.ScheduledEnd,
|
||||||
|
cancellationToken);
|
||||||
|
return savedMeetingNote;
|
||||||
|
}
|
||||||
|
|
||||||
private async Task<List<string>> TransformAttendeesAsync(
|
private async Task<List<string>> TransformAttendeesAsync(
|
||||||
MeetingSessionArtifacts artifacts,
|
MeetingSessionArtifacts artifacts,
|
||||||
IReadOnlyList<string> attendees,
|
IReadOnlyList<string> attendees,
|
||||||
@@ -1771,6 +2022,11 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
.ToList();
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
|
||||||
|
{
|
||||||
|
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
|
||||||
|
}
|
||||||
|
|
||||||
private static TimeSpan GetInactivitySafeguardCheckInterval(
|
private static TimeSpan GetInactivitySafeguardCheckInterval(
|
||||||
RecordingInactivitySafeguardOptions options)
|
RecordingInactivitySafeguardOptions options)
|
||||||
{
|
{
|
||||||
@@ -1827,6 +2083,8 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
private TimeSpan? lastTranscriptSegmentEnd;
|
private TimeSpan? lastTranscriptSegmentEnd;
|
||||||
private long transcriptActivityVersion;
|
private long transcriptActivityVersion;
|
||||||
private DateTimeOffset? inferredEndTime;
|
private DateTimeOffset? inferredEndTime;
|
||||||
|
private bool isTranscriptionPaused;
|
||||||
|
private DateTimeOffset? transcriptionPausedAt;
|
||||||
|
|
||||||
public RecordingRun(
|
public RecordingRun(
|
||||||
CancellationTokenSource captureCancellation,
|
CancellationTokenSource captureCancellation,
|
||||||
@@ -1842,6 +2100,7 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
string launchProfileName,
|
string launchProfileName,
|
||||||
TimeSpan liveSampleBufferDuration,
|
TimeSpan liveSampleBufferDuration,
|
||||||
int maxSpeakerSamples,
|
int maxSpeakerSamples,
|
||||||
|
bool requireNonOverlappingSpeakerSamples,
|
||||||
ILogger logger)
|
ILogger logger)
|
||||||
{
|
{
|
||||||
CaptureCancellationSource = captureCancellation;
|
CaptureCancellationSource = captureCancellation;
|
||||||
@@ -1862,8 +2121,9 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
liveSampleBufferDuration,
|
liveSampleBufferDuration,
|
||||||
maxSpeakerSamples,
|
maxSpeakerSamples,
|
||||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||||
options.SpeakerIdentification.MaximumSampleSegmentGap,
|
options.SpeakerIdentification.MaximumSampleDuration,
|
||||||
logger);
|
logger,
|
||||||
|
requireNonOverlappingSpeakerSamples);
|
||||||
}
|
}
|
||||||
|
|
||||||
public CancellationTokenSource CaptureCancellationSource { get; }
|
public CancellationTokenSource CaptureCancellationSource { get; }
|
||||||
@@ -1908,6 +2168,19 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
|
|
||||||
public bool HasSwitchedProfile { get; private set; }
|
public bool HasSwitchedProfile { get; private set; }
|
||||||
|
|
||||||
|
public bool HasAttachedPromptMetadata { get; private set; }
|
||||||
|
|
||||||
|
public bool IsTranscriptionPaused
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (transcriptActivityGate)
|
||||||
|
{
|
||||||
|
return isTranscriptionPaused;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||||
|
|
||||||
public DateTimeOffset? InferredEndTime
|
public DateTimeOffset? InferredEndTime
|
||||||
@@ -1926,6 +2199,42 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
CaptureCancellationSource.Cancel();
|
CaptureCancellationSource.Cancel();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void MarkPromptMetadataAttached()
|
||||||
|
{
|
||||||
|
HasAttachedPromptMetadata = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool TrySetTranscriptionPaused(
|
||||||
|
bool isPaused,
|
||||||
|
DateTimeOffset changedAt,
|
||||||
|
long? expectedActivityVersion)
|
||||||
|
{
|
||||||
|
lock (transcriptActivityGate)
|
||||||
|
{
|
||||||
|
if (expectedActivityVersion is not null &&
|
||||||
|
transcriptActivityVersion != expectedActivityVersion.Value)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isTranscriptionPaused == isPaused)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
isTranscriptionPaused = isPaused;
|
||||||
|
transcriptionPausedAt = isPaused ? changedAt : null;
|
||||||
|
|
||||||
|
if (!isPaused)
|
||||||
|
{
|
||||||
|
lastTranscriptActivityAt = changedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
transcriptActivityVersion++;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void Abort()
|
public void Abort()
|
||||||
{
|
{
|
||||||
IsAborted = true;
|
IsAborted = true;
|
||||||
@@ -1954,13 +2263,17 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
|
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
await PipelineGate.WaitAsync(cancellationToken);
|
await PipelineGate.WaitAsync(cancellationToken);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
AppendAudio(chunk);
|
var routedChunk = IsTranscriptionPaused
|
||||||
await Pipeline.WriteAsync(chunk, cancellationToken);
|
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
|
||||||
|
: chunk;
|
||||||
|
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
|
||||||
|
AppendAudio(routedChunk);
|
||||||
|
await Pipeline.WriteAsync(routedChunk, cancellationToken);
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
@@ -2144,7 +2457,9 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return new TranscriptActivitySnapshot(
|
return new TranscriptActivitySnapshot(
|
||||||
lastTranscriptActivityAt,
|
lastTranscriptActivityAt,
|
||||||
lastTranscriptSegmentEnd,
|
lastTranscriptSegmentEnd,
|
||||||
transcriptActivityVersion);
|
transcriptActivityVersion,
|
||||||
|
isTranscriptionPaused,
|
||||||
|
transcriptionPausedAt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2164,6 +2479,35 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
|
||||||
|
{
|
||||||
|
lock (transcriptActivityGate)
|
||||||
|
{
|
||||||
|
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
inferredEndTime = endTime;
|
||||||
|
StopCapture();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
|
||||||
|
{
|
||||||
|
lock (transcriptActivityGate)
|
||||||
|
{
|
||||||
|
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
StopCapture();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void AppendAudio(AudioChunk chunk)
|
public void AppendAudio(AudioChunk chunk)
|
||||||
{
|
{
|
||||||
speakerSampleCollector.AppendAudio(chunk);
|
speakerSampleCollector.AppendAudio(chunk);
|
||||||
@@ -2187,9 +2531,11 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
return speakerSampleCollector.Snapshot();
|
return speakerSampleCollector.Snapshot();
|
||||||
}
|
}
|
||||||
|
|
||||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
|
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
|
||||||
|
MeetingNote meetingNote,
|
||||||
|
IReadOnlyList<SpeakerAudioSample> samples)
|
||||||
{
|
{
|
||||||
var speakers = GetUnmappedSampleSpeakers();
|
var speakers = GetUnmappedSampleSpeakers(samples);
|
||||||
if (speakers.Count == 0)
|
if (speakers.Count == 0)
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
@@ -2197,7 +2543,11 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
|
|
||||||
var checkpoint = new LiveIdentificationCheckpoint(
|
var checkpoint = new LiveIdentificationCheckpoint(
|
||||||
speakers,
|
speakers,
|
||||||
BuildAttendeeSignature(meetingNote));
|
BuildAttendeeSignature(meetingNote),
|
||||||
|
Options.SpeakerIdentification.Resemblyzer.Enabled
|
||||||
|
? speakers.Select(speaker => samples.Count(sample =>
|
||||||
|
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
|
||||||
|
: []);
|
||||||
lock (liveIdentificationGate)
|
lock (liveIdentificationGate)
|
||||||
{
|
{
|
||||||
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
|
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
|
||||||
@@ -2255,9 +2605,9 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
StringComparer.OrdinalIgnoreCase);
|
StringComparer.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
|
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
|
||||||
{
|
{
|
||||||
return GetSpeakerSamplesSnapshot()
|
return samples
|
||||||
.Select(sample => sample.Speaker)
|
.Select(sample => sample.Speaker)
|
||||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||||
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
|
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
|
||||||
@@ -2293,7 +2643,9 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
public sealed record TranscriptActivitySnapshot(
|
public sealed record TranscriptActivitySnapshot(
|
||||||
DateTimeOffset LastTranscriptActivityAt,
|
DateTimeOffset LastTranscriptActivityAt,
|
||||||
TimeSpan? LastTranscriptSegmentEnd,
|
TimeSpan? LastTranscriptSegmentEnd,
|
||||||
long ActivityVersion);
|
long ActivityVersion,
|
||||||
|
bool IsTranscriptionPaused,
|
||||||
|
DateTimeOffset? TranscriptionPausedAt);
|
||||||
|
|
||||||
private static int StateRank(AssistantContextState state)
|
private static int StateRank(AssistantContextState state)
|
||||||
{
|
{
|
||||||
@@ -2311,11 +2663,13 @@ public sealed class MeetingRecordingCoordinator
|
|||||||
|
|
||||||
public sealed record LiveIdentificationCheckpoint(
|
public sealed record LiveIdentificationCheckpoint(
|
||||||
IReadOnlyList<string> Speakers,
|
IReadOnlyList<string> Speakers,
|
||||||
string AttendeeSignature)
|
string AttendeeSignature,
|
||||||
|
IReadOnlyList<int> SampleCounts)
|
||||||
{
|
{
|
||||||
public bool Matches(LiveIdentificationCheckpoint other)
|
public bool Matches(LiveIdentificationCheckpoint other)
|
||||||
{
|
{
|
||||||
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
|
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
|
||||||
|
SampleCounts.SequenceEqual(other.SampleCounts) &&
|
||||||
Speakers.Count == other.Speakers.Count &&
|
Speakers.Count == other.Speakers.Count &&
|
||||||
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
|
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
@@ -2348,7 +2702,8 @@ public sealed record RecordingStatus(
|
|||||||
string? AssistantContextPath,
|
string? AssistantContextPath,
|
||||||
string? SummaryPath,
|
string? SummaryPath,
|
||||||
RecordingProcessState State = RecordingProcessState.Idle,
|
RecordingProcessState State = RecordingProcessState.Idle,
|
||||||
string? LaunchProfile = null);
|
string? LaunchProfile = null,
|
||||||
|
bool IsPaused = false);
|
||||||
|
|
||||||
public enum RecordingProcessState
|
public enum RecordingProcessState
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||||
|
{
|
||||||
|
private static readonly TimeSpan DefaultRecoveryDelay = TimeSpan.FromSeconds(1);
|
||||||
|
private readonly IMicrophoneCaptureSourceFactory captureSources;
|
||||||
|
private readonly ILogger<MicrophoneAudioSource> logger;
|
||||||
|
private readonly TimeSpan recoveryDelay;
|
||||||
|
private readonly MicrophoneSwitchMonitor? switchMonitor;
|
||||||
|
private readonly MicrophoneDeviceSelection? selection;
|
||||||
|
|
||||||
|
public MicrophoneAudioSource(
|
||||||
|
IMicrophoneCaptureSourceFactory captureSources,
|
||||||
|
ILogger<MicrophoneAudioSource> logger,
|
||||||
|
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||||
|
MicrophoneDeviceSelection? selection = null)
|
||||||
|
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
internal MicrophoneAudioSource(
|
||||||
|
IMicrophoneCaptureSourceFactory captureSources,
|
||||||
|
ILogger<MicrophoneAudioSource> logger,
|
||||||
|
TimeSpan recoveryDelay,
|
||||||
|
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||||
|
MicrophoneDeviceSelection? selection = null)
|
||||||
|
{
|
||||||
|
this.captureSources = captureSources;
|
||||||
|
this.logger = logger;
|
||||||
|
this.recoveryDelay = recoveryDelay;
|
||||||
|
this.switchMonitor = switchMonitor;
|
||||||
|
this.selection = selection;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||||
|
MeetingAssistantOptions options,
|
||||||
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
using var session = new CaptureSession(selection, cancellationToken);
|
||||||
|
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
|
||||||
|
{
|
||||||
|
yield return chunk;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
session.Stop();
|
||||||
|
await monitoring;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
|
||||||
|
MeetingAssistantOptions options,
|
||||||
|
CaptureSession session,
|
||||||
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var failedAttempts = 0;
|
||||||
|
while (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
var captureToken = session.BeginCapture();
|
||||||
|
IMeetingAudioSource? captureSource = null;
|
||||||
|
IAsyncEnumerator<AudioChunk>? capture = null;
|
||||||
|
Exception? failure = null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
captureSource = captureSources.CreateCapture(options);
|
||||||
|
capture = captureSource
|
||||||
|
.CaptureAsync(options, captureToken)
|
||||||
|
.GetAsyncEnumerator(captureToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
failure = exception;
|
||||||
|
}
|
||||||
|
|
||||||
|
using var captureOwner = captureSource as IDisposable;
|
||||||
|
if (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (capture is not null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (!captureToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
var hasNext = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
hasNext = await capture.MoveNextAsync();
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
failure = exception;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failedAttempts > 0)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Microphone capture recovered after {FailedAttemptCount} failed attempt(s)",
|
||||||
|
failedAttempts);
|
||||||
|
failedAttempts = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
yield return capture.Current;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await capture.DisposeAsync();
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
failure ??= exception;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var switchRequested = captureToken.IsCancellationRequested;
|
||||||
|
session.EndCapture();
|
||||||
|
if (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (switchRequested)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
failedAttempts++;
|
||||||
|
logger.LogWarning(
|
||||||
|
failure,
|
||||||
|
"Microphone capture stopped unexpectedly; re-resolving an available microphone in {RecoveryDelay}",
|
||||||
|
recoveryDelay);
|
||||||
|
|
||||||
|
if (!await WaitForRecoveryAsync(cancellationToken))
|
||||||
|
{
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<bool> WaitForRecoveryAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(recoveryDelay, cancellationToken);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
|
||||||
|
{
|
||||||
|
private readonly object gate = new();
|
||||||
|
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
private CancellationTokenSource? capture;
|
||||||
|
|
||||||
|
public CancellationToken Token => lifetime.Token;
|
||||||
|
|
||||||
|
public CancellationToken BeginCapture()
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
|
||||||
|
return capture.Token;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void EndCapture()
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
capture?.Dispose();
|
||||||
|
capture = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
|
||||||
|
selection.Select(device.Id);
|
||||||
|
capture?.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Stop()
|
||||||
|
{
|
||||||
|
lock (gate) lifetime.Cancel();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
Stop();
|
||||||
|
EndCapture();
|
||||||
|
lifetime.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,8 @@ public sealed class MicrophoneDeviceSelection
|
|||||||
var selected = SelectedDeviceId;
|
var selected = SelectedDeviceId;
|
||||||
return FindById(availableDevices, selected) ??
|
return FindById(availableDevices, selected) ??
|
||||||
FindById(availableDevices, configuredDeviceId) ??
|
FindById(availableDevices, configuredDeviceId) ??
|
||||||
|
FindById(availableDevices, defaultDevice?.Id) ??
|
||||||
|
availableDevices.FirstOrDefault() ??
|
||||||
defaultDevice;
|
defaultDevice;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
public sealed class MicrophoneSwitchMonitor
|
||||||
|
{
|
||||||
|
private readonly IMicrophoneDeviceProvider devices;
|
||||||
|
private readonly IMicrophoneSwitchPromptService prompts;
|
||||||
|
private readonly ILogger<MicrophoneSwitchMonitor> logger;
|
||||||
|
private readonly TimeSpan pollInterval;
|
||||||
|
private readonly TimeSpan promptLifetime;
|
||||||
|
|
||||||
|
public MicrophoneSwitchMonitor(
|
||||||
|
IMicrophoneDeviceProvider devices,
|
||||||
|
IMicrophoneSwitchPromptService prompts,
|
||||||
|
ILogger<MicrophoneSwitchMonitor> logger)
|
||||||
|
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
internal MicrophoneSwitchMonitor(
|
||||||
|
IMicrophoneDeviceProvider devices,
|
||||||
|
IMicrophoneSwitchPromptService prompts,
|
||||||
|
ILogger<MicrophoneSwitchMonitor> logger,
|
||||||
|
TimeSpan pollInterval,
|
||||||
|
TimeSpan promptLifetime)
|
||||||
|
{
|
||||||
|
this.devices = devices;
|
||||||
|
this.prompts = prompts;
|
||||||
|
this.logger = logger;
|
||||||
|
this.pollInterval = pollInterval;
|
||||||
|
this.promptLifetime = promptLifetime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
await Task.Delay(pollInterval, cancellationToken);
|
||||||
|
var available = ReadDevices();
|
||||||
|
if (available is null) continue;
|
||||||
|
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
foreach (var (id, prompt) in pending.ToArray())
|
||||||
|
{
|
||||||
|
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
|
||||||
|
if (prompt.Task.IsCompleted)
|
||||||
|
{
|
||||||
|
prompt.Cancellation.Dispose();
|
||||||
|
pending.Remove(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var device in available)
|
||||||
|
{
|
||||||
|
if (known.Add(device.Id))
|
||||||
|
{
|
||||||
|
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
promptCancellation.CancelAfter(promptLifetime);
|
||||||
|
pending.Add(device.Id, new PendingPrompt(promptCancellation,
|
||||||
|
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
|
||||||
|
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
|
||||||
|
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
|
||||||
|
&& !cancellationToken.IsCancellationRequested
|
||||||
|
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
|
||||||
|
&& !cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
switchMicrophone(device, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return devices.GetAvailableMicrophones();
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Could not check for newly available microphones");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
|
||||||
|
}
|
||||||
@@ -4,44 +4,34 @@ using NAudio.Wave;
|
|||||||
|
|
||||||
namespace MeetingAssistant.Recording;
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
|
||||||
{
|
{
|
||||||
private readonly IMicrophoneDeviceProvider microphones;
|
private readonly IWaveIn capture;
|
||||||
private readonly ILogger<MicrophoneAudioSource> logger;
|
private readonly string sourceName;
|
||||||
|
private readonly ILogger logger;
|
||||||
|
private int disposed;
|
||||||
|
|
||||||
public MicrophoneAudioSource(
|
public NaudioCaptureAudioSource(
|
||||||
IMicrophoneDeviceProvider microphones,
|
IWaveIn capture,
|
||||||
ILogger<MicrophoneAudioSource> logger)
|
string sourceName,
|
||||||
|
ILogger logger)
|
||||||
{
|
{
|
||||||
this.microphones = microphones;
|
this.capture = capture;
|
||||||
|
this.sourceName = sourceName;
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
return CaptureWith(cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(
|
public void Dispose()
|
||||||
MeetingAssistantOptions options,
|
|
||||||
CancellationToken cancellationToken)
|
|
||||||
{
|
{
|
||||||
return CaptureAsync(microphones.CreateCapture(options), options, cancellationToken);
|
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
private IAsyncEnumerable<AudioChunk> CaptureAsync(
|
private async IAsyncEnumerable<AudioChunk> CaptureWith(
|
||||||
IWaveIn capture,
|
|
||||||
MeetingAssistantOptions options,
|
|
||||||
CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
|
|
||||||
return CaptureWith(capture, "microphone", logger, cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static async IAsyncEnumerable<AudioChunk> CaptureWith(
|
|
||||||
IWaveIn capture,
|
|
||||||
string sourceName,
|
|
||||||
ILogger logger,
|
|
||||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var channel = Channel.CreateUnbounded<AudioChunk>();
|
var channel = Channel.CreateUnbounded<AudioChunk>();
|
||||||
@@ -96,7 +86,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
|||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
capture.Dispose();
|
Dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -124,6 +114,6 @@ public sealed class SystemAudioSource : IMeetingAudioSource
|
|||||||
WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels)
|
WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels)
|
||||||
};
|
};
|
||||||
|
|
||||||
return MicrophoneAudioSource.CaptureWith(capture, "system", logger, cancellationToken);
|
return new NaudioCaptureAudioSource(capture, "system", logger).CaptureAsync(cancellationToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,8 +117,25 @@ public sealed class OfflineTranscriptionBacklogProcessor
|
|||||||
pipelineOptions,
|
pipelineOptions,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
|
if (!finishedSegments.Any(segment =>
|
||||||
|
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
|
||||||
|
}
|
||||||
|
|
||||||
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
|
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
|
||||||
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
|
await transcriptStore.AppendLinesAsync(session,
|
||||||
|
[
|
||||||
|
"",
|
||||||
|
"## Offline transcription recovery",
|
||||||
|
"",
|
||||||
|
"> The following transcript was recovered by replaying the entire recording. "
|
||||||
|
+ "It may duplicate earlier transcript passages. "
|
||||||
|
+ "Treat repeated passages as the same discussion when summarizing.",
|
||||||
|
"",
|
||||||
|
.. lines
|
||||||
|
], cancellationToken);
|
||||||
|
|
||||||
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
|
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
|
||||||
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
|
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
|
||||||
|
|||||||
@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
private readonly object gate = new();
|
private readonly object gate = new();
|
||||||
private readonly RollingAudioBuffer audioBuffer;
|
private readonly RollingAudioBuffer audioBuffer;
|
||||||
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||||
private readonly int maxSamplesPerSpeaker;
|
private readonly int maxSamplesPerSpeaker;
|
||||||
private readonly TimeSpan minimumUninterruptedSpeechDuration;
|
private readonly TimeSpan minimumSampleSpeechDuration;
|
||||||
private readonly TimeSpan maximumSegmentGap;
|
private readonly TimeSpan maximumSampleDuration;
|
||||||
|
private readonly bool requireNonOverlappingSamples;
|
||||||
private readonly ILogger? logger;
|
private readonly ILogger? logger;
|
||||||
private PendingSpeakerSpan? pendingSpan;
|
private SpeakerSampleSpan? pendingSpan;
|
||||||
|
|
||||||
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
|
|
||||||
: this(
|
|
||||||
bufferDuration,
|
|
||||||
maxSamplesPerSpeaker,
|
|
||||||
TimeSpan.FromSeconds(30),
|
|
||||||
TimeSpan.FromSeconds(1),
|
|
||||||
logger: null)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
public SpeakerAudioSampleCollector(
|
public SpeakerAudioSampleCollector(
|
||||||
TimeSpan bufferDuration,
|
TimeSpan bufferDuration,
|
||||||
int maxSamplesPerSpeaker,
|
int maxSamplesPerSpeaker,
|
||||||
TimeSpan minimumUninterruptedSpeechDuration,
|
TimeSpan minimumSampleSpeechDuration,
|
||||||
TimeSpan maximumSegmentGap,
|
TimeSpan maximumSampleDuration,
|
||||||
ILogger? logger = null)
|
ILogger? logger = null,
|
||||||
|
bool requireNonOverlappingSamples = false)
|
||||||
{
|
{
|
||||||
|
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||||
|
minimumSampleSpeechDuration,
|
||||||
|
maximumSampleDuration,
|
||||||
|
"Speaker sample collector configuration is invalid");
|
||||||
audioBuffer = new RollingAudioBuffer(bufferDuration);
|
audioBuffer = new RollingAudioBuffer(bufferDuration);
|
||||||
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
|
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
|
||||||
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
|
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
|
||||||
? minimumUninterruptedSpeechDuration
|
this.maximumSampleDuration = maximumSampleDuration;
|
||||||
: TimeSpan.Zero;
|
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
|
||||||
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
|
|
||||||
? maximumSegmentGap
|
|
||||||
: TimeSpan.Zero;
|
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
lock (gate)
|
lock (gate)
|
||||||
{
|
{
|
||||||
samplesBySpeaker.Clear();
|
samplesBySpeaker.Clear();
|
||||||
|
lastAcceptedEndBySpeaker.Clear();
|
||||||
pendingSpan = null;
|
pendingSpan = null;
|
||||||
audioBuffer.Reset();
|
audioBuffer.Reset();
|
||||||
}
|
}
|
||||||
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
TranscriptionSegment sampleSegment;
|
SpeakerSampleSpan? sampleSpan;
|
||||||
PendingSpanReset? reset;
|
SpeakerSampleSpan? previousSpan;
|
||||||
lock (gate)
|
lock (gate)
|
||||||
{
|
{
|
||||||
(sampleSegment, reset) = ExtendPendingSpan(segment);
|
if (requireNonOverlappingSamples &&
|
||||||
|
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
|
||||||
|
segment.Start < lastAcceptedEnd)
|
||||||
|
{
|
||||||
|
logger?.LogInformation(
|
||||||
|
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
|
||||||
|
segment.Speaker,
|
||||||
|
lastAcceptedEnd);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (reset is not null)
|
if (sampleSpan is null)
|
||||||
{
|
{
|
||||||
logger?.LogInformation(
|
logger?.LogInformation(
|
||||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
|
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
|
||||||
reset.PreviousSpeaker,
|
segment.Speaker);
|
||||||
segment.Speaker,
|
return null;
|
||||||
reset.PreviousEnd,
|
|
||||||
segment.Start,
|
|
||||||
reset.Gap,
|
|
||||||
maximumSegmentGap);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
|
var sampleSegment = sampleSpan.ToSegment();
|
||||||
|
|
||||||
|
if (previousSpan is not null)
|
||||||
|
{
|
||||||
|
logger?.LogInformation(
|
||||||
|
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
|
||||||
|
previousSpan.Speaker,
|
||||||
|
segment.Speaker,
|
||||||
|
previousSpan.End,
|
||||||
|
segment.Start);
|
||||||
|
}
|
||||||
|
|
||||||
|
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
|
||||||
if (!score.Accepted)
|
if (!score.Accepted)
|
||||||
{
|
{
|
||||||
logger?.LogInformation(
|
logger?.LogInformation(
|
||||||
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||||
sampleSegment.Speaker,
|
sampleSegment.Speaker,
|
||||||
score.Reason,
|
score.Reason,
|
||||||
|
sampleSpan.SpeechDuration,
|
||||||
sampleSegment.End - sampleSegment.Start,
|
sampleSegment.End - sampleSegment.Start,
|
||||||
minimumUninterruptedSpeechDuration,
|
minimumSampleSpeechDuration,
|
||||||
score.WordCount);
|
score.WordCount);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
|
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
|
||||||
lock (gate)
|
lock (gate)
|
||||||
{
|
{
|
||||||
|
if (requireNonOverlappingSamples &&
|
||||||
|
ReferenceEquals(pendingSpan, sampleSpan))
|
||||||
|
{
|
||||||
|
pendingSpan = null;
|
||||||
|
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
|
||||||
|
}
|
||||||
|
|
||||||
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
|
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
|
||||||
{
|
{
|
||||||
samples = [];
|
samples = [];
|
||||||
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
|
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
|
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
|
||||||
{
|
{
|
||||||
if (pendingSpan is null ||
|
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
|
||||||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
|
|
||||||
{
|
{
|
||||||
var reset = pendingSpan is null
|
pendingSpan = extended;
|
||||||
? null
|
return (pendingSpan, null);
|
||||||
: new PendingSpanReset(
|
|
||||||
pendingSpan.Speaker,
|
|
||||||
pendingSpan.End,
|
|
||||||
segment.Start - pendingSpan.End);
|
|
||||||
pendingSpan = new PendingSpeakerSpan(
|
|
||||||
segment.Speaker,
|
|
||||||
segment.Start,
|
|
||||||
segment.End,
|
|
||||||
[segment.Text]);
|
|
||||||
return (pendingSpan.ToSegment(), reset);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pendingSpan = pendingSpan.Extend(segment);
|
var previousSpan = pendingSpan;
|
||||||
return (pendingSpan.ToSegment(), null);
|
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||||
|
return (pendingSpan, previousSpan);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static SampleScore Score(
|
private static SampleScore Score(
|
||||||
TranscriptionSegment segment,
|
TranscriptionSegment segment,
|
||||||
TimeSpan minimumUninterruptedSpeechDuration)
|
TimeSpan speechDuration,
|
||||||
|
TimeSpan minimumSampleSpeechDuration)
|
||||||
{
|
{
|
||||||
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
|
var durationSeconds = speechDuration.TotalSeconds;
|
||||||
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
|
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
|
||||||
{
|
{
|
||||||
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
|
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
|
||||||
}
|
}
|
||||||
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
|
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
|
||||||
}
|
}
|
||||||
|
|
||||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
|
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
|
||||||
var wordScore = Math.Min(words / 60.0, 1);
|
var wordScore = Math.Min(words / 60.0, 1);
|
||||||
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
|
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
|
||||||
segment.Text.TrimEnd().EndsWith('?') ||
|
segment.Text.TrimEnd().EndsWith('?') ||
|
||||||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
|
|||||||
|
|
||||||
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
|
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
|
||||||
|
|
||||||
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
|
|
||||||
|
|
||||||
private sealed record PendingSpeakerSpan(
|
|
||||||
string Speaker,
|
|
||||||
TimeSpan Start,
|
|
||||||
TimeSpan End,
|
|
||||||
IReadOnlyList<string> TextParts)
|
|
||||||
{
|
|
||||||
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
|
|
||||||
{
|
|
||||||
return this with
|
|
||||||
{
|
|
||||||
End = segment.End > End ? segment.End : End,
|
|
||||||
TextParts = TextParts.Append(segment.Text).ToList()
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
public TranscriptionSegment ToSegment()
|
|
||||||
{
|
|
||||||
return new TranscriptionSegment(
|
|
||||||
Start,
|
|
||||||
End,
|
|
||||||
Speaker,
|
|
||||||
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,25 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
|||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task AppendLinesAsync(
|
||||||
|
TranscriptSession session,
|
||||||
|
IReadOnlyList<string> lines,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (lines.Count == 0)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
return WithFileGateAsync(
|
||||||
|
session.TranscriptPath,
|
||||||
|
() => File.AppendAllTextAsync(
|
||||||
|
session.TranscriptPath,
|
||||||
|
string.Concat(lines.Select(line => line + Environment.NewLine)),
|
||||||
|
cancellationToken),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task ReplaceLineAsync(
|
public async Task ReplaceLineAsync(
|
||||||
TranscriptSession session,
|
TranscriptSession session,
|
||||||
TranscriptLineReference lineReference,
|
TranscriptLineReference lineReference,
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
private const string NotificationGroup = "meeting-inactivity";
|
private const string NotificationGroup = "meeting-inactivity";
|
||||||
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
|
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
|
||||||
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
|
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
|
||||||
private readonly object registrationGate = new();
|
private readonly object notificationGate = new();
|
||||||
private bool registered;
|
private bool registered;
|
||||||
|
|
||||||
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
|
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
|
||||||
@@ -37,24 +37,27 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
EnsureRegistered();
|
lock (notificationGate)
|
||||||
var promptId = Guid.NewGuid().ToString("N");
|
|
||||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
|
||||||
logger.LogInformation(
|
|
||||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
|
||||||
promptId,
|
|
||||||
request.Threshold);
|
|
||||||
var notification = BuildNotification(promptId, request);
|
|
||||||
notification.Show(toast =>
|
|
||||||
{
|
{
|
||||||
toast.Group = NotificationGroup;
|
EnsureRegistered();
|
||||||
toast.Tag = promptId;
|
var promptId = Guid.NewGuid().ToString("N");
|
||||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||||
});
|
logger.LogInformation(
|
||||||
logger.LogInformation(
|
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
promptId,
|
||||||
promptId,
|
request.Threshold);
|
||||||
request.Threshold);
|
var notification = BuildNotification(promptId, request);
|
||||||
|
notification.Show(toast =>
|
||||||
|
{
|
||||||
|
toast.Group = NotificationGroup;
|
||||||
|
toast.Tag = promptId;
|
||||||
|
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||||
|
});
|
||||||
|
logger.LogInformation(
|
||||||
|
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||||
|
promptId,
|
||||||
|
request.Threshold);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (Exception exception)
|
catch (Exception exception)
|
||||||
{
|
{
|
||||||
@@ -76,6 +79,36 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
lock (notificationGate)
|
||||||
|
{
|
||||||
|
if (pendingPrompts.IsEmpty)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
pendingPrompts.Clear();
|
||||||
|
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
|
||||||
|
logger.LogInformation("Dismissed all native Windows inactivity notifications");
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
|
||||||
|
}
|
||||||
|
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public Task StopAsync(CancellationToken cancellationToken)
|
public Task StopAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
@@ -83,29 +116,28 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
if (!registered)
|
lock (notificationGate)
|
||||||
{
|
{
|
||||||
return;
|
if (!registered)
|
||||||
}
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||||
}
|
registered = false;
|
||||||
catch (Exception exception)
|
}
|
||||||
{
|
catch (Exception exception)
|
||||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
{
|
||||||
|
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnsureRegistered()
|
private void EnsureRegistered()
|
||||||
{
|
{
|
||||||
if (registered)
|
lock (notificationGate)
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
lock (registrationGate)
|
|
||||||
{
|
{
|
||||||
if (registered)
|
if (registered)
|
||||||
{
|
{
|
||||||
@@ -122,29 +154,25 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
string promptId,
|
string promptId,
|
||||||
MeetingInactivityPromptRequest request)
|
MeetingInactivityPromptRequest request)
|
||||||
{
|
{
|
||||||
var yesButton = new ToastButton()
|
var builder = new ToastContentBuilder()
|
||||||
.SetContent("Yes")
|
|
||||||
.AddArgument("source", NotificationSource)
|
|
||||||
.AddArgument("promptId", promptId)
|
|
||||||
.AddArgument("response", "stop")
|
|
||||||
.SetBackgroundActivation();
|
|
||||||
|
|
||||||
var noButton = new ToastButton()
|
|
||||||
.SetContent("No")
|
|
||||||
.AddArgument("source", NotificationSource)
|
|
||||||
.AddArgument("promptId", promptId)
|
|
||||||
.AddArgument("response", "continue")
|
|
||||||
.SetBackgroundActivation();
|
|
||||||
|
|
||||||
return new ToastContentBuilder()
|
|
||||||
.AddArgument("source", NotificationSource)
|
.AddArgument("source", NotificationSource)
|
||||||
.AddArgument("promptId", promptId)
|
.AddArgument("promptId", promptId)
|
||||||
.SetToastScenario(ToastScenario.Reminder)
|
.SetToastScenario(ToastScenario.Reminder)
|
||||||
.SetToastDuration(ToastDuration.Long)
|
.SetToastDuration(ToastDuration.Long)
|
||||||
.AddText("Stop meeting?")
|
.AddText("Stop meeting?")
|
||||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
|
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
|
||||||
.AddButton(yesButton)
|
|
||||||
.AddButton(noButton);
|
foreach (var action in MeetingInactivityPromptActions.All)
|
||||||
|
{
|
||||||
|
builder.AddButton(new ToastButton()
|
||||||
|
.SetContent(action.Content)
|
||||||
|
.AddArgument("source", NotificationSource)
|
||||||
|
.AddArgument("promptId", promptId)
|
||||||
|
.AddArgument("response", action.ResponseArgument)
|
||||||
|
.SetBackgroundActivation());
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||||
@@ -177,10 +205,9 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||||
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
|
? MeetingInactivityPromptActions.ParseResponse(responseValue)
|
||||||
? MeetingInactivityPromptResponse.Stop
|
: MeetingInactivityPromptResponse.Continue;
|
||||||
: MeetingInactivityPromptResponse.Continue;
|
|
||||||
_ = Task.Run(async () =>
|
_ = Task.Run(async () =>
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using NAudio.Wave;
|
|||||||
|
|
||||||
namespace MeetingAssistant.Recording;
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider
|
public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||||
{
|
{
|
||||||
private readonly MicrophoneDeviceSelection selection;
|
private readonly MicrophoneDeviceSelection selection;
|
||||||
private readonly ILogger<WindowsMicrophoneDeviceProvider> logger;
|
private readonly ILogger<WindowsMicrophoneDeviceProvider> logger;
|
||||||
@@ -42,21 +42,38 @@ public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider
|
|||||||
selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices));
|
selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices));
|
||||||
}
|
}
|
||||||
|
|
||||||
public IWaveIn CreateCapture(MeetingAssistantOptions options)
|
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||||
{
|
{
|
||||||
var current = GetMicrophoneSnapshot(options).Current;
|
var current = GetMicrophoneSnapshot(options).Current;
|
||||||
|
IWaveIn capture;
|
||||||
if (current is null)
|
if (current is null)
|
||||||
{
|
{
|
||||||
logger.LogInformation("Starting microphone capture from Windows default capture endpoint");
|
logger.LogInformation("Starting microphone capture from Windows default capture endpoint");
|
||||||
return new WasapiCapture();
|
capture = new WasapiCapture();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})",
|
||||||
|
current.Name,
|
||||||
|
current.Id);
|
||||||
|
using var enumerator = new MMDeviceEnumerator();
|
||||||
|
capture = new WasapiCapture(enumerator.GetDevice(current.Id));
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.LogInformation(
|
try
|
||||||
"Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})",
|
{
|
||||||
current.Name,
|
capture.WaveFormat = new WaveFormat(
|
||||||
current.Id);
|
options.Recording.SampleRate,
|
||||||
using var enumerator = new MMDeviceEnumerator();
|
16,
|
||||||
return new WasapiCapture(enumerator.GetDevice(current.Id));
|
options.Recording.Channels);
|
||||||
|
return new NaudioCaptureAudioSource(capture, "microphone", logger);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
capture.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static MicrophoneDevice? GetDefaultMicrophone()
|
private static MicrophoneDevice? GetDefaultMicrophone()
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#if WINDOWS
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using CommunityToolkit.WinUI.Notifications;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
|
||||||
|
{
|
||||||
|
private const string Source = "meeting-microphone-switch";
|
||||||
|
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||||
|
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
|
||||||
|
private readonly object gate = new();
|
||||||
|
private bool registered;
|
||||||
|
|
||||||
|
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
|
||||||
|
{
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
var id = Guid.NewGuid().ToString("N");
|
||||||
|
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
if (!registered)
|
||||||
|
{
|
||||||
|
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||||
|
registered = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
pending[id] = response;
|
||||||
|
new ToastContentBuilder()
|
||||||
|
.AddArgument("source", Source)
|
||||||
|
.AddArgument("promptId", id)
|
||||||
|
.SetToastScenario(ToastScenario.Reminder)
|
||||||
|
.SetToastDuration(ToastDuration.Long)
|
||||||
|
.AddText("New microphone available")
|
||||||
|
.AddText($"Switch this meeting to {microphone.Name}?")
|
||||||
|
.AddButton(new ToastButton().SetContent("Yes")
|
||||||
|
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||||
|
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||||
|
.AddButton(new ToastButton().SetContent("No")
|
||||||
|
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||||
|
.AddArgument("response", "no").SetBackgroundActivation())
|
||||||
|
.Show(toast =>
|
||||||
|
{
|
||||||
|
toast.Tag = id;
|
||||||
|
toast.Group = Source;
|
||||||
|
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
|
||||||
|
return await response.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
pending.TryRemove(id, out _);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ToastNotificationManagerCompat.History.Remove(id, Source);
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||||
|
{
|
||||||
|
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||||
|
if (arguments.GetValueOrDefault("source") != Source ||
|
||||||
|
!arguments.TryGetValue("promptId", out var id) ||
|
||||||
|
!pending.TryRemove(id, out var response)) return;
|
||||||
|
|
||||||
|
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||||
|
registered = false;
|
||||||
|
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||||
|
pending.Clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
@@ -1,15 +1,13 @@
|
|||||||
using System.Net.Http.Headers;
|
|
||||||
using System.Text;
|
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Nodes;
|
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using MeetingAssistant.MeetingNotes;
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Summary;
|
||||||
|
using Microsoft.Extensions.AI;
|
||||||
|
|
||||||
namespace MeetingAssistant.Screenshots;
|
namespace MeetingAssistant.Screenshots;
|
||||||
|
|
||||||
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||||
{
|
{
|
||||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
|
||||||
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
|
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
|
||||||
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
|
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
|
||||||
|
|
||||||
@@ -40,20 +38,30 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
|||||||
: options.Agent.Model;
|
: options.Agent.Model;
|
||||||
var key = ResolveApiKey(options);
|
var key = ResolveApiKey(options);
|
||||||
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
|
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
|
||||||
using var httpClient = CreateHttpClient();
|
var httpClient = CreateHttpClient();
|
||||||
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
|
httpClient.BaseAddress = LiteLlmResponsesChatClient.NormalizeEndpoint(new Uri(endpoint));
|
||||||
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
|
using var chatClient = new LiteLlmResponsesChatClient(
|
||||||
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
|
httpClient,
|
||||||
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
|
key,
|
||||||
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
|
model,
|
||||||
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
|
enableThinking: false,
|
||||||
if (!response.IsSuccessStatusCode)
|
reasoningEffort: "none",
|
||||||
{
|
reconnectionAttempts: options.Agent.ReconnectionAttempts,
|
||||||
throw new InvalidOperationException(
|
reconnectionDelay: options.Agent.ReconnectionDelay,
|
||||||
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
logger: logger,
|
||||||
}
|
firstRequestIsUser: false,
|
||||||
|
useStreaming: options.Agent.UseStreaming);
|
||||||
var text = ParseOutputText(responseJson);
|
var response = await chatClient.GetResponseAsync(
|
||||||
|
[
|
||||||
|
new ChatMessage(
|
||||||
|
ChatRole.User,
|
||||||
|
[
|
||||||
|
new TextContent(CreatePrompt(prompt, imageBytes)),
|
||||||
|
new DataContent(imageBytes, "image/png")
|
||||||
|
])
|
||||||
|
],
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
var text = response.Text ?? string.Empty;
|
||||||
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
|
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
|
||||||
return ParseOcrResult(text);
|
return ParseOcrResult(text);
|
||||||
}
|
}
|
||||||
@@ -65,35 +73,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
|||||||
: new HttpClient(httpMessageHandlerFactory());
|
: new HttpClient(httpMessageHandlerFactory());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
|
|
||||||
{
|
|
||||||
return new JsonObject
|
|
||||||
{
|
|
||||||
["model"] = model,
|
|
||||||
["store"] = false,
|
|
||||||
["input"] = new JsonArray
|
|
||||||
{
|
|
||||||
new JsonObject
|
|
||||||
{
|
|
||||||
["role"] = "user",
|
|
||||||
["content"] = new JsonArray
|
|
||||||
{
|
|
||||||
new JsonObject
|
|
||||||
{
|
|
||||||
["type"] = "input_text",
|
|
||||||
["text"] = prompt
|
|
||||||
},
|
|
||||||
new JsonObject
|
|
||||||
{
|
|
||||||
["type"] = "input_image",
|
|
||||||
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string CreatePrompt(string prompt, byte[] imageBytes)
|
private static string CreatePrompt(string prompt, byte[] imageBytes)
|
||||||
{
|
{
|
||||||
return TryReadPngDimensions(imageBytes, out var width, out var height)
|
return TryReadPngDimensions(imageBytes, out var width, out var height)
|
||||||
@@ -210,46 +189,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
|||||||
bytes[offset + 3];
|
bytes[offset + 3];
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ParseOutputText(string responseJson)
|
|
||||||
{
|
|
||||||
using var document = JsonDocument.Parse(responseJson);
|
|
||||||
var root = document.RootElement;
|
|
||||||
var parts = new List<string>();
|
|
||||||
if (root.TryGetProperty("output_text", out var outputText) &&
|
|
||||||
outputText.ValueKind == JsonValueKind.String &&
|
|
||||||
!string.IsNullOrWhiteSpace(outputText.GetString()))
|
|
||||||
{
|
|
||||||
parts.Add(outputText.GetString()!);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
|
||||||
{
|
|
||||||
foreach (var item in output.EnumerateArray())
|
|
||||||
{
|
|
||||||
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var block in content.EnumerateArray())
|
|
||||||
{
|
|
||||||
if (block.TryGetProperty("type", out var type) &&
|
|
||||||
type.GetString() == "output_text" &&
|
|
||||||
block.TryGetProperty("text", out var text) &&
|
|
||||||
text.ValueKind == JsonValueKind.String &&
|
|
||||||
!string.IsNullOrWhiteSpace(text.GetString()))
|
|
||||||
{
|
|
||||||
parts.Add(text.GetString()!);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return parts.Count == 0
|
|
||||||
? ""
|
|
||||||
: string.Join(Environment.NewLine + Environment.NewLine, parts);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string ResolveApiKey(MeetingAssistantOptions options)
|
private static string ResolveApiKey(MeetingAssistantOptions options)
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
|
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
|
||||||
@@ -278,17 +217,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
|||||||
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
|
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Uri NormalizeEndpoint(Uri endpoint)
|
|
||||||
{
|
|
||||||
var value = endpoint.ToString().TrimEnd('/');
|
|
||||||
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
|
||||||
{
|
|
||||||
value += "/v1";
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Uri(value + "/");
|
|
||||||
}
|
|
||||||
|
|
||||||
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
||||||
private static partial Regex JsonCodeBlockRegex();
|
private static partial Regex JsonCodeBlockRegex();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return await finalizer.FinalizeAsync(
|
return await finalizer.FinalizeEnabledAsync(
|
||||||
wavPath,
|
wavPath,
|
||||||
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
||||||
options.Diarization,
|
options.Diarization,
|
||||||
|
|||||||
@@ -0,0 +1,631 @@
|
|||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
|
||||||
|
{
|
||||||
|
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||||
|
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||||
|
private readonly IResemblyzerVoiceEncoder encoder;
|
||||||
|
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
|
||||||
|
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||||
|
private readonly SpeakerIdentificationOptions options;
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||||
|
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
|
||||||
|
|
||||||
|
public ResemblyzerSpeakerIdentificationService(
|
||||||
|
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||||
|
ISpeakerSnippetExtractor snippetExtractor,
|
||||||
|
IResemblyzerVoiceEncoder encoder,
|
||||||
|
ResemblyzerVoiceClusterMatcher clusterMatcher,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||||
|
IOptions<MeetingAssistantOptions> options,
|
||||||
|
ILogger<ResemblyzerSpeakerIdentificationService> logger)
|
||||||
|
{
|
||||||
|
this.dbContextFactory = dbContextFactory;
|
||||||
|
this.snippetExtractor = snippetExtractor;
|
||||||
|
this.encoder = encoder;
|
||||||
|
this.clusterMatcher = clusterMatcher;
|
||||||
|
this.outlierPruner = outlierPruner;
|
||||||
|
this.options = options.Value.SpeakerIdentification;
|
||||||
|
resemblyzerOptions = this.options.Resemblyzer;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task ApplySpeakerOverrideAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
string sourceSpeaker,
|
||||||
|
string targetSpeaker,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!options.Enabled ||
|
||||||
|
string.IsNullOrWhiteSpace(sourceSpeaker) ||
|
||||||
|
string.IsNullOrWhiteSpace(targetSpeaker) ||
|
||||||
|
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sourceLabel = sourceSpeaker.Trim();
|
||||||
|
var targetName = targetSpeaker.Trim();
|
||||||
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||||
|
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||||
|
var target = identities
|
||||||
|
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||||
|
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||||
|
.ThenBy(identity => identity.Id)
|
||||||
|
.FirstOrDefault();
|
||||||
|
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||||
|
var sourceCandidate = identities
|
||||||
|
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||||
|
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||||
|
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||||
|
.OrderBy(identity => identity.Id)
|
||||||
|
.FirstOrDefault();
|
||||||
|
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
|
||||||
|
if (target is null && sourceCandidate is null && vectors.Count == 0)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
|
||||||
|
sourceLabel,
|
||||||
|
targetName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
if (target is null)
|
||||||
|
{
|
||||||
|
target = sourceCandidate ?? new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CreatedAt = now
|
||||||
|
};
|
||||||
|
if (target.Id == 0)
|
||||||
|
{
|
||||||
|
context.SpeakerIdentities.Add(target);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||||
|
{
|
||||||
|
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||||
|
target,
|
||||||
|
sourceCandidate,
|
||||||
|
options.MaxSnippetsPerSpeaker,
|
||||||
|
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||||
|
outlierPruner);
|
||||||
|
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||||
|
}
|
||||||
|
|
||||||
|
target.CanonicalName = targetName;
|
||||||
|
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||||
|
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
|
||||||
|
outlierPruner.AddAndPrune(
|
||||||
|
target,
|
||||||
|
vectors,
|
||||||
|
now);
|
||||||
|
target.UpdatedAt = now;
|
||||||
|
await context.SaveChangesAsync(cancellationToken);
|
||||||
|
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||||
|
target.References,
|
||||||
|
sourceLabel,
|
||||||
|
targetName,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task DeleteSpeakerIdentityAsync(
|
||||||
|
string identity,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||||
|
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||||
|
var target = identities.FirstOrDefault(candidate =>
|
||||||
|
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
|
||||||
|
if (target is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
context.SpeakerIdentities.Remove(target);
|
||||||
|
await context.SaveChangesAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
bool final,
|
||||||
|
bool allowAudioFallback,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!options.Enabled || request.Segments.Count == 0)
|
||||||
|
{
|
||||||
|
return EmptyResult(request.Segments);
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||||
|
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||||
|
var knownMappings = request.KnownSpeakerMappings ??
|
||||||
|
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
if (final && knownMappings.Count > 0)
|
||||||
|
{
|
||||||
|
await PersistMappedSpeakerEvidenceAsync(
|
||||||
|
context,
|
||||||
|
request,
|
||||||
|
knownMappings,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
var identifiedNames = knownMappings.Values
|
||||||
|
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||||
|
.Select(name => name.Trim())
|
||||||
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
foreach (var segmentSpeaker in request.Segments
|
||||||
|
.Select(segment => segment.Speaker)
|
||||||
|
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||||
|
{
|
||||||
|
identifiedNames.Add(segmentSpeaker.Trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||||
|
var pendingAudits = new List<PendingIdentificationAudit>();
|
||||||
|
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
|
||||||
|
foreach (var speaker in request.Segments
|
||||||
|
.Select(segment => segment.Speaker)
|
||||||
|
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||||
|
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var vectors = await ResolveAutomaticVectorsAsync(
|
||||||
|
request,
|
||||||
|
speaker,
|
||||||
|
allowAudioFallback,
|
||||||
|
cancellationToken);
|
||||||
|
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
|
||||||
|
speaker,
|
||||||
|
vectors.Count,
|
||||||
|
resemblyzerOptions.RequiredVectorsPerSpeaker);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var (identity, decision) = await FindMatchAsync(
|
||||||
|
context,
|
||||||
|
attendees,
|
||||||
|
identifiedNames,
|
||||||
|
vectors,
|
||||||
|
cancellationToken);
|
||||||
|
if (identity is null)
|
||||||
|
{
|
||||||
|
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
|
||||||
|
{
|
||||||
|
unmatchedSpeakers.Add((speaker, vectors));
|
||||||
|
}
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var previousCanonicalName = identity.CanonicalName;
|
||||||
|
var previousReferenceCount = identity.References.Count;
|
||||||
|
outlierPruner.AddAndPrune(
|
||||||
|
identity,
|
||||||
|
vectors,
|
||||||
|
now);
|
||||||
|
SpeakerIdentityReferences.AddIfMissing(
|
||||||
|
identity,
|
||||||
|
CreateReference(request.MeetingNote, now),
|
||||||
|
now);
|
||||||
|
if (identity.References.Count != previousReferenceCount)
|
||||||
|
{
|
||||||
|
identity.UpdatedAt = now;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (final)
|
||||||
|
{
|
||||||
|
UpdateMatchedIdentity(identity, attendees);
|
||||||
|
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||||
|
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||||
|
{
|
||||||
|
pendingAudits.Add(new PendingIdentificationAudit(
|
||||||
|
identity,
|
||||||
|
speaker,
|
||||||
|
identity.CanonicalName));
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||||
|
{
|
||||||
|
matchedAcceptedNames.Add(acceptedName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var displayName = identity.GetDisplayName();
|
||||||
|
if (!string.IsNullOrWhiteSpace(displayName))
|
||||||
|
{
|
||||||
|
mappings[speaker] = displayName;
|
||||||
|
identifiedNames.Add(displayName);
|
||||||
|
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||||
|
{
|
||||||
|
identifiedNames.Add(acceptedName);
|
||||||
|
}
|
||||||
|
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||||
|
displayName,
|
||||||
|
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (final)
|
||||||
|
{
|
||||||
|
LearnUnmatchedSpeakers(
|
||||||
|
context,
|
||||||
|
request.MeetingNote,
|
||||||
|
attendees,
|
||||||
|
matchedAcceptedNames,
|
||||||
|
unmatchedSpeakers,
|
||||||
|
pendingAudits);
|
||||||
|
}
|
||||||
|
|
||||||
|
await context.SaveChangesAsync(cancellationToken);
|
||||||
|
await AppendAuditsAsync(pendingAudits, cancellationToken);
|
||||||
|
var relabeled = request.Segments
|
||||||
|
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
|
||||||
|
? segment with { Speaker = name }
|
||||||
|
: segment)
|
||||||
|
.ToList();
|
||||||
|
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
|
||||||
|
SpeakerIdentityDbContext context,
|
||||||
|
IReadOnlyList<string> attendees,
|
||||||
|
IReadOnlySet<string> identifiedNames,
|
||||||
|
IReadOnlyList<float[]> queryVectors,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||||
|
var identities = await LoadIdentities(context)
|
||||||
|
.OrderByDescending(identity => identity.References.Count)
|
||||||
|
.ThenBy(identity => identity.Id)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var candidates = identities
|
||||||
|
.Select(identity => new
|
||||||
|
{
|
||||||
|
Identity = identity,
|
||||||
|
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||||
|
IsActive = identity.UpdatedAt >= activeCutoff
|
||||||
|
})
|
||||||
|
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||||
|
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
|
||||||
|
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
|
||||||
|
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||||
|
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||||
|
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
|
||||||
|
.ThenBy(candidate => candidate.Identity.Id)
|
||||||
|
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||||
|
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
|
||||||
|
candidate.Identity.Id,
|
||||||
|
SpeakerVoiceVectors.DecodeCompatible(
|
||||||
|
candidate.Identity,
|
||||||
|
resemblyzerOptions.ModelId,
|
||||||
|
logger)))
|
||||||
|
.Where(candidate => candidate.Vectors.Count > 0)
|
||||||
|
.ToList();
|
||||||
|
var match = clusterMatcher.Match(queryVectors, candidates);
|
||||||
|
return (
|
||||||
|
match.IdentityId is { } identityId
|
||||||
|
? identities.Single(identity => identity.Id == identityId)
|
||||||
|
: null,
|
||||||
|
match);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LearnUnmatchedSpeakers(
|
||||||
|
SpeakerIdentityDbContext context,
|
||||||
|
MeetingNote meetingNote,
|
||||||
|
IReadOnlyList<string> attendees,
|
||||||
|
IReadOnlySet<string> matchedAcceptedNames,
|
||||||
|
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
|
||||||
|
ICollection<PendingIdentificationAudit> pendingAudits)
|
||||||
|
{
|
||||||
|
var candidates = attendees
|
||||||
|
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Order(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToList();
|
||||||
|
if (candidates.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var (speaker, vectors) in unmatchedSpeakers)
|
||||||
|
{
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var identity = new SpeakerIdentity
|
||||||
|
{
|
||||||
|
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
|
||||||
|
CreatedAt = now,
|
||||||
|
UpdatedAt = now,
|
||||||
|
CandidateNames = candidates
|
||||||
|
.Select(name => new SpeakerCandidateName { Name = name })
|
||||||
|
.ToList(),
|
||||||
|
References = [CreateReference(meetingNote, now)]
|
||||||
|
};
|
||||||
|
outlierPruner.AddAndPrune(
|
||||||
|
identity,
|
||||||
|
vectors,
|
||||||
|
now);
|
||||||
|
context.SpeakerIdentities.Add(identity);
|
||||||
|
logger.LogInformation(
|
||||||
|
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
|
||||||
|
speaker,
|
||||||
|
identity.VoiceVectors.Count,
|
||||||
|
string.Join(", ", candidates));
|
||||||
|
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||||
|
{
|
||||||
|
pendingAudits.Add(new PendingIdentificationAudit(
|
||||||
|
identity,
|
||||||
|
speaker,
|
||||||
|
identity.CanonicalName));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task AppendAuditsAsync(
|
||||||
|
IEnumerable<PendingIdentificationAudit> pendingAudits,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
foreach (var audit in pendingAudits)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||||
|
audit.Identity.References,
|
||||||
|
audit.Speaker,
|
||||||
|
audit.Name,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
|
||||||
|
audit.Identity.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
string speaker,
|
||||||
|
bool allowAudioFallback,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||||
|
var samples = await ResolveWavSamplesAsync(
|
||||||
|
request,
|
||||||
|
speaker,
|
||||||
|
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||||
|
allowAudioFallback,
|
||||||
|
cancellationToken);
|
||||||
|
if (samples.Count < requiredCount)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task PersistMappedSpeakerEvidenceAsync(
|
||||||
|
SpeakerIdentityDbContext context,
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
IReadOnlyDictionary<string, string> knownMappings,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||||
|
foreach (var (speaker, mappedName) in knownMappings)
|
||||||
|
{
|
||||||
|
var identity = identities
|
||||||
|
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
|
||||||
|
.OrderBy(candidate =>
|
||||||
|
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||||
|
.ThenBy(candidate => candidate.Id)
|
||||||
|
.FirstOrDefault();
|
||||||
|
if (identity is null)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
|
||||||
|
speaker,
|
||||||
|
mappedName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
|
||||||
|
if (vectors.Count == 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var previousReferenceCount = identity.References.Count;
|
||||||
|
var vectorUpdate = outlierPruner.AddAndPrune(
|
||||||
|
identity,
|
||||||
|
vectors,
|
||||||
|
now);
|
||||||
|
SpeakerIdentityReferences.AddIfMissing(
|
||||||
|
identity,
|
||||||
|
CreateReference(request.MeetingNote, now),
|
||||||
|
now);
|
||||||
|
if (identity.References.Count != previousReferenceCount)
|
||||||
|
{
|
||||||
|
identity.UpdatedAt = now;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
|
||||||
|
vectorUpdate.AddedCount,
|
||||||
|
speaker,
|
||||||
|
identity.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
string speaker,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var samples = await ResolveWavSamplesAsync(
|
||||||
|
request,
|
||||||
|
speaker,
|
||||||
|
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||||
|
allowAudioFallback: true,
|
||||||
|
cancellationToken);
|
||||||
|
if (samples.Count == 0)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
|
||||||
|
SpeakerIdentificationRequest request,
|
||||||
|
string speaker,
|
||||||
|
int maxSamples,
|
||||||
|
bool allowAudioFallback,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var suppliedSamples = request.Samples?
|
||||||
|
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
|
||||||
|
.Where(sample => sample.WavBytes.Length > 0)
|
||||||
|
.OrderByDescending(sample => sample.Score)
|
||||||
|
.Take(maxSamples)
|
||||||
|
.ToList()
|
||||||
|
?? [];
|
||||||
|
var wavSamples = suppliedSamples
|
||||||
|
.Select(sample => sample.WavBytes)
|
||||||
|
.ToList();
|
||||||
|
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
|
||||||
|
{
|
||||||
|
return wavSamples;
|
||||||
|
}
|
||||||
|
|
||||||
|
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||||
|
request.Segments,
|
||||||
|
speaker,
|
||||||
|
options.MinimumSampleSpeechDuration,
|
||||||
|
options.MaximumSampleDuration,
|
||||||
|
request.Segments.Count);
|
||||||
|
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
|
||||||
|
{
|
||||||
|
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
|
||||||
|
request.AudioPath,
|
||||||
|
span,
|
||||||
|
cancellationToken);
|
||||||
|
if (wavBytes.Length > 0)
|
||||||
|
{
|
||||||
|
wavSamples.Add(wavBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (wavSamples.Count >= maxSamples)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
|
||||||
|
wavSamples.Count,
|
||||||
|
maxSamples,
|
||||||
|
speaker,
|
||||||
|
suppliedSamples.Count,
|
||||||
|
wavSamples.Count - suppliedSamples.Count);
|
||||||
|
return wavSamples;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool OverlapsSuppliedSample(
|
||||||
|
IReadOnlyList<TranscriptionSegment> span,
|
||||||
|
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
|
||||||
|
{
|
||||||
|
if (span.Count == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var start = span[0].Start;
|
||||||
|
var end = span[^1].End;
|
||||||
|
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||||
|
{
|
||||||
|
return context.SpeakerIdentities
|
||||||
|
.AsSplitQuery()
|
||||||
|
.Include(identity => identity.Aliases)
|
||||||
|
.Include(identity => identity.CandidateNames)
|
||||||
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
|
.Include(identity => identity.References);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
|
||||||
|
{
|
||||||
|
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||||
|
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||||
|
{
|
||||||
|
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
|
||||||
|
{
|
||||||
|
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record PendingIdentificationAudit(
|
||||||
|
SpeakerIdentity Identity,
|
||||||
|
string Speaker,
|
||||||
|
string Name);
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||||
|
{
|
||||||
|
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||||
|
private readonly ResemblyzerVoiceClusterMatcher matcher;
|
||||||
|
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||||
|
private readonly SpeakerIdentificationOptions options;
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||||
|
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
|
||||||
|
|
||||||
|
public ResemblyzerSpeakerIdentityMergeService(
|
||||||
|
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||||
|
ResemblyzerVoiceClusterMatcher matcher,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||||
|
IOptions<MeetingAssistantOptions> options,
|
||||||
|
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
|
||||||
|
{
|
||||||
|
this.dbContextFactory = dbContextFactory;
|
||||||
|
this.matcher = matcher;
|
||||||
|
this.outlierPruner = outlierPruner;
|
||||||
|
this.options = options.Value.SpeakerIdentification;
|
||||||
|
resemblyzerOptions = this.options.Resemblyzer;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||||
|
TimeSpan? recentIdentityAge,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||||
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||||
|
var identities = await context.SpeakerIdentities
|
||||||
|
.AsSplitQuery()
|
||||||
|
.Include(identity => identity.Aliases)
|
||||||
|
.Include(identity => identity.CandidateNames)
|
||||||
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
|
.Include(identity => identity.References)
|
||||||
|
.OrderByDescending(identity => identity.References.Count)
|
||||||
|
.ThenBy(identity => identity.Id)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||||
|
var recentIds = identities
|
||||||
|
.Where(identity => identity.CreatedAt >= cutoff)
|
||||||
|
.Select(identity => identity.Id)
|
||||||
|
.ToList();
|
||||||
|
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||||
|
var attempts = 0;
|
||||||
|
var mergedPairs = 0;
|
||||||
|
var pendingAudits = new List<PendingMergeAudit>();
|
||||||
|
|
||||||
|
foreach (var sourceId in recentIds)
|
||||||
|
{
|
||||||
|
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||||
|
if (source is null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
|
||||||
|
source,
|
||||||
|
resemblyzerOptions.ModelId,
|
||||||
|
logger)
|
||||||
|
.Take(required * 2)
|
||||||
|
.ToList();
|
||||||
|
if (sourceVectors.Count < required * 2)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
|
||||||
|
source.Id,
|
||||||
|
sourceVectors.Count,
|
||||||
|
required * 2);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var candidates = identities
|
||||||
|
.Where(identity => identity.Id != source.Id)
|
||||||
|
.Where(identity => identity.VoiceVectors.Any(vector =>
|
||||||
|
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||||
|
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||||
|
.Select(identity => new ResemblyzerVoiceVectorCandidate(
|
||||||
|
identity.Id,
|
||||||
|
SpeakerVoiceVectors.DecodeCompatible(
|
||||||
|
identity,
|
||||||
|
resemblyzerOptions.ModelId,
|
||||||
|
logger)))
|
||||||
|
.Where(candidate => candidate.Vectors.Count > 0)
|
||||||
|
.ToList();
|
||||||
|
if (candidates.Count == 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
attempts++;
|
||||||
|
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
|
||||||
|
if (!first.Accepted || first.IdentityId is not { } targetId)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
attempts++;
|
||||||
|
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
|
||||||
|
if (!second.Accepted || second.IdentityId != targetId)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
|
||||||
|
source.Id,
|
||||||
|
first.IdentityId,
|
||||||
|
second.IdentityId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
|
||||||
|
if (target is null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||||
|
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||||
|
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||||
|
target,
|
||||||
|
source,
|
||||||
|
options.MaxSnippetsPerSpeaker,
|
||||||
|
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||||
|
outlierPruner);
|
||||||
|
pendingAudits.Add(new PendingMergeAudit(
|
||||||
|
target,
|
||||||
|
targetName,
|
||||||
|
sourceName));
|
||||||
|
context.SpeakerIdentities.Remove(source);
|
||||||
|
identities.Remove(source);
|
||||||
|
mergedPairs++;
|
||||||
|
}
|
||||||
|
|
||||||
|
await context.SaveChangesAsync(cancellationToken);
|
||||||
|
foreach (var audit in pendingAudits)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
|
||||||
|
audit.Identity.References,
|
||||||
|
audit.TargetName,
|
||||||
|
audit.SourceName,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
|
||||||
|
audit.Identity.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new SpeakerIdentityMergeResult(
|
||||||
|
recentIds.Count,
|
||||||
|
identities.Count,
|
||||||
|
attempts,
|
||||||
|
mergedPairs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record PendingMergeAudit(
|
||||||
|
SpeakerIdentity Identity,
|
||||||
|
string TargetName,
|
||||||
|
string SourceName);
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed record ResemblyzerVoiceVectorCandidate(
|
||||||
|
int IdentityId,
|
||||||
|
IReadOnlyList<float[]> Vectors);
|
||||||
|
|
||||||
|
public sealed record ResemblyzerVoiceClusterMatchResult(
|
||||||
|
int? IdentityId,
|
||||||
|
string Reason,
|
||||||
|
double Cohesion,
|
||||||
|
double? BestSimilarity,
|
||||||
|
double? RunnerUpSimilarity)
|
||||||
|
{
|
||||||
|
public bool Accepted => IdentityId.HasValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ResemblyzerVoiceClusterMatcher
|
||||||
|
{
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||||
|
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
|
||||||
|
|
||||||
|
public ResemblyzerVoiceClusterMatcher(
|
||||||
|
ResemblyzerSpeakerRecognitionOptions options,
|
||||||
|
ILogger<ResemblyzerVoiceClusterMatcher> logger)
|
||||||
|
{
|
||||||
|
this.options = options;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResemblyzerVoiceClusterMatchResult Match(
|
||||||
|
IReadOnlyList<float[]> queryVectors,
|
||||||
|
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
|
||||||
|
{
|
||||||
|
var requiredCount = options.RequiredVectorsPerSpeaker;
|
||||||
|
if (queryVectors.Count < requiredCount)
|
||||||
|
{
|
||||||
|
return Reject(
|
||||||
|
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
|
||||||
|
cohesion: 0,
|
||||||
|
bestSimilarity: null,
|
||||||
|
runnerUpSimilarity: null);
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalizedQuery = queryVectors
|
||||||
|
.Take(requiredCount)
|
||||||
|
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||||
|
.ToList();
|
||||||
|
var cohesion = MeanPairwiseCosine(normalizedQuery);
|
||||||
|
if (cohesion < options.MinimumClusterCohesion)
|
||||||
|
{
|
||||||
|
return Reject(
|
||||||
|
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
|
||||||
|
cohesion,
|
||||||
|
bestSimilarity: null,
|
||||||
|
runnerUpSimilarity: null);
|
||||||
|
}
|
||||||
|
|
||||||
|
var scored = candidates
|
||||||
|
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
|
||||||
|
.Where(candidate => candidate is not null)
|
||||||
|
.Select(candidate => candidate!.Value)
|
||||||
|
.OrderByDescending(candidate => candidate.Similarity)
|
||||||
|
.ThenBy(candidate => candidate.IdentityId)
|
||||||
|
.ToList();
|
||||||
|
if (scored.Count == 0)
|
||||||
|
{
|
||||||
|
return Reject("no compatible identity vectors were available", cohesion, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
var best = scored[0];
|
||||||
|
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
|
||||||
|
if (best.Similarity < options.MinimumIdentitySimilarity)
|
||||||
|
{
|
||||||
|
return Reject(
|
||||||
|
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
|
||||||
|
cohesion,
|
||||||
|
best.Similarity,
|
||||||
|
runnerUp);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runnerUp is { } runnerUpSimilarity &&
|
||||||
|
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
|
||||||
|
{
|
||||||
|
return Reject(
|
||||||
|
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
|
||||||
|
cohesion,
|
||||||
|
best.Similarity,
|
||||||
|
runnerUpSimilarity);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
|
||||||
|
best.IdentityId,
|
||||||
|
cohesion,
|
||||||
|
best.Similarity,
|
||||||
|
runnerUp);
|
||||||
|
return new ResemblyzerVoiceClusterMatchResult(
|
||||||
|
best.IdentityId,
|
||||||
|
"accepted",
|
||||||
|
cohesion,
|
||||||
|
best.Similarity,
|
||||||
|
runnerUp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ResemblyzerVoiceClusterMatchResult Reject(
|
||||||
|
string reason,
|
||||||
|
double cohesion,
|
||||||
|
double? bestSimilarity,
|
||||||
|
double? runnerUpSimilarity)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
|
||||||
|
reason,
|
||||||
|
cohesion,
|
||||||
|
bestSimilarity,
|
||||||
|
runnerUpSimilarity);
|
||||||
|
return new ResemblyzerVoiceClusterMatchResult(
|
||||||
|
null,
|
||||||
|
reason,
|
||||||
|
cohesion,
|
||||||
|
bestSimilarity,
|
||||||
|
runnerUpSimilarity);
|
||||||
|
}
|
||||||
|
|
||||||
|
private (int IdentityId, double Similarity)? ScoreCandidate(
|
||||||
|
IReadOnlyList<float[]> normalizedQuery,
|
||||||
|
ResemblyzerVoiceVectorCandidate candidate)
|
||||||
|
{
|
||||||
|
if (candidate.Vectors.Count == 0)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var normalizedCandidate = candidate.Vectors
|
||||||
|
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||||
|
.ToList();
|
||||||
|
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
|
||||||
|
var similarities = normalizedQuery
|
||||||
|
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
|
||||||
|
.Order()
|
||||||
|
.ToList();
|
||||||
|
return (candidate.IdentityId, Median(similarities));
|
||||||
|
}
|
||||||
|
catch (InvalidDataException exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
|
||||||
|
candidate.IdentityId);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float[] Centroid(IReadOnlyList<float[]> vectors)
|
||||||
|
{
|
||||||
|
var centroid = new float[ResemblyzerVectorContract.Dimensions];
|
||||||
|
foreach (var vector in vectors)
|
||||||
|
{
|
||||||
|
for (var index = 0; index < centroid.Length; index++)
|
||||||
|
{
|
||||||
|
centroid[index] += vector[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var index = 0; index < centroid.Length; index++)
|
||||||
|
{
|
||||||
|
centroid[index] /= vectors.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
return centroid;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
|
||||||
|
{
|
||||||
|
if (vectors.Count < 2)
|
||||||
|
{
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
double total = 0;
|
||||||
|
var pairs = 0;
|
||||||
|
for (var first = 0; first < vectors.Count - 1; first++)
|
||||||
|
{
|
||||||
|
for (var second = first + 1; second < vectors.Count; second++)
|
||||||
|
{
|
||||||
|
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
|
||||||
|
pairs++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total / pairs;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static double Median(IReadOnlyList<double> sortedValues)
|
||||||
|
{
|
||||||
|
var middle = sortedValues.Count / 2;
|
||||||
|
return sortedValues.Count % 2 == 0
|
||||||
|
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
|
||||||
|
: sortedValues[middle];
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed record ResemblyzerVoiceVectorUpdateResult(
|
||||||
|
int AddedCount,
|
||||||
|
int RemovedCount)
|
||||||
|
{
|
||||||
|
public bool Changed => AddedCount > 0 || RemovedCount > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ResemblyzerVoiceVectorOutlierPruner
|
||||||
|
{
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||||
|
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
|
||||||
|
|
||||||
|
public ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
ResemblyzerSpeakerRecognitionOptions options,
|
||||||
|
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
|
||||||
|
{
|
||||||
|
this.options = options;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Prune(SpeakerIdentity identity)
|
||||||
|
{
|
||||||
|
var compatible = new List<DecodedSpeakerVoiceVector>();
|
||||||
|
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
|
||||||
|
identity,
|
||||||
|
options.ModelId,
|
||||||
|
logger))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
compatible.Add(new DecodedSpeakerVoiceVector(
|
||||||
|
entry.Stored,
|
||||||
|
SpeakerVoiceVectors.Normalize(
|
||||||
|
entry.Vector,
|
||||||
|
$"Stored voice vector {entry.Stored.Id}")));
|
||||||
|
}
|
||||||
|
catch (InvalidDataException exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
|
||||||
|
entry.Stored.Id,
|
||||||
|
identity.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var minimumVectors = options.OutlierPruningMinimumVectors;
|
||||||
|
if (compatible.Count < minimumVectors)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
|
||||||
|
var ranked = clusters
|
||||||
|
.GroupBy(cluster => cluster)
|
||||||
|
.Where(group => group.Key > 0)
|
||||||
|
.Select(group => new { Id = group.Key, Count = group.Count() })
|
||||||
|
.OrderByDescending(group => group.Count)
|
||||||
|
.ThenBy(group => group.Id)
|
||||||
|
.ToList();
|
||||||
|
if (ranked.Count == 0)
|
||||||
|
{
|
||||||
|
return Skip(identity, "no dense cluster was found");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
|
||||||
|
{
|
||||||
|
return Skip(identity, "no uniquely largest dense cluster was found");
|
||||||
|
}
|
||||||
|
|
||||||
|
var dominant = ranked[0];
|
||||||
|
var ratio = (double)dominant.Count / compatible.Count;
|
||||||
|
if (ratio < options.OutlierPruningMinimumClusterRatio)
|
||||||
|
{
|
||||||
|
return Skip(
|
||||||
|
identity,
|
||||||
|
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var removed = compatible
|
||||||
|
.Where((_, index) => clusters[index] != dominant.Id)
|
||||||
|
.Select(item => item.Stored)
|
||||||
|
.ToList();
|
||||||
|
foreach (var vector in removed)
|
||||||
|
{
|
||||||
|
identity.VoiceVectors.Remove(vector);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (removed.Count > 0)
|
||||||
|
{
|
||||||
|
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
|
||||||
|
removed.Count,
|
||||||
|
identity.Id,
|
||||||
|
dominant.Count,
|
||||||
|
compatible.Count);
|
||||||
|
return removed.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IReadOnlyList<float[]> vectors,
|
||||||
|
DateTimeOffset createdAt)
|
||||||
|
{
|
||||||
|
var removedBeforeAdding = Prune(identity);
|
||||||
|
var added = SpeakerVoiceVectors.AddDistinct(
|
||||||
|
identity,
|
||||||
|
vectors,
|
||||||
|
options.ModelId,
|
||||||
|
options.MaxVectorsPerIdentity,
|
||||||
|
createdAt);
|
||||||
|
var removedAfterAdding = Prune(identity);
|
||||||
|
var result = new ResemblyzerVoiceVectorUpdateResult(
|
||||||
|
added,
|
||||||
|
removedBeforeAdding + removedAfterAdding);
|
||||||
|
if (result.Changed)
|
||||||
|
{
|
||||||
|
identity.UpdatedAt = createdAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
|
||||||
|
{
|
||||||
|
var neighbors = Enumerable.Range(0, vectors.Count)
|
||||||
|
.Select(index => Enumerable.Range(0, vectors.Count)
|
||||||
|
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
|
||||||
|
.ToList())
|
||||||
|
.ToList();
|
||||||
|
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
|
||||||
|
var labels = new int[vectors.Count];
|
||||||
|
var clusterId = 0;
|
||||||
|
|
||||||
|
for (var point = 0; point < vectors.Count; point++)
|
||||||
|
{
|
||||||
|
if (labels[point] != 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (neighbors[point].Count < minimumNeighbors)
|
||||||
|
{
|
||||||
|
labels[point] = -1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
clusterId++;
|
||||||
|
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
|
||||||
|
}
|
||||||
|
|
||||||
|
return labels;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ExpandCluster(
|
||||||
|
int seed,
|
||||||
|
int clusterId,
|
||||||
|
IReadOnlyList<List<int>> neighbors,
|
||||||
|
int[] labels,
|
||||||
|
int minimumNeighbors)
|
||||||
|
{
|
||||||
|
labels[seed] = clusterId;
|
||||||
|
var pending = new Queue<int>(neighbors[seed]);
|
||||||
|
var queued = neighbors[seed].ToHashSet();
|
||||||
|
while (pending.TryDequeue(out var point))
|
||||||
|
{
|
||||||
|
if (labels[point] == -1)
|
||||||
|
{
|
||||||
|
labels[point] = clusterId;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (labels[point] != 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
labels[point] = clusterId;
|
||||||
|
if (neighbors[point].Count < minimumNeighbors)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var neighbor in neighbors[point])
|
||||||
|
{
|
||||||
|
if (queued.Add(neighbor))
|
||||||
|
{
|
||||||
|
pending.Enqueue(neighbor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int Skip(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
string reason)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
|
||||||
|
identity.Id,
|
||||||
|
reason);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed class ResemblyzerWarmupHostedService : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly IResemblyzerVoiceEncoder encoder;
|
||||||
|
private readonly bool speakerIdentificationEnabled;
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||||
|
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
|
||||||
|
public ResemblyzerWarmupHostedService(
|
||||||
|
IResemblyzerVoiceEncoder encoder,
|
||||||
|
IOptions<MeetingAssistantOptions> options,
|
||||||
|
ILogger<ResemblyzerWarmupHostedService> logger)
|
||||||
|
{
|
||||||
|
this.encoder = encoder;
|
||||||
|
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
|
||||||
|
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
if (!speakerIdentificationEnabled || !options.Enabled)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Yield();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||||
|
VaultPath.Resolve(options.RuntimeFolder),
|
||||||
|
options.ModelId);
|
||||||
|
await encoder.WarmUpAsync(stoppingToken);
|
||||||
|
logger.LogInformation(
|
||||||
|
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||||
|
VaultPath.Resolve(options.RuntimeFolder),
|
||||||
|
options.ModelId);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
|
||||||
|
VaultPath.Resolve(options.RuntimeFolder));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
|
|||||||
|
|
||||||
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
||||||
|
|
||||||
|
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
|
||||||
|
|
||||||
public List<SpeakerIdentityReference> References { get; set; } = [];
|
public List<SpeakerIdentityReference> References { get; set; } = [];
|
||||||
|
|
||||||
[NotMapped]
|
[NotMapped]
|
||||||
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
|
|||||||
string.IsNullOrWhiteSpace(reference.TranscriptPath);
|
string.IsNullOrWhiteSpace(reference.TranscriptPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||||
{
|
{
|
||||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||||
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
|
|||||||
public DateTimeOffset CreatedAt { get; set; }
|
public DateTimeOffset CreatedAt { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
public int Id { get; set; }
|
||||||
|
|
||||||
|
public int SpeakerIdentityId { get; set; }
|
||||||
|
|
||||||
|
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||||
|
|
||||||
|
public string ModelId { get; set; } = "";
|
||||||
|
|
||||||
|
public int Dimensions { get; set; }
|
||||||
|
|
||||||
|
public byte[] VectorBytes { get; set; } = [];
|
||||||
|
|
||||||
|
public string Fingerprint { get; set; } = "";
|
||||||
|
|
||||||
|
public DateTimeOffset CreatedAt { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class SpeakerAlias
|
public sealed class SpeakerAlias
|
||||||
{
|
{
|
||||||
public int Id { get; set; }
|
public int Id { get; set; }
|
||||||
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
|||||||
|
|
||||||
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
||||||
|
|
||||||
|
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
|
||||||
|
|
||||||
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
|
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
|
||||||
|
|
||||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||||
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
|||||||
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
||||||
.OnDelete(DeleteBehavior.Cascade);
|
.OnDelete(DeleteBehavior.Cascade);
|
||||||
|
|
||||||
|
entity.HasMany(identity => identity.VoiceVectors)
|
||||||
|
.WithOne(vector => vector.SpeakerIdentity)
|
||||||
|
.HasForeignKey(vector => vector.SpeakerIdentityId)
|
||||||
|
.OnDelete(DeleteBehavior.Cascade);
|
||||||
|
|
||||||
entity.HasMany(identity => identity.References)
|
entity.HasMany(identity => identity.References)
|
||||||
.WithOne(reference => reference.SpeakerIdentity)
|
.WithOne(reference => reference.SpeakerIdentity)
|
||||||
.HasForeignKey(reference => reference.SpeakerIdentityId)
|
.HasForeignKey(reference => reference.SpeakerIdentityId)
|
||||||
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
|||||||
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
||||||
.IsUnique();
|
.IsUnique();
|
||||||
|
|
||||||
|
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||||
|
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
|
||||||
|
.IsUnique();
|
||||||
|
|
||||||
|
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||||
|
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
|
||||||
|
|
||||||
modelBuilder.Entity<SpeakerIdentityReference>()
|
modelBuilder.Entity<SpeakerIdentityReference>()
|
||||||
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
|
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
|
||||||
.IsUnique();
|
.IsUnique();
|
||||||
|
|||||||
@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
|||||||
{
|
{
|
||||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||||
private readonly ISpeakerIdentityMatcher matcher;
|
private readonly ISpeakerIdentityMatcher matcher;
|
||||||
|
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||||
private readonly SpeakerIdentificationOptions options;
|
private readonly SpeakerIdentificationOptions options;
|
||||||
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
||||||
|
|
||||||
public SpeakerIdentityMergeService(
|
public SpeakerIdentityMergeService(
|
||||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||||
ISpeakerIdentityMatcher matcher,
|
ISpeakerIdentityMatcher matcher,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||||
IOptions<MeetingAssistantOptions> options,
|
IOptions<MeetingAssistantOptions> options,
|
||||||
ILogger<SpeakerIdentityMergeService> logger)
|
ILogger<SpeakerIdentityMergeService> logger)
|
||||||
{
|
{
|
||||||
this.dbContextFactory = dbContextFactory;
|
this.dbContextFactory = dbContextFactory;
|
||||||
this.matcher = matcher;
|
this.matcher = matcher;
|
||||||
|
this.outlierPruner = outlierPruner;
|
||||||
this.options = options.Value.SpeakerIdentification;
|
this.options = options.Value.SpeakerIdentification;
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
}
|
}
|
||||||
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
|||||||
|
|
||||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||||
var identities = await context.SpeakerIdentities
|
var identities = await context.SpeakerIdentities
|
||||||
|
.AsSplitQuery()
|
||||||
.Include(identity => identity.Aliases)
|
.Include(identity => identity.Aliases)
|
||||||
.Include(identity => identity.CandidateNames)
|
.Include(identity => identity.CandidateNames)
|
||||||
.Include(identity => identity.Snippets)
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
.Include(identity => identity.References)
|
.Include(identity => identity.References)
|
||||||
.OrderByDescending(identity => identity.References.Count)
|
.OrderByDescending(identity => identity.References.Count)
|
||||||
.ThenBy(identity => identity.Id)
|
.ThenBy(identity => identity.Id)
|
||||||
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
|||||||
|
|
||||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||||
SpeakerIdentityMerger.MergeInto(
|
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||||
target,
|
target,
|
||||||
source,
|
source,
|
||||||
options.MaxSnippetsPerSpeaker);
|
options.MaxSnippetsPerSpeaker,
|
||||||
|
options.Resemblyzer.MaxVectorsPerIdentity,
|
||||||
|
outlierPruner);
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
||||||
source.Id,
|
source.Id,
|
||||||
|
|||||||
@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
|
|||||||
|
|
||||||
internal static class SpeakerIdentityMerger
|
internal static class SpeakerIdentityMerger
|
||||||
{
|
{
|
||||||
|
public static void MergeIntoAndPrune(
|
||||||
|
SpeakerIdentity target,
|
||||||
|
SpeakerIdentity source,
|
||||||
|
int maxSnippets,
|
||||||
|
int maxVoiceVectors,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
|
||||||
|
{
|
||||||
|
outlierPruner.Prune(target);
|
||||||
|
outlierPruner.Prune(source);
|
||||||
|
MergeInto(target, source, maxSnippets, maxVoiceVectors);
|
||||||
|
outlierPruner.Prune(target);
|
||||||
|
}
|
||||||
|
|
||||||
public static void MergeInto(
|
public static void MergeInto(
|
||||||
SpeakerIdentity target,
|
SpeakerIdentity target,
|
||||||
SpeakerIdentity source,
|
SpeakerIdentity source,
|
||||||
int maxSnippets)
|
int maxSnippets,
|
||||||
|
int maxVoiceVectors = int.MaxValue)
|
||||||
{
|
{
|
||||||
AddAlias(target, source.CanonicalName);
|
AddAlias(target, source.CanonicalName);
|
||||||
foreach (var alias in source.Aliases)
|
foreach (var alias in source.Aliases)
|
||||||
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
|
|||||||
.ToList();
|
.ToList();
|
||||||
target.Snippets.Clear();
|
target.Snippets.Clear();
|
||||||
target.Snippets.AddRange(retainedSnippets);
|
target.Snippets.AddRange(retainedSnippets);
|
||||||
|
|
||||||
|
var retainedVectors = target.VoiceVectors
|
||||||
|
.Concat(source.VoiceVectors)
|
||||||
|
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||||
|
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
|
||||||
|
.OrderByDescending(vector => vector.CreatedAt)
|
||||||
|
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||||
|
.Take(Math.Max(1, maxVoiceVectors))
|
||||||
|
.Select(vector => new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
SpeakerIdentity = target,
|
||||||
|
ModelId = vector.ModelId,
|
||||||
|
Dimensions = vector.Dimensions,
|
||||||
|
VectorBytes = vector.VectorBytes.ToArray(),
|
||||||
|
Fingerprint = vector.Fingerprint,
|
||||||
|
CreatedAt = vector.CreatedAt
|
||||||
|
})
|
||||||
|
.ToList();
|
||||||
|
target.VoiceVectors.Clear();
|
||||||
|
target.VoiceVectors.AddRange(retainedVectors);
|
||||||
target.UpdatedAt = DateTimeOffset.UtcNow;
|
target.UpdatedAt = DateTimeOffset.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
internal static class SpeakerIdentityNaming
|
||||||
|
{
|
||||||
|
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||||
|
{
|
||||||
|
return new[] { identity.CanonicalName }
|
||||||
|
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||||
|
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||||
|
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||||
|
.Select(name => name!.Trim())
|
||||||
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool MatchesAnyAcceptedName(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IReadOnlySet<string> names)
|
||||||
|
{
|
||||||
|
return GetAcceptedNames(identity).Any(names.Contains);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool MatchesAttendees(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IReadOnlyList<string> attendees)
|
||||||
|
{
|
||||||
|
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||||
|
{
|
||||||
|
return attendees
|
||||||
|
.Select(MeetingAttendeeNames.NormalizeDisplayName)
|
||||||
|
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||||
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Order(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool IsDiarizedSpeakerLabel(string speaker)
|
||||||
|
{
|
||||||
|
var normalized = speaker.Trim();
|
||||||
|
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool UpdateCandidateNames(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IReadOnlyList<string> attendees)
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var currentCandidates = identity.CandidateNames
|
||||||
|
.Select(candidate => candidate.Name)
|
||||||
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||||
|
var aliasToCandidate = identity.Aliases
|
||||||
|
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||||
|
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||||
|
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||||
|
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||||
|
var intersection = attendees
|
||||||
|
.Select(attendee => currentCandidates.Contains(attendee)
|
||||||
|
? attendee
|
||||||
|
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||||
|
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||||
|
? fallbackAliasCandidate
|
||||||
|
: null))
|
||||||
|
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||||
|
.Select(candidate => candidate!)
|
||||||
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Order(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToList();
|
||||||
|
var resetToAttendees = intersection.Count == 0;
|
||||||
|
SetCandidates(identity, resetToAttendees ? attendees : intersection);
|
||||||
|
if (intersection.Count == 1)
|
||||||
|
{
|
||||||
|
identity.CanonicalName = intersection[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
return resetToAttendees;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void SetCandidates(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IEnumerable<string> candidates)
|
||||||
|
{
|
||||||
|
identity.CandidateNames.Clear();
|
||||||
|
identity.CandidateNames.AddRange(candidates
|
||||||
|
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||||
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Order(StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
|
|||||||
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
|
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
|
||||||
""",
|
""",
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
await context.Database.ExecuteSqlRawAsync(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
|
||||||
|
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
|
||||||
|
"SpeakerIdentityId" INTEGER NOT NULL,
|
||||||
|
"ModelId" TEXT NOT NULL,
|
||||||
|
"Dimensions" INTEGER NOT NULL,
|
||||||
|
"VectorBytes" BLOB NOT NULL,
|
||||||
|
"Fingerprint" TEXT NOT NULL,
|
||||||
|
"CreatedAt" TEXT NOT NULL,
|
||||||
|
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
|
||||||
|
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
""",
|
||||||
|
cancellationToken);
|
||||||
|
await context.Database.ExecuteSqlRawAsync(
|
||||||
|
"""
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
|
||||||
|
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
|
||||||
|
""",
|
||||||
|
cancellationToken);
|
||||||
|
await context.Database.ExecuteSqlRawAsync(
|
||||||
|
"""
|
||||||
|
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
|
||||||
|
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
|
||||||
|
""",
|
||||||
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
|
|
||||||
target.CanonicalName = targetName;
|
target.CanonicalName = targetName;
|
||||||
target.UpdatedAt = now;
|
target.UpdatedAt = now;
|
||||||
ResetCandidates(target, [targetName]);
|
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||||
AddMeetingReference(target, meetingReference);
|
AddMeetingReference(target, meetingReference);
|
||||||
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
||||||
await context.SaveChangesAsync(cancellationToken);
|
await context.SaveChangesAsync(cancellationToken);
|
||||||
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||||
|
|
||||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||||
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||||
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
foreach (var speaker in request.Segments
|
foreach (var speaker in request.Segments
|
||||||
.Select(segment => segment.Speaker)
|
.Select(segment => segment.Speaker)
|
||||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
|
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||||
{
|
{
|
||||||
alreadyIdentifiedNames.Add(speaker.Trim());
|
alreadyIdentifiedNames.Add(speaker.Trim());
|
||||||
}
|
}
|
||||||
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||||
{
|
{
|
||||||
matchedAcceptedNames.Add(acceptedName);
|
matchedAcceptedNames.Add(acceptedName);
|
||||||
}
|
}
|
||||||
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
{
|
{
|
||||||
speakerMappings[speaker] = speakerName;
|
speakerMappings[speaker] = speakerName;
|
||||||
alreadyIdentifiedNames.Add(speakerName);
|
alreadyIdentifiedNames.Add(speakerName);
|
||||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||||
{
|
{
|
||||||
alreadyIdentifiedNames.Add(acceptedName);
|
alreadyIdentifiedNames.Add(acceptedName);
|
||||||
}
|
}
|
||||||
|
|
||||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||||
speakerName,
|
speakerName,
|
||||||
GetAcceptedNames(identity).ToList()));
|
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
.Select(identity => new
|
.Select(identity => new
|
||||||
{
|
{
|
||||||
Identity = identity,
|
Identity = identity,
|
||||||
IsAttendee = MatchesAttendees(identity, attendees),
|
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||||
IsActive = identity.UpdatedAt >= activeCutoff
|
IsActive = identity.UpdatedAt >= activeCutoff
|
||||||
})
|
})
|
||||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||||
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
string operation,
|
string operation,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||||
request.Segments,
|
request.Segments,
|
||||||
speaker,
|
speaker,
|
||||||
options.MaximumSampleSegmentGap,
|
options.MinimumSampleSpeechDuration,
|
||||||
options.MinimumSampleSpeechDuration);
|
options.MaximumSampleDuration);
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
||||||
operation,
|
operation,
|
||||||
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
.Include(identity => identity.References)
|
.Include(identity => identity.References)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
return identities
|
return identities
|
||||||
.Where(identity => GetAcceptedNames(identity).Contains(name))
|
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
|
||||||
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||||
.ThenBy(identity => identity.Id)
|
.ThenBy(identity => identity.Id)
|
||||||
.FirstOrDefault();
|
.FirstOrDefault();
|
||||||
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
return candidates
|
return candidates
|
||||||
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
|
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||||
.Where(identity => identity.CandidateNames.Any(candidate =>
|
.Where(identity => identity.CandidateNames.Any(candidate =>
|
||||||
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
|
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
|
||||||
identity.Aliases.Any(alias =>
|
identity.Aliases.Any(alias =>
|
||||||
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool IsSameReference(
|
|
||||||
SpeakerIdentityReference first,
|
|
||||||
SpeakerIdentityReference second)
|
|
||||||
{
|
|
||||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
|
||||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool MatchesAcceptedNames(
|
private static bool MatchesAcceptedNames(
|
||||||
SpeakerIdentity identity,
|
SpeakerIdentity identity,
|
||||||
IReadOnlySet<string> names)
|
IReadOnlySet<string> names)
|
||||||
{
|
{
|
||||||
return GetAcceptedNames(identity).Any(names.Contains);
|
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
|
||||||
}
|
|
||||||
|
|
||||||
private static bool MatchesAttendees(
|
|
||||||
SpeakerIdentity identity,
|
|
||||||
IReadOnlyList<string> attendees)
|
|
||||||
{
|
|
||||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
||||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||||
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
var currentCandidates = identity.CandidateNames
|
var currentCandidates = identity.CandidateNames
|
||||||
.Select(candidate => candidate.Name)
|
.Select(candidate => candidate.Name)
|
||||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||||
var aliasToCandidate = identity.Aliases
|
if (resetToAttendees)
|
||||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
|
||||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
|
||||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
|
||||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
|
||||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
|
||||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
|
||||||
var intersection = attendees
|
|
||||||
.Select(attendee => currentCandidates.Contains(attendee)
|
|
||||||
? attendee
|
|
||||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
|
||||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
|
||||||
? fallbackAliasCandidate
|
|
||||||
: null))
|
|
||||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
|
||||||
.Select(candidate => candidate!)
|
|
||||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.Order(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (intersection.Count == 0)
|
|
||||||
{
|
{
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
||||||
identity.Id,
|
identity.Id,
|
||||||
FormatNames(attendees));
|
FormatNames(attendees));
|
||||||
ResetCandidates(identity, attendees);
|
|
||||||
ReplaceOldestSnippet(identity, snippet);
|
ReplaceOldestSnippet(identity, snippet);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
|
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
|
||||||
identity.Id,
|
identity.Id,
|
||||||
FormatNames(currentCandidates),
|
FormatNames(currentCandidates),
|
||||||
FormatNames(attendees),
|
FormatNames(attendees),
|
||||||
FormatNames(intersection));
|
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
|
||||||
ResetCandidates(identity, intersection);
|
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||||
if (intersection.Count == 1)
|
|
||||||
{
|
{
|
||||||
identity.CanonicalName = intersection[0];
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
||||||
identity.Id,
|
identity.Id,
|
||||||
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
|
||||||
{
|
|
||||||
identity.CandidateNames.Clear();
|
|
||||||
identity.CandidateNames.AddRange(candidates
|
|
||||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.Order(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
|
||||||
{
|
|
||||||
return new[]
|
|
||||||
{
|
|
||||||
identity.CanonicalName
|
|
||||||
}
|
|
||||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
|
||||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
|
||||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
|
||||||
.Select(name => name!.Trim())
|
|
||||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
|
||||||
{
|
|
||||||
return attendees
|
|
||||||
.Select(NormalizeAttendee)
|
|
||||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
|
||||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.Order(StringComparer.OrdinalIgnoreCase)
|
|
||||||
.ToList();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string NormalizeAttendee(string attendee)
|
|
||||||
{
|
|
||||||
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsDiarizedSpeakerLabel(string speaker)
|
|
||||||
{
|
|
||||||
var normalized = speaker.Trim();
|
|
||||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
|
||||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
|
||||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||||
{
|
{
|
||||||
return SpeakerIdentityReferences.Create(
|
return SpeakerIdentityReferences.Create(
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public sealed record SpeakerSampleCollectionPolicy(
|
||||||
|
int MinimumRetainedSamples,
|
||||||
|
bool RequireNonOverlappingSamples)
|
||||||
|
{
|
||||||
|
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
|
||||||
|
|
||||||
|
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
|
||||||
|
{
|
||||||
|
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int ResolveRetainedSampleLimit(int configuredLimit)
|
||||||
|
{
|
||||||
|
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static class SpeakerSampleDurationConfiguration
|
||||||
|
{
|
||||||
|
public static void ValidateOrThrow(
|
||||||
|
SpeakerIdentificationOptions options,
|
||||||
|
string context)
|
||||||
|
{
|
||||||
|
ValidateOrThrow(
|
||||||
|
options.MinimumSampleSpeechDuration,
|
||||||
|
options.MaximumSampleDuration,
|
||||||
|
context);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void ValidateOrThrow(
|
||||||
|
TimeSpan minimumDuration,
|
||||||
|
TimeSpan maximumDuration,
|
||||||
|
string context)
|
||||||
|
{
|
||||||
|
var error = GetError(minimumDuration, maximumDuration);
|
||||||
|
if (error is not null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"{context}: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string? GetError(
|
||||||
|
TimeSpan minimumDuration,
|
||||||
|
TimeSpan maximumDuration)
|
||||||
|
{
|
||||||
|
if (minimumDuration < TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (maximumDuration <= TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return minimumDuration > maximumDuration
|
||||||
|
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static class ResemblyzerSpeakerRecognitionConfiguration
|
||||||
|
{
|
||||||
|
public static void ValidateOrThrow(
|
||||||
|
ResemblyzerSpeakerRecognitionOptions options,
|
||||||
|
string context)
|
||||||
|
{
|
||||||
|
var error = GetError(options);
|
||||||
|
if (error is not null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"{context}: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
|
||||||
|
{
|
||||||
|
if (options.RequiredVectorsPerSpeaker < 1)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.OutlierPruningMinimumNeighbors < 1 ||
|
||||||
|
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
|
||||||
|
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsCosine(options.MinimumClusterCohesion))
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsCosine(options.MinimumIdentitySimilarity))
|
||||||
|
{
|
||||||
|
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return !double.IsFinite(options.MinimumSimilarityMargin) ||
|
||||||
|
options.MinimumSimilarityMargin is < 0 or > 2
|
||||||
|
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsCosine(double value)
|
||||||
|
{
|
||||||
|
return double.IsFinite(value) && value is >= -1 and <= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
|
||||||
|
: IValidateOptions<MeetingAssistantOptions>
|
||||||
|
{
|
||||||
|
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
var error = SpeakerSampleDurationConfiguration.GetError(
|
||||||
|
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||||
|
options.SpeakerIdentification.MaximumSampleDuration) ??
|
||||||
|
ResemblyzerSpeakerRecognitionConfiguration.GetError(
|
||||||
|
options.SpeakerIdentification.Resemblyzer);
|
||||||
|
return error is null
|
||||||
|
? ValidateOptionsResult.Success
|
||||||
|
: ValidateOptionsResult.Fail(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
internal sealed record SpeakerSampleSpan(
|
||||||
|
string Speaker,
|
||||||
|
TimeSpan Start,
|
||||||
|
TimeSpan End,
|
||||||
|
TimeSpan SpeechDuration,
|
||||||
|
TimeSpan MaximumDuration,
|
||||||
|
IReadOnlyList<TranscriptionSegment> Segments)
|
||||||
|
{
|
||||||
|
public static SpeakerSampleSpan? Create(
|
||||||
|
TranscriptionSegment segment,
|
||||||
|
TimeSpan maximumDuration)
|
||||||
|
{
|
||||||
|
if (maximumDuration <= TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
throw new ArgumentOutOfRangeException(
|
||||||
|
nameof(maximumDuration),
|
||||||
|
maximumDuration,
|
||||||
|
"Speaker sample maximum duration must be greater than zero.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
|
||||||
|
return boundedEnd > segment.Start
|
||||||
|
? new SpeakerSampleSpan(
|
||||||
|
segment.Speaker,
|
||||||
|
segment.Start,
|
||||||
|
boundedEnd,
|
||||||
|
boundedEnd - segment.Start,
|
||||||
|
maximumDuration,
|
||||||
|
[segment with { End = boundedEnd }])
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool TryExtend(
|
||||||
|
TranscriptionSegment segment,
|
||||||
|
out SpeakerSampleSpan extended)
|
||||||
|
{
|
||||||
|
extended = this;
|
||||||
|
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
segment.Start >= Start + MaximumDuration)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var boundedEnd = Min(segment.End, Start + MaximumDuration);
|
||||||
|
if (boundedEnd <= segment.Start)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var uncoveredStart = segment.Start > End ? segment.Start : End;
|
||||||
|
var additionalSpeechDuration = boundedEnd > uncoveredStart
|
||||||
|
? boundedEnd - uncoveredStart
|
||||||
|
: TimeSpan.Zero;
|
||||||
|
extended = this with
|
||||||
|
{
|
||||||
|
End = boundedEnd > End ? boundedEnd : End,
|
||||||
|
SpeechDuration = SpeechDuration + additionalSpeechDuration,
|
||||||
|
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
|
||||||
|
};
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TranscriptionSegment ToSegment()
|
||||||
|
{
|
||||||
|
return new TranscriptionSegment(
|
||||||
|
Start,
|
||||||
|
End,
|
||||||
|
Speaker,
|
||||||
|
string.Join(' ', Segments
|
||||||
|
.Select(segment => segment.Text)
|
||||||
|
.Where(text => !string.IsNullOrWhiteSpace(text))));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TimeSpan Min(TimeSpan left, TimeSpan right)
|
||||||
|
{
|
||||||
|
return left < right ? left : right;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
|
|||||||
|
|
||||||
internal static class SpeakerSampleSpanSelector
|
internal static class SpeakerSampleSpanSelector
|
||||||
{
|
{
|
||||||
public static bool CanExtend(
|
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
|
||||||
string currentSpeaker,
|
|
||||||
TimeSpan currentEnd,
|
|
||||||
TranscriptionSegment nextSegment,
|
|
||||||
TimeSpan maximumSegmentGap)
|
|
||||||
{
|
|
||||||
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
|
|
||||||
nextSegment.Start - currentEnd <= maximumSegmentGap;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
|
|
||||||
IReadOnlyList<TranscriptionSegment> segments,
|
IReadOnlyList<TranscriptionSegment> segments,
|
||||||
string speaker,
|
string speaker,
|
||||||
TimeSpan maximumSegmentGap,
|
TimeSpan minimumDuration,
|
||||||
TimeSpan minimumDuration)
|
TimeSpan maximumSampleDuration)
|
||||||
{
|
{
|
||||||
var best = new List<TranscriptionSegment>();
|
SpeakerSampleSpan? best = null;
|
||||||
var current = new List<TranscriptionSegment>();
|
SpeakerSampleSpan? current = null;
|
||||||
|
|
||||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||||
{
|
{
|
||||||
if (current.Count == 0)
|
if (!IsSpeaker(segment, speaker))
|
||||||
{
|
{
|
||||||
if (IsSpeaker(segment, speaker))
|
current = null;
|
||||||
{
|
|
||||||
current.Add(segment);
|
|
||||||
best = LongerSpan(current, best);
|
|
||||||
}
|
|
||||||
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
|
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||||
|
if (current is not null &&
|
||||||
|
(best is null || current.SpeechDuration > best.SpeechDuration))
|
||||||
{
|
{
|
||||||
current.Clear();
|
best = current;
|
||||||
if (!IsSpeaker(segment, speaker))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
current.Add(segment);
|
|
||||||
best = LongerSpan(current, best);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return SpanDuration(best) >= minimumDuration
|
return best is not null && best.SpeechDuration >= minimumDuration
|
||||||
? best
|
? best.Segments
|
||||||
: [];
|
: [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
|
||||||
|
IReadOnlyList<TranscriptionSegment> segments,
|
||||||
|
string speaker,
|
||||||
|
TimeSpan minimumDuration,
|
||||||
|
TimeSpan maximumSampleDuration,
|
||||||
|
int maxSpans)
|
||||||
|
{
|
||||||
|
if (maxSpans <= 0)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
var completed = new List<SpeakerSampleSpan>();
|
||||||
|
SpeakerSampleSpan? current = null;
|
||||||
|
TimeSpan? lastCompletedEnd = null;
|
||||||
|
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||||
|
{
|
||||||
|
if (!IsSpeaker(segment, speaker))
|
||||||
|
{
|
||||||
|
current = null;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastCompletedEnd is { } end && segment.Start < end)
|
||||||
|
{
|
||||||
|
current = null;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||||
|
if (current is null || current.SpeechDuration < minimumDuration)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
completed.Add(current);
|
||||||
|
lastCompletedEnd = current.End;
|
||||||
|
current = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return completed
|
||||||
|
.OrderByDescending(span => span.SpeechDuration)
|
||||||
|
.ThenBy(span => span.Start)
|
||||||
|
.Take(maxSpans)
|
||||||
|
.OrderBy(span => span.Start)
|
||||||
|
.Select(span => span.Segments)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
||||||
{
|
{
|
||||||
return segments.Count == 0
|
return segments.Count == 0
|
||||||
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
|
|||||||
: segments[^1].End - segments[0].Start;
|
: segments[^1].End - segments[0].Start;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static SpeakerSampleSpan? ExtendOrStart(
|
||||||
|
SpeakerSampleSpan? current,
|
||||||
|
TranscriptionSegment segment,
|
||||||
|
TimeSpan maximumSampleDuration)
|
||||||
|
{
|
||||||
|
return current is not null && current.TryExtend(segment, out var extended)
|
||||||
|
? extended
|
||||||
|
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||||
|
}
|
||||||
|
|
||||||
private static bool IsSpeaker(
|
private static bool IsSpeaker(
|
||||||
TranscriptionSegment segment,
|
TranscriptionSegment segment,
|
||||||
string speaker)
|
string speaker)
|
||||||
{
|
{
|
||||||
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<TranscriptionSegment> LongerSpan(
|
|
||||||
List<TranscriptionSegment> current,
|
|
||||||
List<TranscriptionSegment> best)
|
|
||||||
{
|
|
||||||
return SpanDuration(current) > SpanDuration(best)
|
|
||||||
? current.ToList()
|
|
||||||
: best;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
using System.Buffers.Binary;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
internal sealed record DecodedSpeakerVoiceVector(
|
||||||
|
SpeakerVoiceVector Stored,
|
||||||
|
float[] Vector);
|
||||||
|
|
||||||
|
internal static class ResemblyzerVectorContract
|
||||||
|
{
|
||||||
|
public const int Dimensions = 256;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static class SpeakerVoiceVectors
|
||||||
|
{
|
||||||
|
public static float[] Decode(SpeakerVoiceVector stored)
|
||||||
|
{
|
||||||
|
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
|
||||||
|
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
|
||||||
|
{
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var vector = new float[stored.Dimensions];
|
||||||
|
for (var index = 0; index < vector.Length; index++)
|
||||||
|
{
|
||||||
|
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
|
||||||
|
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
|
||||||
|
if (!float.IsFinite(vector[index]))
|
||||||
|
{
|
||||||
|
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int AddDistinct(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
IEnumerable<float[]> vectors,
|
||||||
|
string modelId,
|
||||||
|
int maximumCount,
|
||||||
|
DateTimeOffset createdAt)
|
||||||
|
{
|
||||||
|
var limit = Math.Max(1, maximumCount);
|
||||||
|
var fingerprints = identity.VoiceVectors
|
||||||
|
.Select(vector => vector.Fingerprint)
|
||||||
|
.ToHashSet(StringComparer.Ordinal);
|
||||||
|
var added = 0;
|
||||||
|
foreach (var vector in vectors)
|
||||||
|
{
|
||||||
|
if (identity.VoiceVectors.Count >= limit)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var bytes = Encode(vector);
|
||||||
|
var fingerprint = Fingerprint(modelId, bytes);
|
||||||
|
if (!fingerprints.Add(fingerprint))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||||
|
{
|
||||||
|
ModelId = modelId,
|
||||||
|
Dimensions = vector.Length,
|
||||||
|
VectorBytes = bytes,
|
||||||
|
Fingerprint = fingerprint,
|
||||||
|
CreatedAt = createdAt
|
||||||
|
});
|
||||||
|
added++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return added;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IReadOnlyList<float[]> DecodeCompatible(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
string modelId,
|
||||||
|
ILogger logger)
|
||||||
|
{
|
||||||
|
return DecodeCompatibleEntries(identity, modelId, logger)
|
||||||
|
.Select(entry => entry.Vector)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
|
||||||
|
SpeakerIdentity identity,
|
||||||
|
string modelId,
|
||||||
|
ILogger logger)
|
||||||
|
{
|
||||||
|
var vectors = new List<DecodedSpeakerVoiceVector>();
|
||||||
|
foreach (var stored in identity.VoiceVectors
|
||||||
|
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
|
||||||
|
.OrderBy(vector => vector.CreatedAt)
|
||||||
|
.ThenBy(vector => vector.Id))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
|
||||||
|
}
|
||||||
|
catch (InvalidDataException exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
|
||||||
|
stored.Id,
|
||||||
|
identity.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return vectors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static double Cosine(float[] first, float[] second)
|
||||||
|
{
|
||||||
|
double dot = 0;
|
||||||
|
for (var index = 0; index < first.Length; index++)
|
||||||
|
{
|
||||||
|
dot += first[index] * second[index];
|
||||||
|
}
|
||||||
|
|
||||||
|
return dot;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static float[] Normalize(float[]? vector, string description)
|
||||||
|
{
|
||||||
|
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
double magnitudeSquared = 0;
|
||||||
|
foreach (var value in vector)
|
||||||
|
{
|
||||||
|
if (!float.IsFinite(value))
|
||||||
|
{
|
||||||
|
throw new InvalidDataException($"{description} contains a non-finite value.");
|
||||||
|
}
|
||||||
|
|
||||||
|
magnitudeSquared += value * value;
|
||||||
|
}
|
||||||
|
|
||||||
|
var magnitude = Math.Sqrt(magnitudeSquared);
|
||||||
|
if (magnitude <= double.Epsilon)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException($"{description} has zero magnitude.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return vector.Select(value => (float)(value / magnitude)).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] Encode(float[] vector)
|
||||||
|
{
|
||||||
|
if (vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var bytes = new byte[vector.Length * sizeof(float)];
|
||||||
|
for (var index = 0; index < vector.Length; index++)
|
||||||
|
{
|
||||||
|
if (!float.IsFinite(vector[index]))
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Voice vector contains a non-finite value.");
|
||||||
|
}
|
||||||
|
|
||||||
|
BinaryPrimitives.WriteSingleLittleEndian(
|
||||||
|
bytes.AsSpan(index * sizeof(float), sizeof(float)),
|
||||||
|
vector[index]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Fingerprint(string modelId, byte[] vectorBytes)
|
||||||
|
{
|
||||||
|
var modelBytes = Encoding.UTF8.GetBytes(modelId);
|
||||||
|
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
|
||||||
|
modelBytes.CopyTo(input, 0);
|
||||||
|
input[modelBytes.Length] = 0;
|
||||||
|
vectorBytes.CopyTo(input, modelBytes.Length + 1);
|
||||||
|
return Convert.ToHexString(SHA256.HashData(input));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Speakers;
|
||||||
|
|
||||||
|
public interface IResemblyzerVoiceEncoder
|
||||||
|
{
|
||||||
|
Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||||
|
IReadOnlyList<byte[]> wavSamples,
|
||||||
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
Task WarmUpAsync(CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
|
||||||
|
{
|
||||||
|
private const string EnvironmentSchemaVersion = "venv-v2";
|
||||||
|
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
|
||||||
|
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
|
||||||
|
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
|
||||||
|
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
|
||||||
|
private const string LibrosaRequirement = "librosa>=0.9.1";
|
||||||
|
private const string ScipyRequirement = "scipy>=1.2.1";
|
||||||
|
private readonly SemaphoreSlim commandLock = new(1, 1);
|
||||||
|
private readonly ICommandRunner commandRunner;
|
||||||
|
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||||
|
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
|
||||||
|
private string? verifiedEnvironmentPythonPath;
|
||||||
|
|
||||||
|
public VenvResemblyzerVoiceEncoder(
|
||||||
|
ICommandRunner commandRunner,
|
||||||
|
IOptions<MeetingAssistantOptions> options,
|
||||||
|
ILogger<VenvResemblyzerVoiceEncoder> logger)
|
||||||
|
{
|
||||||
|
this.commandRunner = commandRunner;
|
||||||
|
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||||
|
IReadOnlyList<byte[]> wavSamples,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (wavSamples.Count == 0)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (wavSamples.Any(sample => sample.Length == 0))
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||||
|
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(inputFolder);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
for (var index = 0; index < wavSamples.Count; index++)
|
||||||
|
{
|
||||||
|
await File.WriteAllBytesAsync(
|
||||||
|
Path.Combine(inputFolder, $"{index:D4}.wav"),
|
||||||
|
wavSamples[index],
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
await commandLock.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
commandLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Directory.Delete(inputFolder, recursive: true);
|
||||||
|
}
|
||||||
|
catch (DirectoryNotFoundException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
catch (IOException exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||||
|
}
|
||||||
|
catch (UnauthorizedAccessException exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
|
||||||
|
string inputFolder,
|
||||||
|
int expectedCount,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return await RunWithTimeoutAsync(async token =>
|
||||||
|
{
|
||||||
|
var pythonPath = await EnsureEnvironmentAsync(token);
|
||||||
|
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
|
||||||
|
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
|
||||||
|
var result = await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
[scriptPath, Path.GetFullPath(inputFolder)],
|
||||||
|
"encoding",
|
||||||
|
token);
|
||||||
|
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
|
||||||
|
}, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await commandLock.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await RunWithTimeoutAsync(async token =>
|
||||||
|
{
|
||||||
|
await EnsureEnvironmentAsync(token);
|
||||||
|
return true;
|
||||||
|
}, cancellationToken);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
commandLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
ValidateDependencySettings();
|
||||||
|
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||||
|
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
|
||||||
|
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
|
||||||
|
var readyPath = Path.Combine(environmentFolder, ".ready");
|
||||||
|
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& File.Exists(pythonPath)
|
||||||
|
&& File.Exists(readyPath))
|
||||||
|
{
|
||||||
|
return pythonPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (File.Exists(pythonPath) && File.Exists(readyPath))
|
||||||
|
{
|
||||||
|
var verification = await commandRunner.RunAsync(
|
||||||
|
pythonPath,
|
||||||
|
WarmupArguments,
|
||||||
|
cancellationToken);
|
||||||
|
if (verification.ExitCode == 0)
|
||||||
|
{
|
||||||
|
verifiedEnvironmentPythonPath = pythonPath;
|
||||||
|
return pythonPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogWarning(
|
||||||
|
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
|
||||||
|
verification.ExitCode,
|
||||||
|
verification.StandardError);
|
||||||
|
File.Delete(readyPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
Directory.CreateDirectory(environmentFolder);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
options.PythonCommand,
|
||||||
|
["-m", "venv", "--clear", environmentFolder],
|
||||||
|
"virtual environment creation",
|
||||||
|
cancellationToken);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
["-m", "pip", "install", "--upgrade", "pip"],
|
||||||
|
"pip upgrade",
|
||||||
|
cancellationToken);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
[
|
||||||
|
"-m", "pip", "install",
|
||||||
|
"--index-url", options.TorchIndexUrl,
|
||||||
|
TorchRequirement
|
||||||
|
],
|
||||||
|
"CPU PyTorch installation",
|
||||||
|
cancellationToken);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
[
|
||||||
|
"-m", "pip", "install",
|
||||||
|
.. RuntimeDependencyRequirements
|
||||||
|
],
|
||||||
|
"Resemblyzer dependency installation",
|
||||||
|
cancellationToken);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
[
|
||||||
|
"-m", "pip", "install", "--no-deps",
|
||||||
|
ResemblyzerRequirement
|
||||||
|
],
|
||||||
|
"Resemblyzer installation",
|
||||||
|
cancellationToken);
|
||||||
|
await RunRequiredAsync(
|
||||||
|
pythonPath,
|
||||||
|
WarmupArguments,
|
||||||
|
"environment verification",
|
||||||
|
cancellationToken);
|
||||||
|
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
|
||||||
|
verifiedEnvironmentPythonPath = pythonPath;
|
||||||
|
return pythonPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<CommandResult> RunRequiredAsync(
|
||||||
|
string fileName,
|
||||||
|
IReadOnlyList<string> arguments,
|
||||||
|
string operation,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
|
||||||
|
ThrowIfFailed(result, operation);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string BuildEncodingScript()
|
||||||
|
{
|
||||||
|
return
|
||||||
|
"import json\n"
|
||||||
|
+ "import sys\n"
|
||||||
|
+ "import numpy as np\n"
|
||||||
|
+ "from pathlib import Path\n"
|
||||||
|
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
|
||||||
|
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
|
||||||
|
+ "vectors = []\n"
|
||||||
|
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
|
||||||
|
+ " wav = preprocess_wav(path)\n"
|
||||||
|
+ " vector = encoder.embed_utterance(wav)\n"
|
||||||
|
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
|
||||||
|
+ $"print('{JsonStart}')\n"
|
||||||
|
+ "print(json.dumps(vectors, allow_nan=False))\n"
|
||||||
|
+ $"print('{JsonEnd}')\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
|
||||||
|
{
|
||||||
|
var json = ExtractJson(output);
|
||||||
|
float[][]? vectors;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
vectors = JsonSerializer.Deserialize<float[][]>(json);
|
||||||
|
}
|
||||||
|
catch (JsonException exception)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (vectors is null || vectors.Length != expectedCount)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return vectors
|
||||||
|
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ExtractJson(string output)
|
||||||
|
{
|
||||||
|
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||||
|
if (start < 0)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
|
||||||
|
}
|
||||||
|
|
||||||
|
start += JsonStart.Length;
|
||||||
|
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||||
|
if (end < 0)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return output[start..end].Trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private string BuildEnvironmentKey()
|
||||||
|
{
|
||||||
|
var settings = string.Join('\n', EnvironmentManifest);
|
||||||
|
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
|
||||||
|
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
|
||||||
|
}
|
||||||
|
|
||||||
|
private string TorchRequirement => $"torch=={options.TorchVersion}";
|
||||||
|
|
||||||
|
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
|
||||||
|
|
||||||
|
private static string[] WarmupArguments =>
|
||||||
|
[
|
||||||
|
"-c",
|
||||||
|
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
|
||||||
|
];
|
||||||
|
|
||||||
|
private string[] RuntimeDependencyRequirements =>
|
||||||
|
[
|
||||||
|
NumpyBeforePython313Requirement,
|
||||||
|
NumpyFromPython313Requirement,
|
||||||
|
LibrosaRequirement,
|
||||||
|
ScipyRequirement,
|
||||||
|
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
|
||||||
|
];
|
||||||
|
|
||||||
|
private IEnumerable<string> EnvironmentManifest =>
|
||||||
|
[
|
||||||
|
EnvironmentSchemaVersion,
|
||||||
|
options.PythonCommand,
|
||||||
|
options.TorchIndexUrl,
|
||||||
|
TorchRequirement,
|
||||||
|
.. RuntimeDependencyRequirements,
|
||||||
|
ResemblyzerRequirement
|
||||||
|
];
|
||||||
|
|
||||||
|
private static string GetEnvironmentPythonPath(string environmentFolder)
|
||||||
|
{
|
||||||
|
return OperatingSystem.IsWindows()
|
||||||
|
? Path.Combine(environmentFolder, "Scripts", "python.exe")
|
||||||
|
: Path.Combine(environmentFolder, "bin", "python");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateDependencySettings()
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(options.PythonCommand))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
|
||||||
|
}
|
||||||
|
|
||||||
|
ValidateVersion(options.PackageVersion, "Resemblyzer package");
|
||||||
|
ValidateVersion(options.TorchVersion, "PyTorch");
|
||||||
|
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
|
||||||
|
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|
||||||
|
|| indexUri.Scheme != Uri.UriSchemeHttps)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ValidateVersion(string version, string dependency)
|
||||||
|
{
|
||||||
|
if (!PackageVersionPattern().IsMatch(version))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<T> RunWithTimeoutAsync<T>(
|
||||||
|
Func<CancellationToken, Task<T>> operation,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
|
||||||
|
? new CancellationTokenSource(options.CommandTimeout)
|
||||||
|
: null;
|
||||||
|
using var linkedSource = timeoutSource is null
|
||||||
|
? null
|
||||||
|
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return await operation(linkedSource?.Token ?? cancellationToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (
|
||||||
|
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||||
|
{
|
||||||
|
throw new TimeoutException(
|
||||||
|
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ThrowIfFailed(CommandResult result, string operation)
|
||||||
|
{
|
||||||
|
if (result.ExitCode != 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
|
||||||
|
private static partial Regex PackageVersionPattern();
|
||||||
|
}
|
||||||
@@ -18,6 +18,12 @@ public sealed class BoundMeetingProjectResolver
|
|||||||
|
|
||||||
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
|
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
|
||||||
|
|
||||||
|
public List<BoundMeetingProject> GetAllProjects() => !Directory.Exists(ProjectsRoot) ? [] :
|
||||||
|
Directory.EnumerateDirectories(ProjectsRoot)
|
||||||
|
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
|
||||||
|
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
|
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
|
||||||
MeetingSessionArtifacts artifacts,
|
MeetingSessionArtifacts artifacts,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -33,10 +39,8 @@ public sealed class BoundMeetingProjectResolver
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return Directory.EnumerateDirectories(ProjectsRoot)
|
return GetAllProjects()
|
||||||
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
|
|
||||||
.Where(project => projectNames.Contains(project.Name))
|
.Where(project => projectNames.Contains(project.Name))
|
||||||
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
|
|
||||||
.ToList();
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
using System.Text.Json.Nodes;
|
||||||
|
using Microsoft.Extensions.AI;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Summary;
|
||||||
|
|
||||||
|
internal static class FunctionInvocationGuard
|
||||||
|
{
|
||||||
|
public static ValueTask<object?> InvokeAsync(
|
||||||
|
FunctionInvocationContext context,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (context.CallContent.Exception is not null)
|
||||||
|
{
|
||||||
|
return ValueTask.FromResult<object?>(new JsonObject
|
||||||
|
{
|
||||||
|
["error"] = new JsonObject
|
||||||
|
{
|
||||||
|
["code"] = "invalid_tool_arguments",
|
||||||
|
["message"] = "Tool arguments must be a valid JSON object."
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return context.Function.InvokeAsync(context.Arguments, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,20 +1,27 @@
|
|||||||
using System.Net.Http.Headers;
|
using System.Net.Http.Headers;
|
||||||
|
using System.ClientModel;
|
||||||
|
using System.ClientModel.Primitives;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Nodes;
|
using System.Text.Json.Nodes;
|
||||||
using Microsoft.Agents.AI.Compaction;
|
using Microsoft.Agents.AI.Compaction;
|
||||||
using Microsoft.Extensions.AI;
|
using Microsoft.Extensions.AI;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using OpenAI.Responses;
|
||||||
|
|
||||||
namespace MeetingAssistant.Summary;
|
namespace MeetingAssistant.Summary;
|
||||||
|
|
||||||
#pragma warning disable MAAI001
|
#pragma warning disable MAAI001
|
||||||
|
#pragma warning disable OPENAI001
|
||||||
public sealed class LiteLlmResponsesChatClient : IChatClient
|
public sealed class LiteLlmResponsesChatClient : IChatClient
|
||||||
{
|
{
|
||||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||||
|
|
||||||
private readonly HttpClient httpClient;
|
private readonly HttpClient httpClient;
|
||||||
|
private readonly ResponsesClient responsesClient;
|
||||||
|
private readonly AsyncLocal<string?> requestInitiator = new();
|
||||||
private readonly string model;
|
private readonly string model;
|
||||||
|
private readonly bool useStreaming;
|
||||||
private readonly bool enableThinking;
|
private readonly bool enableThinking;
|
||||||
private readonly string reasoningEffort;
|
private readonly string reasoningEffort;
|
||||||
private readonly int reconnectionAttempts;
|
private readonly int reconnectionAttempts;
|
||||||
@@ -37,7 +44,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
ILogger? logger = null,
|
ILogger? logger = null,
|
||||||
bool firstRequestIsUser = true,
|
bool firstRequestIsUser = true,
|
||||||
Action? retrying = null,
|
Action? retrying = null,
|
||||||
Action<string>? reasoningSummaryChanged = null)
|
Action<string>? reasoningSummaryChanged = null,
|
||||||
|
bool useStreaming = true)
|
||||||
: this(
|
: this(
|
||||||
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
|
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
|
||||||
apiKey,
|
apiKey,
|
||||||
@@ -50,7 +58,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
logger,
|
logger,
|
||||||
firstRequestIsUser,
|
firstRequestIsUser,
|
||||||
retrying,
|
retrying,
|
||||||
reasoningSummaryChanged)
|
reasoningSummaryChanged,
|
||||||
|
useStreaming)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,11 +75,14 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
ILogger? logger = null,
|
ILogger? logger = null,
|
||||||
bool firstRequestIsUser = true,
|
bool firstRequestIsUser = true,
|
||||||
Action? retrying = null,
|
Action? retrying = null,
|
||||||
Action<string>? reasoningSummaryChanged = null)
|
Action<string>? reasoningSummaryChanged = null,
|
||||||
|
bool useStreaming = true)
|
||||||
{
|
{
|
||||||
this.httpClient = httpClient;
|
this.httpClient = httpClient;
|
||||||
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||||||
|
responsesClient = CreateResponsesClient(httpClient, apiKey, requestInitiator);
|
||||||
this.model = model;
|
this.model = model;
|
||||||
|
this.useStreaming = useStreaming;
|
||||||
this.enableThinking = enableThinking;
|
this.enableThinking = enableThinking;
|
||||||
this.reasoningEffort = reasoningEffort;
|
this.reasoningEffort = reasoningEffort;
|
||||||
this.reconnectionAttempts = Math.Max(0, reconnectionAttempts);
|
this.reconnectionAttempts = Math.Max(0, reconnectionAttempts);
|
||||||
@@ -87,20 +99,50 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
httpClient.Dispose();
|
httpClient.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal static LiteLlmResponsesChatClient Create(
|
||||||
|
AgentOptions options,
|
||||||
|
string apiKey,
|
||||||
|
LiteLlmResponsesCompactionOptions? compactionOptions,
|
||||||
|
ILogger? logger,
|
||||||
|
bool firstRequestIsUser,
|
||||||
|
Action? retrying = null,
|
||||||
|
Action<string>? reasoningSummaryChanged = null)
|
||||||
|
{
|
||||||
|
return new LiteLlmResponsesChatClient(
|
||||||
|
new Uri(options.Endpoint),
|
||||||
|
apiKey,
|
||||||
|
options.Model,
|
||||||
|
options.EnableThinking,
|
||||||
|
options.ReasoningEffort switch
|
||||||
|
{
|
||||||
|
ReasoningEffortOption.None => "none",
|
||||||
|
ReasoningEffortOption.Low => "low",
|
||||||
|
ReasoningEffortOption.High => "high",
|
||||||
|
ReasoningEffortOption.ExtraHigh => "xhigh",
|
||||||
|
_ => "medium"
|
||||||
|
},
|
||||||
|
options.ReconnectionAttempts,
|
||||||
|
options.ReconnectionDelay,
|
||||||
|
compactionOptions,
|
||||||
|
logger,
|
||||||
|
firstRequestIsUser,
|
||||||
|
retrying,
|
||||||
|
reasoningSummaryChanged,
|
||||||
|
options.UseStreaming);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<ChatResponse> GetResponseAsync(
|
public async Task<ChatResponse> GetResponseAsync(
|
||||||
IEnumerable<ChatMessage> messages,
|
IEnumerable<ChatMessage> messages,
|
||||||
ChatOptions? options = null,
|
ChatOptions? options = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
|
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
|
||||||
var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
|
var result = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
|
||||||
|
ReportVisibleReasoningSummaries(result.ReasoningSummaries);
|
||||||
var response = ParseResponseJson(responseJson, out var reasoningSummaries);
|
LogResponseDiagnostics(result.Response);
|
||||||
ReportVisibleReasoningSummaries(reasoningSummaries);
|
ThrowIfResponseHasNoContent(result.Response);
|
||||||
LogResponseDiagnostics(responseJson, response);
|
LogResponseUsage(result.Response.Usage);
|
||||||
ThrowIfResponseHasNoContent(responseJson, response);
|
return result.Response;
|
||||||
LogResponseUsage(response.Usage);
|
|
||||||
return response;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
|
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
|
||||||
@@ -127,79 +169,14 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
: null;
|
: null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ChatResponse ParseResponseJson(string responseJson)
|
private static IReadOnlyList<string> ExtractVisibleReasoningSummaries(ChatResponse response)
|
||||||
{
|
{
|
||||||
return ParseResponseJson(responseJson, out _);
|
return response.Messages
|
||||||
}
|
.SelectMany(message => message.Contents)
|
||||||
|
.OfType<TextReasoningContent>()
|
||||||
private static ChatResponse ParseResponseJson(
|
.Select(content => content.Text)
|
||||||
string responseJson,
|
.Where(text => !string.IsNullOrWhiteSpace(text))
|
||||||
out IReadOnlyList<string> reasoningSummaries)
|
.ToArray();
|
||||||
{
|
|
||||||
using var document = JsonDocument.Parse(responseJson);
|
|
||||||
var root = document.RootElement;
|
|
||||||
var contents = new List<AIContent>();
|
|
||||||
reasoningSummaries = ExtractVisibleReasoningSummaries(root);
|
|
||||||
|
|
||||||
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
|
||||||
{
|
|
||||||
foreach (var item in output.EnumerateArray())
|
|
||||||
{
|
|
||||||
var type = GetString(item, "type");
|
|
||||||
if (type == "message")
|
|
||||||
{
|
|
||||||
AddMessageContent(contents, item);
|
|
||||||
}
|
|
||||||
else if (type == "function_call")
|
|
||||||
{
|
|
||||||
contents.Add(new FunctionCallContent(
|
|
||||||
GetRequiredString(item, "call_id"),
|
|
||||||
GetRequiredString(item, "name"),
|
|
||||||
ParseArguments(GetString(item, "arguments"))));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var message = new ChatMessage
|
|
||||||
{
|
|
||||||
Role = ChatRole.Assistant,
|
|
||||||
Contents = contents
|
|
||||||
};
|
|
||||||
|
|
||||||
return new ChatResponse(message)
|
|
||||||
{
|
|
||||||
ResponseId = GetString(root, "id"),
|
|
||||||
ModelId = GetString(root, "model"),
|
|
||||||
CreatedAt = GetUnixTimestamp(root, "created_at"),
|
|
||||||
Usage = ParseUsage(root),
|
|
||||||
RawRepresentation = responseJson
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static IReadOnlyList<string> ExtractVisibleReasoningSummaries(string responseJson)
|
|
||||||
{
|
|
||||||
using var document = JsonDocument.Parse(responseJson);
|
|
||||||
return ExtractVisibleReasoningSummaries(document.RootElement);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IReadOnlyList<string> ExtractVisibleReasoningSummaries(JsonElement root)
|
|
||||||
{
|
|
||||||
var summaries = new List<string>();
|
|
||||||
|
|
||||||
if (!root.TryGetProperty("output", out var output) || output.ValueKind != JsonValueKind.Array)
|
|
||||||
{
|
|
||||||
return summaries;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var item in output.EnumerateArray())
|
|
||||||
{
|
|
||||||
if (GetString(item, "type") == "reasoning")
|
|
||||||
{
|
|
||||||
AddVisibleReasoningSummaryText(summaries, item);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summaries;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ReportVisibleReasoningSummaries(IReadOnlyList<string> summaries)
|
private void ReportVisibleReasoningSummaries(IReadOnlyList<string> summaries)
|
||||||
@@ -407,7 +384,9 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
return payload;
|
return payload;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<string> PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken)
|
private async Task<ResponsesChatResult> PostWithRetryAsync(
|
||||||
|
JsonObject payload,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var payloadJson = payload.ToJsonString(JsonOptions);
|
var payloadJson = payload.ToJsonString(JsonOptions);
|
||||||
Exception? lastException = null;
|
Exception? lastException = null;
|
||||||
@@ -418,26 +397,36 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
{
|
{
|
||||||
var initiator = NextInitiator();
|
var initiator = NextInitiator();
|
||||||
LogRequestDiagnostics(payload, initiator, attempt);
|
LogRequestDiagnostics(payload, initiator, attempt);
|
||||||
using var request = CreateJsonRequest(
|
|
||||||
"responses",
|
|
||||||
payloadJson,
|
|
||||||
initiator);
|
|
||||||
using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
|
||||||
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
||||||
|
|
||||||
if (response.IsSuccessStatusCode)
|
var createOptions = ModelReaderWriter.Read<CreateResponseOptions>(
|
||||||
|
BinaryData.FromString(payloadJson),
|
||||||
|
ModelReaderWriterOptions.Json)
|
||||||
|
?? throw new InvalidOperationException("Unable to create OpenAI Responses request options.");
|
||||||
|
createOptions.StreamingEnabled = useStreaming;
|
||||||
|
|
||||||
|
var previousInitiator = requestInitiator.Value;
|
||||||
|
requestInitiator.Value = initiator;
|
||||||
|
try
|
||||||
{
|
{
|
||||||
return responseJson;
|
return useStreaming
|
||||||
|
? await GetStreamingResponseAsync(createOptions, cancellationToken).ConfigureAwait(false)
|
||||||
|
: CreateNonStreamingResult((await responsesClient
|
||||||
|
.CreateResponseAsync(createOptions, cancellationToken)
|
||||||
|
.ConfigureAwait(false))
|
||||||
|
.Value
|
||||||
|
.AsChatResponse(createOptions));
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
var exception = new InvalidOperationException(
|
|
||||||
$"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
|
||||||
if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts)
|
|
||||||
{
|
{
|
||||||
throw exception;
|
requestInitiator.Value = previousInitiator;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
lastException = exception;
|
catch (ClientResultException exception)
|
||||||
|
when (!IsRetryableStatusCode(exception.Status) || attempt == reconnectionAttempts)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"LiteLLM Responses request failed with {exception.Status}: {exception.Message}",
|
||||||
|
exception);
|
||||||
}
|
}
|
||||||
catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts)
|
catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts)
|
||||||
{
|
{
|
||||||
@@ -451,6 +440,48 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed.");
|
throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<ResponsesChatResult> GetStreamingResponseAsync(
|
||||||
|
CreateResponseOptions createOptions,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var updates = new List<ChatResponseUpdate>();
|
||||||
|
var reasoningSummaries = new Dictionary<(string ItemId, int SummaryIndex), StringBuilder>();
|
||||||
|
await foreach (var update in responsesClient
|
||||||
|
.CreateResponseStreamingAsync(createOptions, cancellationToken)
|
||||||
|
.AsChatResponseUpdatesAsync(createOptions, cancellationToken)
|
||||||
|
.ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
updates.Add(update);
|
||||||
|
var reasoningUpdate = update.RawRepresentation
|
||||||
|
as StreamingResponseReasoningSummaryTextDeltaUpdate;
|
||||||
|
if (reasoningUpdate is null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var key = (reasoningUpdate.ItemId, reasoningUpdate.SummaryIndex);
|
||||||
|
if (!reasoningSummaries.TryGetValue(key, out var summary))
|
||||||
|
{
|
||||||
|
summary = new StringBuilder();
|
||||||
|
reasoningSummaries[key] = summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
summary.Append(reasoningUpdate.Delta);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ResponsesChatResult(
|
||||||
|
updates.ToChatResponse(),
|
||||||
|
reasoningSummaries.Values
|
||||||
|
.Select(summary => summary.ToString())
|
||||||
|
.Where(summary => !string.IsNullOrWhiteSpace(summary))
|
||||||
|
.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ResponsesChatResult CreateNonStreamingResult(ChatResponse response)
|
||||||
|
{
|
||||||
|
return new ResponsesChatResult(response, ExtractVisibleReasoningSummaries(response));
|
||||||
|
}
|
||||||
|
|
||||||
private HttpRequestMessage CreateJsonRequest(
|
private HttpRequestMessage CreateJsonRequest(
|
||||||
string requestUri,
|
string requestUri,
|
||||||
string payloadJson,
|
string payloadJson,
|
||||||
@@ -489,7 +520,9 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
|
|
||||||
private static bool IsRetryableException(Exception exception)
|
private static bool IsRetryableException(Exception exception)
|
||||||
{
|
{
|
||||||
return exception is HttpRequestException or TaskCanceledException;
|
return exception is ClientResultException clientException
|
||||||
|
&& IsRetryableStatusCode(clientException.Status)
|
||||||
|
|| exception is HttpRequestException or TaskCanceledException;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void LogContextWindow(int estimatedTokens, bool compacted, string source)
|
private void LogContextWindow(int estimatedTokens, bool compacted, string source)
|
||||||
@@ -561,18 +594,18 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
instructionsPreview);
|
instructionsPreview);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void LogResponseDiagnostics(string responseJson, ChatResponse response)
|
private void LogResponseDiagnostics(ChatResponse response)
|
||||||
{
|
{
|
||||||
if (logger is null)
|
if (logger is null)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var outputTypes = GetOutputItemTypes(responseJson);
|
|
||||||
var parsedTypes = response.Messages
|
var parsedTypes = response.Messages
|
||||||
.SelectMany(message => message.Contents)
|
.SelectMany(message => message.Contents)
|
||||||
.Select(content => content.GetType().Name)
|
.Select(content => content.GetType().Name)
|
||||||
.ToArray();
|
.ToArray();
|
||||||
|
var outputTypes = GetOutputItemTypes(response);
|
||||||
var functionCalls = response.Messages
|
var functionCalls = response.Messages
|
||||||
.SelectMany(message => message.Contents)
|
.SelectMany(message => message.Contents)
|
||||||
.OfType<FunctionCallContent>()
|
.OfType<FunctionCallContent>()
|
||||||
@@ -590,40 +623,29 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
if (parsedTypes.Length == 0)
|
if (parsedTypes.Length == 0)
|
||||||
{
|
{
|
||||||
logger.LogWarning(
|
logger.LogWarning(
|
||||||
"LiteLLM Responses response contained no parseable message text or function calls. Raw response preview: {ResponsePreview}",
|
"LiteLLM Responses response contained no parseable message text or function calls.");
|
||||||
Truncate(responseJson, maxLength: 6000));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void ThrowIfResponseHasNoContent(string responseJson, ChatResponse response)
|
private static void ThrowIfResponseHasNoContent(ChatResponse response)
|
||||||
{
|
{
|
||||||
if (response.Messages.SelectMany(message => message.Contents).Any())
|
if (response.Messages.SelectMany(message => message.Contents).Any())
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var outputTypes = GetOutputItemTypes(responseJson);
|
var outputTypes = GetOutputItemTypes(response);
|
||||||
throw new InvalidOperationException(
|
throw new InvalidOperationException(
|
||||||
"LiteLLM Responses returned no parseable message text or function calls. " +
|
"LiteLLM Responses returned no parseable message text or function calls. " +
|
||||||
$"ResponseId={response.ResponseId ?? "<none>"}, outputTypes=[{string.Join(", ", outputTypes)}].");
|
$"ResponseId={response.ResponseId ?? "<none>"}, outputTypes=[{string.Join(", ", outputTypes)}].");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string[] GetOutputItemTypes(string responseJson)
|
private static string[] GetOutputItemTypes(ChatResponse response)
|
||||||
{
|
{
|
||||||
try
|
return response.Messages
|
||||||
{
|
.SelectMany(message => message.Contents)
|
||||||
using var document = JsonDocument.Parse(responseJson);
|
.Select(content => content.RawRepresentation?.GetType().Name ?? content.GetType().Name)
|
||||||
return document.RootElement.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array
|
.ToArray();
|
||||||
? output
|
|
||||||
.EnumerateArray()
|
|
||||||
.Select(item => GetString(item, "type") ?? item.ValueKind.ToString())
|
|
||||||
.ToArray()
|
|
||||||
: [];
|
|
||||||
}
|
|
||||||
catch (JsonException)
|
|
||||||
{
|
|
||||||
return ["<invalid-json>"];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string Truncate(string value, int maxLength)
|
private static string Truncate(string value, int maxLength)
|
||||||
@@ -636,9 +658,9 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
|
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
|
||||||
{
|
{
|
||||||
var role = message.Role.Value;
|
var role = message.Role.Value;
|
||||||
var text = message.Text;
|
|
||||||
if (role == ChatRole.System.Value)
|
if (role == ChatRole.System.Value)
|
||||||
{
|
{
|
||||||
|
var text = message.Text;
|
||||||
if (!string.IsNullOrWhiteSpace(text))
|
if (!string.IsNullOrWhiteSpace(text))
|
||||||
{
|
{
|
||||||
instructions.AppendLine(text);
|
instructions.AppendLine(text);
|
||||||
@@ -647,12 +669,37 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!string.IsNullOrWhiteSpace(text))
|
var isAssistant = role == ChatRole.Assistant.Value;
|
||||||
|
var messageContent = new JsonArray();
|
||||||
|
foreach (var content in message.Contents)
|
||||||
|
{
|
||||||
|
if (content is TextContent textContent && !string.IsNullOrWhiteSpace(textContent.Text))
|
||||||
|
{
|
||||||
|
messageContent.Add(new JsonObject
|
||||||
|
{
|
||||||
|
["type"] = isAssistant ? "output_text" : "input_text",
|
||||||
|
["text"] = textContent.Text
|
||||||
|
});
|
||||||
|
}
|
||||||
|
else if (!isAssistant &&
|
||||||
|
content is DataContent dataContent &&
|
||||||
|
dataContent.HasTopLevelMediaType("image"))
|
||||||
|
{
|
||||||
|
messageContent.Add(new JsonObject
|
||||||
|
{
|
||||||
|
["type"] = "input_image",
|
||||||
|
["image_url"] = dataContent.Uri.ToString()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (messageContent.Count > 0)
|
||||||
{
|
{
|
||||||
input.Add(new JsonObject
|
input.Add(new JsonObject
|
||||||
{
|
{
|
||||||
["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user",
|
["type"] = "message",
|
||||||
["content"] = text
|
["role"] = isAssistant ? "assistant" : "user",
|
||||||
|
["content"] = messageContent
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -665,7 +712,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
["type"] = "function_call",
|
["type"] = "function_call",
|
||||||
["call_id"] = call.CallId,
|
["call_id"] = call.CallId,
|
||||||
["name"] = call.Name,
|
["name"] = call.Name,
|
||||||
["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions)
|
["arguments"] = SerializeFunctionArguments(call)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
else if (content is FunctionResultContent result)
|
else if (content is FunctionResultContent result)
|
||||||
@@ -702,110 +749,15 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void AddMessageContent(List<AIContent> contents, JsonElement item)
|
private static string SerializeFunctionArguments(FunctionCallContent call)
|
||||||
{
|
{
|
||||||
if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array)
|
if (call.RawRepresentation is FunctionCallResponseItem responseItem
|
||||||
|
&& responseItem.FunctionArguments is not null)
|
||||||
{
|
{
|
||||||
return;
|
return responseItem.FunctionArguments.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var content in messageContent.EnumerateArray())
|
return JsonSerializer.Serialize(call.Arguments, JsonOptions);
|
||||||
{
|
|
||||||
if (GetString(content, "type") == "output_text")
|
|
||||||
{
|
|
||||||
var text = GetString(content, "text");
|
|
||||||
if (!string.IsNullOrEmpty(text))
|
|
||||||
{
|
|
||||||
contents.Add(new TextContent(text));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void AddVisibleReasoningSummaryText(List<string> summaries, JsonElement item)
|
|
||||||
{
|
|
||||||
AddVisibleReasoningSummaryText(summaries, item, "summary");
|
|
||||||
AddVisibleReasoningSummaryText(summaries, item, "content");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void AddVisibleReasoningSummaryText(
|
|
||||||
List<string> summaries,
|
|
||||||
JsonElement item,
|
|
||||||
string propertyName)
|
|
||||||
{
|
|
||||||
if (!item.TryGetProperty(propertyName, out var property))
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (property.ValueKind == JsonValueKind.String)
|
|
||||||
{
|
|
||||||
AddNonEmpty(summaries, property.GetString());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (property.ValueKind != JsonValueKind.Array)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var summaryItem in property.EnumerateArray())
|
|
||||||
{
|
|
||||||
if (summaryItem.ValueKind == JsonValueKind.String)
|
|
||||||
{
|
|
||||||
AddNonEmpty(summaries, summaryItem.GetString());
|
|
||||||
}
|
|
||||||
else if (summaryItem.ValueKind == JsonValueKind.Object)
|
|
||||||
{
|
|
||||||
if (propertyName == "content" && !IsSummaryContent(summaryItem))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
AddNonEmpty(summaries, GetString(summaryItem, "text"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsSummaryContent(JsonElement item)
|
|
||||||
{
|
|
||||||
var type = GetString(item, "type");
|
|
||||||
return !string.IsNullOrWhiteSpace(type)
|
|
||||||
&& type.Contains("summary", StringComparison.OrdinalIgnoreCase);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void AddNonEmpty(List<string> values, string? value)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrWhiteSpace(value))
|
|
||||||
{
|
|
||||||
values.Add(value.Trim());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Dictionary<string, object?> ParseArguments(string? arguments)
|
|
||||||
{
|
|
||||||
if (string.IsNullOrWhiteSpace(arguments))
|
|
||||||
{
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
using var document = JsonDocument.Parse(arguments);
|
|
||||||
return document.RootElement.EnumerateObject()
|
|
||||||
.ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static object? ConvertJsonValue(JsonElement element)
|
|
||||||
{
|
|
||||||
return element.ValueKind switch
|
|
||||||
{
|
|
||||||
JsonValueKind.String => element.GetString(),
|
|
||||||
JsonValueKind.Number when element.TryGetInt64(out var integer) => integer,
|
|
||||||
JsonValueKind.Number when element.TryGetDouble(out var number) => number,
|
|
||||||
JsonValueKind.True => true,
|
|
||||||
JsonValueKind.False => false,
|
|
||||||
JsonValueKind.Null => null,
|
|
||||||
_ => JsonSerializer.Deserialize<object>(element.GetRawText(), JsonOptions)
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ResultToString(object? result)
|
private static string ResultToString(object? result)
|
||||||
@@ -818,58 +770,13 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private static UsageDetails? ParseUsage(JsonElement root)
|
|
||||||
{
|
|
||||||
if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object)
|
|
||||||
{
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens");
|
|
||||||
var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens");
|
|
||||||
var totalTokens = GetInt(usage, "total_tokens");
|
|
||||||
return new UsageDetails
|
|
||||||
{
|
|
||||||
InputTokenCount = inputTokens,
|
|
||||||
OutputTokenCount = outputTokens,
|
|
||||||
TotalTokenCount = totalTokens
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int EstimateTokens(JsonObject payload)
|
private static int EstimateTokens(JsonObject payload)
|
||||||
{
|
{
|
||||||
var json = payload.ToJsonString(JsonOptions);
|
var json = payload.ToJsonString(JsonOptions);
|
||||||
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
|
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int? GetInt(JsonElement element, string propertyName)
|
internal static Uri NormalizeEndpoint(Uri endpoint)
|
||||||
{
|
|
||||||
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value)
|
|
||||||
? value
|
|
||||||
: null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string GetRequiredString(JsonElement element, string propertyName)
|
|
||||||
{
|
|
||||||
return GetString(element, propertyName)
|
|
||||||
?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'.");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string? GetString(JsonElement element, string propertyName)
|
|
||||||
{
|
|
||||||
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
|
|
||||||
? property.GetString()
|
|
||||||
: null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName)
|
|
||||||
{
|
|
||||||
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value)
|
|
||||||
? DateTimeOffset.FromUnixTimeSeconds(value)
|
|
||||||
: null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Uri NormalizeEndpoint(Uri endpoint)
|
|
||||||
{
|
{
|
||||||
var value = endpoint.ToString().TrimEnd('/');
|
var value = endpoint.ToString().TrimEnd('/');
|
||||||
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
||||||
@@ -884,5 +791,60 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
|||||||
{
|
{
|
||||||
return value.TrimStart('/');
|
return value.TrimStart('/');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static ResponsesClient CreateResponsesClient(
|
||||||
|
HttpClient httpClient,
|
||||||
|
string apiKey,
|
||||||
|
AsyncLocal<string?> requestInitiator)
|
||||||
|
{
|
||||||
|
var options = new ResponsesClientOptions
|
||||||
|
{
|
||||||
|
Endpoint = httpClient.BaseAddress
|
||||||
|
?? throw new InvalidOperationException("LiteLLM HTTP client requires a base address."),
|
||||||
|
Transport = new HttpClientPipelineTransport(httpClient),
|
||||||
|
RetryPolicy = new ClientRetryPolicy(0)
|
||||||
|
};
|
||||||
|
options.AddPolicy(
|
||||||
|
new InitiatorHeaderPolicy(requestInitiator),
|
||||||
|
PipelinePosition.PerCall);
|
||||||
|
|
||||||
|
return new ResponsesClient(
|
||||||
|
new ApiKeyCredential(apiKey),
|
||||||
|
options);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record ResponsesChatResult(
|
||||||
|
ChatResponse Response,
|
||||||
|
IReadOnlyList<string> ReasoningSummaries);
|
||||||
|
|
||||||
|
private sealed class InitiatorHeaderPolicy(AsyncLocal<string?> requestInitiator) : PipelinePolicy
|
||||||
|
{
|
||||||
|
public override void Process(
|
||||||
|
PipelineMessage message,
|
||||||
|
IReadOnlyList<PipelinePolicy> pipeline,
|
||||||
|
int currentIndex)
|
||||||
|
{
|
||||||
|
SetHeader(message);
|
||||||
|
ProcessNext(message, pipeline, currentIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override ValueTask ProcessAsync(
|
||||||
|
PipelineMessage message,
|
||||||
|
IReadOnlyList<PipelinePolicy> pipeline,
|
||||||
|
int currentIndex)
|
||||||
|
{
|
||||||
|
SetHeader(message);
|
||||||
|
return ProcessNextAsync(message, pipeline, currentIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SetHeader(PipelineMessage message)
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrWhiteSpace(requestInitiator.Value))
|
||||||
|
{
|
||||||
|
message.Request.Headers.Set("X-Initiator", requestInitiator.Value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
#pragma warning restore OPENAI001
|
||||||
#pragma warning restore MAAI001
|
#pragma warning restore MAAI001
|
||||||
|
|||||||
@@ -14,18 +14,26 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
|
|||||||
write_summary is only for the current meeting's summary file. Past project meeting summaries are read-only historical context; use list_past_project_meetings and read_past_project_meeting_summary to inspect them, and never try to mutate them.
|
write_summary is only for the current meeting's summary file. Past project meeting summaries are read-only historical context; use list_past_project_meetings and read_past_project_meeting_summary to inspect them, and never try to mutate them.
|
||||||
If the meeting note has no title, or still has a generated default title like `Meeting yyyy-MM-dd HH:mm`, provide a concise title parameter to write_summary when the purpose of the meeting is clear from transcript, user notes, or assistant context. If the purpose is not clear, omit the title parameter.
|
If the meeting note has no title, or still has a generated default title like `Meeting yyyy-MM-dd HH:mm`, provide a concise title parameter to write_summary when the purpose of the meeting is clear from transcript, user notes, or assistant context. If the purpose is not clear, omit the title parameter.
|
||||||
Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.
|
Use read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.
|
||||||
Use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. write_context appends by default; use replace_file=true only when you are intentionally replacing the whole assistant context body. Keep user-facing summary content in the summary note.
|
Treat the assistant context as persistent meeting-specific memory and use read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Whenever you encounter unexpected problems, discover missing information, or make assumptions while summarizing, use write_context to append a concise note so later agents working on this meeting can understand them. Also record useful internal notes, requests for future tools, suggested improvements, and relevant context discovered from other sources. write_context appends by default; use replace_file=true only when you are intentionally replacing the whole assistant context body. Keep user-facing summary content in the summary note.
|
||||||
Use add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time.
|
Use add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time.
|
||||||
Use add_attendee and remove_attendee to sharpen the meeting attendees list from clear transcript evidence and screenshot OCR participant evidence. Treat OCR that says visible people are a partial screenshot result as incomplete evidence; do not remove attendees solely because they are absent from a partial screenshot.
|
Use add_attendee and remove_attendee to sharpen the meeting attendees list from clear transcript evidence and screenshot OCR participant evidence. Treat OCR that says visible people are a partial screenshot result as incomplete evidence; do not remove attendees solely because they are absent from a partial screenshot.
|
||||||
Use override_speaker only when you are very certain that a transcript speaker label belongs to a named person, for example from user notes, OCR evidence with a matching timestamp, or very clear context cues. Provide the exact speaker label from the transcript and the replacement speaker name. If the replacement speaker already exists in the transcript, use merge=true only when you are certain both speaker labels are the same identity; otherwise do not merge them.
|
Use override_speaker only when you are very certain that a transcript speaker label belongs to a named person, for example from user notes, OCR evidence with a matching timestamp, or very clear context cues. Provide the exact speaker label from the transcript and the replacement speaker name. If the replacement speaker already exists in the transcript, use merge=true only when you are certain both speaker labels are the same identity; otherwise do not merge them.
|
||||||
Use delete_identity only when you are certain that an existing speaker identity was wrongfully matched. Provide the exact identity name currently used in the transcript.
|
Use delete_identity only when you are certain that an existing speaker identity was wrongfully matched. Provide the exact identity name currently used in the transcript.
|
||||||
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
|
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
|
||||||
Use list_projects first to see which projects are bound to this meeting. Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
|
Use search, list_past_project_meetings, read_past_project_meeting_summary, and read_projectfile before changing existing project files. search includes both project files and past meeting summaries for the requested current-meeting project scope.
|
||||||
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
|
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
|
||||||
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
|
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. When embedding them, encode spaces in image-link targets as `%20`; never leave literal spaces in the link target. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
|
||||||
Keep the output grounded in the source material and explicitly say when a section has no known items.
|
Keep the output grounded in the source material and explicitly say when a section has no known items.
|
||||||
""";
|
""";
|
||||||
|
|
||||||
|
private const string ProjectAndRequestInstructions = """
|
||||||
|
Project discovery and approval:
|
||||||
|
list_projects returns every configured project's id, displayname, and description; only projects in the current meeting note's projects frontmatter are accessible for reads, writes, search, and historical summaries. Catalog visibility does not grant file access.
|
||||||
|
Before finalizing the summary, use the catalog when meeting evidence suggests a missing project association. Call request_project_association with its exact id and a short evidence-based reason (one line, at most 100 characters). It waits for user approval for up to ten minutes. A denial or timeout grants no access; continue with the existing scope. Approval adds the project to the meeting and returns its current project instructions; follow them immediately.
|
||||||
|
For each associated project, maintain a root PROJECT.md with YAML frontmatter name (display name) and description (at most 256 characters). Describe the project's purpose and distinguishing context so future summarizers can select it. Read existing content first and use write_projectfile to create missing metadata or update it from grounded knowledge, preserving other fields and body notes. Do not invent facts or write unassociated projects.
|
||||||
|
If a workflow improvement would help, use request_workflow_change with a short one-line intention (at most 100 characters) and a detailed self-contained change prompt, preferably a scenario/specification with expected behavior. This returns requested immediately and does not mean approved or implemented. Continue summarizing; only user approval starts the separate settings agent. You cannot change workflows directly.
|
||||||
|
""";
|
||||||
|
|
||||||
private readonly MeetingAssistantOptions options;
|
private readonly MeetingAssistantOptions options;
|
||||||
private readonly BoundMeetingProjectResolver projectResolver;
|
private readonly BoundMeetingProjectResolver projectResolver;
|
||||||
|
|
||||||
@@ -50,6 +58,7 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
|
|||||||
var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt)
|
var instructions = string.IsNullOrWhiteSpace(options.Agent.InitialPrompt)
|
||||||
? DefaultInitialPrompt
|
? DefaultInitialPrompt
|
||||||
: options.Agent.InitialPrompt.Trim();
|
: options.Agent.InitialPrompt.Trim();
|
||||||
|
instructions = instructions.TrimEnd() + "\n\n" + ProjectAndRequestInstructions;
|
||||||
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, options, cancellationToken);
|
var projectInstructions = await BuildProjectInstructionsAsync(artifacts, options, cancellationToken);
|
||||||
return string.IsNullOrWhiteSpace(projectInstructions)
|
return string.IsNullOrWhiteSpace(projectInstructions)
|
||||||
? instructions
|
? instructions
|
||||||
@@ -60,44 +69,15 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
|
|||||||
MeetingSessionArtifacts artifacts,
|
MeetingSessionArtifacts artifacts,
|
||||||
MeetingAssistantOptions options,
|
MeetingAssistantOptions options,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
|
||||||
var projects = await GetBoundProjectsWithInstructionsAsync(artifacts, options, cancellationToken);
|
|
||||||
if (projects.Count == 0)
|
|
||||||
{
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
var blocks = projects.Select(project =>
|
|
||||||
$"# {project.Name}\n\n{project.Instructions.Trim()}");
|
|
||||||
return "---\nprojects:\n\n" + string.Join("\n\n", blocks);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<List<ProjectInstructions>> GetBoundProjectsWithInstructionsAsync(
|
|
||||||
MeetingSessionArtifacts artifacts,
|
|
||||||
MeetingAssistantOptions options,
|
|
||||||
CancellationToken cancellationToken)
|
|
||||||
{
|
{
|
||||||
var resolver = ReferenceEquals(options, this.options)
|
var resolver = ReferenceEquals(options, this.options)
|
||||||
? projectResolver
|
? projectResolver
|
||||||
: new BoundMeetingProjectResolver(options);
|
: new BoundMeetingProjectResolver(options);
|
||||||
var projects = new List<ProjectInstructions>();
|
var blocks = new List<string>();
|
||||||
foreach (var project in await resolver.GetBoundProjectsAsync(artifacts, cancellationToken))
|
foreach (var project in await resolver.GetBoundProjectsAsync(artifacts, cancellationToken))
|
||||||
{
|
{
|
||||||
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
|
blocks.Add(await ProjectMetadata.BuildContextAsync(project, cancellationToken));
|
||||||
if (!File.Exists(agentsPath))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var content = await File.ReadAllTextAsync(agentsPath, cancellationToken);
|
|
||||||
if (!string.IsNullOrWhiteSpace(content))
|
|
||||||
{
|
|
||||||
projects.Add(new ProjectInstructions(project.Name, content));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return blocks.Count == 0 ? "" : "---\nprojects:\n\n" + string.Join("\n\n", blocks);
|
||||||
return projects;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed record ProjectInstructions(string Name, string Instructions);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ using MeetingAssistant.MeetingNotes;
|
|||||||
using MeetingAssistant.Speakers;
|
using MeetingAssistant.Speakers;
|
||||||
using MeetingAssistant.Transcription;
|
using MeetingAssistant.Transcription;
|
||||||
using MeetingAssistant.Workflow;
|
using MeetingAssistant.Workflow;
|
||||||
|
using System.Text.Json;
|
||||||
|
using MeetingAssistant.Notifications;
|
||||||
|
using YamlDotNet.RepresentationModel;
|
||||||
using YamlDotNet.Serialization;
|
using YamlDotNet.Serialization;
|
||||||
|
|
||||||
namespace MeetingAssistant.Summary;
|
namespace MeetingAssistant.Summary;
|
||||||
@@ -16,6 +19,8 @@ public sealed class MeetingSummaryTools
|
|||||||
private readonly SummaryAgentWriteAudit? writeAudit;
|
private readonly SummaryAgentWriteAudit? writeAudit;
|
||||||
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
||||||
private readonly BoundMeetingProjectResolver projectResolver;
|
private readonly BoundMeetingProjectResolver projectResolver;
|
||||||
|
private readonly AgentApprovalService? approvals;
|
||||||
|
private readonly WorkflowChangeRequestService? workflowRequests;
|
||||||
|
|
||||||
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
|
public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
|
||||||
: this(artifacts, new MeetingAssistantOptions(), null)
|
: this(artifacts, new MeetingAssistantOptions(), null)
|
||||||
@@ -27,7 +32,9 @@ public sealed class MeetingSummaryTools
|
|||||||
MeetingAssistantOptions options,
|
MeetingAssistantOptions options,
|
||||||
IDictationWordStore? dictationWordStore = null,
|
IDictationWordStore? dictationWordStore = null,
|
||||||
SummaryAgentWriteAudit? writeAudit = null,
|
SummaryAgentWriteAudit? writeAudit = null,
|
||||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
|
||||||
|
AgentApprovalService? approvals = null,
|
||||||
|
WorkflowChangeRequestService? workflowRequests = null)
|
||||||
{
|
{
|
||||||
this.artifacts = artifacts;
|
this.artifacts = artifacts;
|
||||||
this.options = options;
|
this.options = options;
|
||||||
@@ -35,6 +42,8 @@ public sealed class MeetingSummaryTools
|
|||||||
this.writeAudit = writeAudit;
|
this.writeAudit = writeAudit;
|
||||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||||
projectResolver = new BoundMeetingProjectResolver(options);
|
projectResolver = new BoundMeetingProjectResolver(options);
|
||||||
|
this.approvals = approvals;
|
||||||
|
this.workflowRequests = workflowRequests;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<string> ReadTranscript(int? @from = null, int? to = null, int? tail = null)
|
public Task<string> ReadTranscript(int? @from = null, int? to = null, int? tail = null)
|
||||||
@@ -291,8 +300,71 @@ public sealed class MeetingSummaryTools
|
|||||||
|
|
||||||
public async Task<string> ListProjects()
|
public async Task<string> ListProjects()
|
||||||
{
|
{
|
||||||
var projects = await GetBoundProjectsAsync();
|
var projects = await Task.WhenAll(projectResolver.GetAllProjects().Select(project => ProjectMetadata.ReadAsync(project)));
|
||||||
return string.Join('\n', projects.Select(project => project.Name));
|
return JsonSerializer.Serialize(projects.Select(project => new
|
||||||
|
{
|
||||||
|
id = project.Id, displayname = project.DisplayName, description = project.Description
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<string> RequestProjectAssociation(string project_id, string reason, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(reason) || reason.Length > 100 || reason.Contains('\n') || reason.Contains('\r'))
|
||||||
|
return "Refused: reason must be a nonempty single line of at most 100 characters.";
|
||||||
|
var project = projectResolver.GetAllProjects()
|
||||||
|
.FirstOrDefault(candidate => string.Equals(candidate.Name, project_id, StringComparison.OrdinalIgnoreCase));
|
||||||
|
if (project is null) return "Refused: unknown project ID.";
|
||||||
|
var associated = await projectResolver.ReadMeetingProjectNamesAsync(artifacts, cancellationToken);
|
||||||
|
if (!associated.Contains(project.Name))
|
||||||
|
{
|
||||||
|
var metadata = await ProjectMetadata.ReadAsync(project, cancellationToken);
|
||||||
|
if (approvals is null || !await approvals.ConfirmAsync("Project association requested",
|
||||||
|
$"Associate {metadata.DisplayName} ({metadata.Id}) with {Path.GetFileName(artifacts.MeetingNotePath)}?\n{reason}",
|
||||||
|
cancellationToken))
|
||||||
|
return "Denied: project association was not approved.";
|
||||||
|
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
if (!Directory.Exists(project.Path) || !File.Exists(artifacts.MeetingNotePath))
|
||||||
|
return "Refused: the project or meeting note no longer exists.";
|
||||||
|
await AddProjectAssociationAsync(project.Name, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
return "Approved: project is associated with this meeting.\n\n"
|
||||||
|
+ await ProjectMetadata.BuildContextAsync(project, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public string RequestWorkflowChange(string intention, string detailed_prompt)
|
||||||
|
=> workflowRequests?.Request(intention, detailed_prompt, artifacts.MeetingNotePath)
|
||||||
|
?? "Refused: workflow change requests are unavailable.";
|
||||||
|
|
||||||
|
private async Task AddProjectAssociationAsync(string projectId, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var content = await File.ReadAllTextAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||||
|
var document = MarkdownDocumentParser.SplitRequired(content,
|
||||||
|
"Meeting note requires frontmatter.", "Meeting note frontmatter is not closed.");
|
||||||
|
var yaml = new YamlStream();
|
||||||
|
yaml.Load(new StringReader(document.Frontmatter));
|
||||||
|
var mapping = (YamlMappingNode)yaml.Documents.Single().RootNode;
|
||||||
|
mapping.Children.TryGetValue(new YamlScalarNode("projects"), out var existing);
|
||||||
|
var projects = existing as YamlSequenceNode ?? new YamlSequenceNode();
|
||||||
|
if (existing is YamlScalarNode scalar && !string.IsNullOrWhiteSpace(scalar.Value))
|
||||||
|
projects.Add(new YamlScalarNode(scalar.Value));
|
||||||
|
if (!projects.Children.OfType<YamlScalarNode>().Any(value => string.Equals(value.Value, projectId, StringComparison.OrdinalIgnoreCase)))
|
||||||
|
projects.Add(new YamlScalarNode(projectId));
|
||||||
|
mapping.Children[new YamlScalarNode("projects")] = projects;
|
||||||
|
using var writer = new StringWriter();
|
||||||
|
yaml.Save(writer, assignAnchors: false);
|
||||||
|
var pendingPath = artifacts.MeetingNotePath + "." + Guid.NewGuid().ToString("N") + ".tmp";
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await File.WriteAllTextAsync(pendingPath, "---\n" + writer + "---\n\n" + document.Body, cancellationToken);
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
File.Replace(pendingPath, artifacts.MeetingNotePath, destinationBackupFileName: null);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(pendingPath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<string> ListProjectFiles(string project)
|
public async Task<string> ListProjectFiles(string project)
|
||||||
@@ -375,10 +447,18 @@ public sealed class MeetingSummaryTools
|
|||||||
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for append; set replace_file=true only for whole-file replacement.";
|
return "Refused: supply either both from and to for replacement, insert for insertion, or no line arguments for append; set replace_file=true only for whole-file replacement.";
|
||||||
}
|
}
|
||||||
|
|
||||||
var target = ResolveExistingProjectFilePath(project, path);
|
var target = await ResolveBoundProjectFileTargetAsync(project, path);
|
||||||
if (target is null)
|
if (target is null)
|
||||||
{
|
{
|
||||||
return "Refused: project does not exist or the path escapes the project folder.";
|
return "Refused: project is not assigned to this meeting, does not exist, or the path escapes the project folder.";
|
||||||
|
}
|
||||||
|
|
||||||
|
var existingContent = File.Exists(target.Path) ? await File.ReadAllTextAsync(target.Path) : "";
|
||||||
|
var updatedContent = AgentFileToolContent.ApplyLineEdit(existingContent, content, editMode);
|
||||||
|
if (string.Equals(Path.GetRelativePath(target.Project.Path, target.Path), "PROJECT.md", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& !ProjectMetadata.IsValidContent(updatedContent))
|
||||||
|
{
|
||||||
|
return "Refused: PROJECT.md requires YAML frontmatter name and description; description must be at most 256 characters.";
|
||||||
}
|
}
|
||||||
|
|
||||||
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
|
Directory.CreateDirectory(Path.GetDirectoryName(target.Path)!);
|
||||||
@@ -386,11 +466,11 @@ public sealed class MeetingSummaryTools
|
|||||||
{
|
{
|
||||||
await writeAudit.CaptureFileWriteAsync(
|
await writeAudit.CaptureFileWriteAsync(
|
||||||
target.Path,
|
target.Path,
|
||||||
() => AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode));
|
() => File.WriteAllTextAsync(target.Path, updatedContent));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
await AgentFileToolContent.WriteFileContentAsync(target.Path, content, editMode);
|
await File.WriteAllTextAsync(target.Path, updatedContent);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $"{target.Project.Name}/{AgentFileToolContent.ToToolPath(path)}";
|
return $"{target.Project.Name}/{AgentFileToolContent.ToToolPath(path)}";
|
||||||
@@ -589,7 +669,7 @@ public sealed class MeetingSummaryTools
|
|||||||
return AgentFileToolContent.IsWithinDirectory(projectRoot, fullPath) ? fullPath : null;
|
return AgentFileToolContent.IsWithinDirectory(projectRoot, fullPath) ? fullPath : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ProjectFileTarget? ResolveExistingProjectFilePath(string project, string path)
|
private async Task<ProjectFileTarget?> ResolveBoundProjectFileTargetAsync(string project, string path)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(project) ||
|
if (string.IsNullOrWhiteSpace(project) ||
|
||||||
string.IsNullOrWhiteSpace(path) ||
|
string.IsNullOrWhiteSpace(path) ||
|
||||||
@@ -598,15 +678,7 @@ public sealed class MeetingSummaryTools
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
var projectsRoot = GetProjectsRoot();
|
var projectFolder = await ResolveBoundProjectAsync(project);
|
||||||
if (!Directory.Exists(projectsRoot))
|
|
||||||
{
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
var projectFolder = Directory.EnumerateDirectories(projectsRoot)
|
|
||||||
.Select(candidate => new BoundMeetingProject(Path.GetFileName(candidate), candidate))
|
|
||||||
.FirstOrDefault(candidate => string.Equals(candidate.Name, project, StringComparison.OrdinalIgnoreCase));
|
|
||||||
if (projectFolder is null)
|
if (projectFolder is null)
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using MeetingAssistant.MeetingNotes;
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Notifications;
|
||||||
using MeetingAssistant.Transcription;
|
using MeetingAssistant.Transcription;
|
||||||
using MeetingAssistant.Workflow;
|
using MeetingAssistant.Workflow;
|
||||||
using Microsoft.Agents.AI;
|
using Microsoft.Agents.AI;
|
||||||
@@ -19,6 +20,8 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
private readonly IMeetingSummaryInstructionBuilder instructionBuilder;
|
private readonly IMeetingSummaryInstructionBuilder instructionBuilder;
|
||||||
private readonly IDictationWordStore dictationWordStore;
|
private readonly IDictationWordStore dictationWordStore;
|
||||||
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
||||||
|
private readonly AgentApprovalService approvals;
|
||||||
|
private readonly WorkflowChangeRequestService workflowRequests;
|
||||||
|
|
||||||
public OpenAiMeetingSummaryAgentPipeline(
|
public OpenAiMeetingSummaryAgentPipeline(
|
||||||
IOptions<MeetingAssistantOptions> options,
|
IOptions<MeetingAssistantOptions> options,
|
||||||
@@ -28,7 +31,9 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
IMeetingSummaryFailureWriter failureWriter,
|
IMeetingSummaryFailureWriter failureWriter,
|
||||||
IMeetingSummaryInstructionBuilder instructionBuilder,
|
IMeetingSummaryInstructionBuilder instructionBuilder,
|
||||||
IDictationWordStore dictationWordStore,
|
IDictationWordStore dictationWordStore,
|
||||||
IMeetingWorkflowEngine meetingWorkflowEngine)
|
IMeetingWorkflowEngine meetingWorkflowEngine,
|
||||||
|
AgentApprovalService approvals,
|
||||||
|
WorkflowChangeRequestService workflowRequests)
|
||||||
{
|
{
|
||||||
this.options = options.Value;
|
this.options = options.Value;
|
||||||
this.loggerFactory = loggerFactory;
|
this.loggerFactory = loggerFactory;
|
||||||
@@ -38,6 +43,8 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
this.instructionBuilder = instructionBuilder;
|
this.instructionBuilder = instructionBuilder;
|
||||||
this.dictationWordStore = dictationWordStore;
|
this.dictationWordStore = dictationWordStore;
|
||||||
this.meetingWorkflowEngine = meetingWorkflowEngine;
|
this.meetingWorkflowEngine = meetingWorkflowEngine;
|
||||||
|
this.approvals = approvals;
|
||||||
|
this.workflowRequests = workflowRequests;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<MeetingSummaryRunResult> RunAsync(
|
public async Task<MeetingSummaryRunResult> RunAsync(
|
||||||
@@ -60,35 +67,33 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
options,
|
options,
|
||||||
dictationWordStore,
|
dictationWordStore,
|
||||||
writeAudit,
|
writeAudit,
|
||||||
meetingWorkflowEngine);
|
meetingWorkflowEngine,
|
||||||
|
approvals,
|
||||||
|
workflowRequests);
|
||||||
var tools = CreateTools(meetingTools);
|
var tools = CreateTools(meetingTools);
|
||||||
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
|
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
|
||||||
using var compactionSummaryClient = new LiteLlmResponsesChatClient(
|
using var compactionSummaryClient = LiteLlmResponsesChatClient.Create(
|
||||||
new Uri(agentOptions.Endpoint),
|
agentOptions,
|
||||||
key,
|
key,
|
||||||
agentOptions.Model,
|
|
||||||
agentOptions.EnableThinking,
|
|
||||||
ToReasoningEffortValue(agentOptions.ReasoningEffort),
|
|
||||||
agentOptions.ReconnectionAttempts,
|
|
||||||
agentOptions.ReconnectionDelay,
|
|
||||||
compactionOptions: null,
|
compactionOptions: null,
|
||||||
logger,
|
logger,
|
||||||
firstRequestIsUser: false);
|
firstRequestIsUser: false);
|
||||||
var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient);
|
var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient);
|
||||||
using var chatClient = new LiteLlmResponsesChatClient(
|
using var chatClient = LiteLlmResponsesChatClient.Create(
|
||||||
new Uri(agentOptions.Endpoint),
|
agentOptions,
|
||||||
key,
|
key,
|
||||||
agentOptions.Model,
|
|
||||||
agentOptions.EnableThinking,
|
|
||||||
ToReasoningEffortValue(agentOptions.ReasoningEffort),
|
|
||||||
agentOptions.ReconnectionAttempts,
|
|
||||||
agentOptions.ReconnectionDelay,
|
|
||||||
compactionOptions,
|
compactionOptions,
|
||||||
logger,
|
logger,
|
||||||
firstRequestIsUser: false);
|
firstRequestIsUser: false);
|
||||||
var agent = chatClient
|
var agent = chatClient
|
||||||
.AsBuilder()
|
.AsBuilder()
|
||||||
.UseFunctionInvocation()
|
.UseFunctionInvocation(
|
||||||
|
loggerFactory,
|
||||||
|
client =>
|
||||||
|
{
|
||||||
|
client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync;
|
||||||
|
client.AllowConcurrentInvocation = false;
|
||||||
|
})
|
||||||
.Build()
|
.Build()
|
||||||
.AsAIAgent(
|
.AsAIAgent(
|
||||||
instructions,
|
instructions,
|
||||||
@@ -184,7 +189,15 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
AIFunctionFactory.Create(
|
AIFunctionFactory.Create(
|
||||||
tools.ListProjects,
|
tools.ListProjects,
|
||||||
"list_projects",
|
"list_projects",
|
||||||
"List the configured project folders bound to the current meeting note frontmatter."),
|
"List all configured projects with id, displayname and description. Catalog visibility does not grant access; project files require current meeting association."),
|
||||||
|
AIFunctionFactory.Create(
|
||||||
|
tools.RequestProjectAssociation,
|
||||||
|
"request_project_association",
|
||||||
|
"Request user approval to associate a project_id with this meeting. reason must be one line and at most 100 characters. Waits up to ten minutes; approval adds the project and returns its AGENTS.md if present."),
|
||||||
|
AIFunctionFactory.Create(
|
||||||
|
tools.RequestWorkflowChange,
|
||||||
|
"request_workflow_change",
|
||||||
|
"Request a workflow change with a one-line intention (maximum 100 characters) and a detailed_prompt describing the desired scenario/specification. Returns requested immediately; user approval separately opens and starts the settings agent."),
|
||||||
AIFunctionFactory.Create(
|
AIFunctionFactory.Create(
|
||||||
tools.ListProjectFiles,
|
tools.ListProjectFiles,
|
||||||
"list_projectfiles",
|
"list_projectfiles",
|
||||||
@@ -196,7 +209,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
AIFunctionFactory.Create(
|
AIFunctionFactory.Create(
|
||||||
tools.WriteProjectFile,
|
tools.WriteProjectFile,
|
||||||
"write_projectfile",
|
"write_projectfile",
|
||||||
"Write a file inside an existing project folder. With no line arguments, append or create the file. Set replace_file=true only when intentionally replacing the whole file. With from and to, replace that inclusive 1-based line range. With insert, insert content at that 1-based line position."),
|
"Write a file inside a project associated with the current meeting. Root PROJECT.md requires YAML name and description (maximum 256 characters). With no line arguments, append or create. Set replace_file=true only for intentional whole-file replacement. With from and to, replace that inclusive 1-based line range; with insert, insert at that line."),
|
||||||
AIFunctionFactory.Create(
|
AIFunctionFactory.Create(
|
||||||
tools.ListPastProjectMeetings,
|
tools.ListPastProjectMeetings,
|
||||||
"list_past_project_meetings",
|
"list_past_project_meetings",
|
||||||
@@ -312,18 +325,6 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
|
|||||||
return new PipelineCompactionStrategy(strategies);
|
return new PipelineCompactionStrategy(strategies);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ToReasoningEffortValue(ReasoningEffortOption effort)
|
|
||||||
{
|
|
||||||
return effort switch
|
|
||||||
{
|
|
||||||
ReasoningEffortOption.None => "none",
|
|
||||||
ReasoningEffortOption.Low => "low",
|
|
||||||
ReasoningEffortOption.High => "high",
|
|
||||||
ReasoningEffortOption.ExtraHigh => "xhigh",
|
|
||||||
_ => "medium"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort)
|
private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort)
|
||||||
{
|
{
|
||||||
return effort switch
|
return effort switch
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using YamlDotNet.Core;
|
||||||
|
using YamlDotNet.Serialization;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Summary;
|
||||||
|
|
||||||
|
public sealed record ProjectMetadata(string Id, string DisplayName, string Description)
|
||||||
|
{
|
||||||
|
public static async Task<string> BuildContextAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var metadata = await ReadAsync(project, cancellationToken);
|
||||||
|
var context = $"# {metadata.Id}\nDisplay name: {metadata.DisplayName}\nDescription: {metadata.Description}";
|
||||||
|
var agentsPath = Path.Combine(project.Path, "AGENTS.md");
|
||||||
|
if (File.Exists(agentsPath))
|
||||||
|
{
|
||||||
|
var instructions = await File.ReadAllTextAsync(agentsPath, cancellationToken);
|
||||||
|
if (!string.IsNullOrWhiteSpace(instructions)) context += "\n\n" + instructions.Trim();
|
||||||
|
}
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool IsValidContent(string content) => Parse(content) is not null;
|
||||||
|
|
||||||
|
public static async Task<ProjectMetadata> ReadAsync(BoundMeetingProject project, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(project.Path, "PROJECT.md");
|
||||||
|
var metadata = File.Exists(path) ? Parse(await File.ReadAllTextAsync(path, cancellationToken)) : null;
|
||||||
|
return new(project.Name,
|
||||||
|
string.IsNullOrWhiteSpace(metadata?.Name) ? project.Name : metadata.Name.Trim(),
|
||||||
|
metadata?.Description?.Trim() ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Frontmatter? Parse(string content)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var document = MarkdownDocumentParser.SplitOptional(content);
|
||||||
|
if (!document.HasFrontmatter) return null;
|
||||||
|
var metadata = FrontmatterYamlDeserializer.Create().Deserialize<Frontmatter>(document.Frontmatter);
|
||||||
|
return string.IsNullOrWhiteSpace(metadata?.Name) || metadata.Description is null || metadata.Description.Length > 256
|
||||||
|
? null
|
||||||
|
: metadata;
|
||||||
|
}
|
||||||
|
catch (YamlException) { return null; }
|
||||||
|
catch (InvalidDataException) { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class Frontmatter
|
||||||
|
{
|
||||||
|
[YamlMember(Alias = "name")]
|
||||||
|
public string? Name { get; set; }
|
||||||
|
[YamlMember(Alias = "description")]
|
||||||
|
public string? Description { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
using MeetingAssistant.Notifications;
|
||||||
|
using MeetingAssistant.Workflow;
|
||||||
|
using System.Threading.Channels;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Summary;
|
||||||
|
|
||||||
|
public sealed class WorkflowChangeRequestService(
|
||||||
|
AgentApprovalService approvals,
|
||||||
|
IWorkflowRulesEditorWindowService editor,
|
||||||
|
ILogger<WorkflowChangeRequestService> logger) : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly Channel<RequestDetails> requests = Channel.CreateUnbounded<RequestDetails>(
|
||||||
|
new UnboundedChannelOptions { SingleReader = true });
|
||||||
|
|
||||||
|
public string Request(string intention, string detailedPrompt, string meetingNotePath)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(intention) || intention.Length > 100 || intention.Contains('\r') || intention.Contains('\n'))
|
||||||
|
return "Refused: intention must be a nonempty single line of at most 100 characters.";
|
||||||
|
if (string.IsNullOrWhiteSpace(detailedPrompt))
|
||||||
|
return "Refused: supply a detailed change prompt, preferably a scenario or specification.";
|
||||||
|
var request = new RequestDetails(Guid.NewGuid().ToString("N"), intention, detailedPrompt, meetingNotePath);
|
||||||
|
return requests.Writer.TryWrite(request) ? $"Requested: {request.Id}. User approval is pending."
|
||||||
|
: "Refused: application is stopping.";
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
var pending = new List<Task>();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await foreach (var request in requests.Reader.ReadAllAsync(stoppingToken))
|
||||||
|
{
|
||||||
|
pending.RemoveAll(task => task.IsCompleted);
|
||||||
|
pending.Add(HandleAsync(request, stoppingToken));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await Task.WhenAll(pending);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task HandleAsync(RequestDetails request, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (await approvals.ConfirmAsync($"Workflow change requested: {request.Intention}",
|
||||||
|
$"The summarizer for {Path.GetFileName(request.MeetingNotePath)} requests a workflow change. Open the settings agent to apply it?",
|
||||||
|
cancellationToken))
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
editor.ShowWithPrompt(request.DetailedPrompt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { }
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "Workflow request {RequestId} could not be handled", request.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override Task StopAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
requests.Writer.TryComplete();
|
||||||
|
return base.StopAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record RequestDetails(string Id, string Intention, string DetailedPrompt, string MeetingNotePath);
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ public enum MeetingTaskbarAction
|
|||||||
OpenSubmenu,
|
OpenSubmenu,
|
||||||
StartRecording,
|
StartRecording,
|
||||||
StopRecording,
|
StopRecording,
|
||||||
|
PauseTranscription,
|
||||||
|
UnpauseTranscription,
|
||||||
AbortRecording,
|
AbortRecording,
|
||||||
SwitchProfile,
|
SwitchProfile,
|
||||||
SelectMicrophone,
|
SelectMicrophone,
|
||||||
@@ -26,7 +28,8 @@ public sealed record MeetingTaskbarMenuItem(
|
|||||||
string? ProfileName = null,
|
string? ProfileName = null,
|
||||||
string? MicrophoneDeviceId = null,
|
string? MicrophoneDeviceId = null,
|
||||||
bool IsChecked = false,
|
bool IsChecked = false,
|
||||||
IReadOnlyList<MeetingTaskbarMenuItem>? Items = null);
|
IReadOnlyList<MeetingTaskbarMenuItem>? Items = null,
|
||||||
|
bool StartsSection = false);
|
||||||
|
|
||||||
public static class MeetingTaskbarMenuBuilder
|
public static class MeetingTaskbarMenuBuilder
|
||||||
{
|
{
|
||||||
@@ -41,23 +44,38 @@ public static class MeetingTaskbarMenuBuilder
|
|||||||
new("Open agent", MeetingTaskbarAction.EditRules)
|
new("Open agent", MeetingTaskbarAction.EditRules)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (status.IsRecording)
|
||||||
|
{
|
||||||
|
items.Add(new MeetingTaskbarMenuItem(
|
||||||
|
"Finish meeting",
|
||||||
|
MeetingTaskbarAction.StopRecording,
|
||||||
|
StartsSection: true));
|
||||||
|
}
|
||||||
|
|
||||||
|
var secondaryControls = new List<MeetingTaskbarMenuItem>();
|
||||||
|
if (status.IsRecording)
|
||||||
|
{
|
||||||
|
secondaryControls.Add(new MeetingTaskbarMenuItem(
|
||||||
|
status.IsPaused ? "Unpause transcription" : "Pause transcription",
|
||||||
|
status.IsPaused
|
||||||
|
? MeetingTaskbarAction.UnpauseTranscription
|
||||||
|
: MeetingTaskbarAction.PauseTranscription));
|
||||||
|
}
|
||||||
|
|
||||||
if (microphones is { Count: > 0 })
|
if (microphones is { Count: > 0 })
|
||||||
{
|
{
|
||||||
items.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
|
secondaryControls.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (status.IsRecording)
|
if (status.IsRecording)
|
||||||
{
|
{
|
||||||
items.Add(new MeetingTaskbarMenuItem(
|
secondaryControls.Add(new MeetingTaskbarMenuItem(
|
||||||
"Stop meeting recording and transcribe",
|
|
||||||
MeetingTaskbarAction.StopRecording));
|
|
||||||
items.Add(new MeetingTaskbarMenuItem(
|
|
||||||
"Cancel meeting recording and discard",
|
"Cancel meeting recording and discard",
|
||||||
MeetingTaskbarAction.AbortRecording));
|
MeetingTaskbarAction.AbortRecording));
|
||||||
|
|
||||||
foreach (var profile in launchProfiles.Where(profile => !IsActiveProfile(profile, status)))
|
foreach (var profile in launchProfiles.Where(profile => !IsActiveProfile(profile, status)))
|
||||||
{
|
{
|
||||||
items.Add(new MeetingTaskbarMenuItem(
|
secondaryControls.Add(new MeetingTaskbarMenuItem(
|
||||||
AppendHotkey($"Switch to {profile.Name}", profile.Options.Hotkey.Toggle),
|
AppendHotkey($"Switch to {profile.Name}", profile.Options.Hotkey.Toggle),
|
||||||
MeetingTaskbarAction.SwitchProfile,
|
MeetingTaskbarAction.SwitchProfile,
|
||||||
profile.Name));
|
profile.Name));
|
||||||
@@ -67,16 +85,18 @@ public static class MeetingTaskbarMenuBuilder
|
|||||||
{
|
{
|
||||||
foreach (var profile in launchProfiles)
|
foreach (var profile in launchProfiles)
|
||||||
{
|
{
|
||||||
items.Add(new MeetingTaskbarMenuItem(
|
secondaryControls.Add(new MeetingTaskbarMenuItem(
|
||||||
AppendHotkey($"Start meeting recording ({profile.Name})", profile.Options.Hotkey.Toggle),
|
AppendHotkey($"Start meeting recording ({profile.Name})", profile.Options.Hotkey.Toggle),
|
||||||
MeetingTaskbarAction.StartRecording,
|
MeetingTaskbarAction.StartRecording,
|
||||||
profile.Name));
|
profile.Name));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
AddSection(items, secondaryControls);
|
||||||
items.Add(new MeetingTaskbarMenuItem(
|
items.Add(new MeetingTaskbarMenuItem(
|
||||||
"Exit",
|
"Exit",
|
||||||
MeetingTaskbarAction.Exit));
|
MeetingTaskbarAction.Exit,
|
||||||
|
StartsSection: true));
|
||||||
|
|
||||||
return new MeetingTaskbarMenu(
|
return new MeetingTaskbarMenu(
|
||||||
status.State,
|
status.State,
|
||||||
@@ -102,6 +122,19 @@ public static class MeetingTaskbarMenuBuilder
|
|||||||
Items: microphoneItems);
|
Items: microphoneItems);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void AddSection(
|
||||||
|
List<MeetingTaskbarMenuItem> items,
|
||||||
|
IReadOnlyList<MeetingTaskbarMenuItem> section)
|
||||||
|
{
|
||||||
|
if (section.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
items.Add(section[0] with { StartsSection = true });
|
||||||
|
items.AddRange(section.Skip(1));
|
||||||
|
}
|
||||||
|
|
||||||
private static string BuildTooltip(RecordingStatus status)
|
private static string BuildTooltip(RecordingStatus status)
|
||||||
{
|
{
|
||||||
return status.State switch
|
return status.State switch
|
||||||
|
|||||||
@@ -196,14 +196,12 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
|||||||
var popupMenu = new PopupMenu();
|
var popupMenu = new PopupMenu();
|
||||||
for (var index = 0; index < menu.Items.Count; index++)
|
for (var index = 0; index < menu.Items.Count; index++)
|
||||||
{
|
{
|
||||||
if (index == 1 ||
|
var menuItem = menu.Items[index];
|
||||||
(menu.Items[index].Action == MeetingTaskbarAction.Exit &&
|
if (index > 0 && menuItem.StartsSection)
|
||||||
menu.Items[index - 1].Action != MeetingTaskbarAction.EditRules))
|
|
||||||
{
|
{
|
||||||
popupMenu.Items.Add(new PopupMenuSeparator());
|
popupMenu.Items.Add(new PopupMenuSeparator());
|
||||||
}
|
}
|
||||||
|
|
||||||
var menuItem = menu.Items[index];
|
|
||||||
popupMenu.Items.Add(BuildPopupItem(menuItem));
|
popupMenu.Items.Add(BuildPopupItem(menuItem));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,6 +249,12 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
|||||||
case MeetingTaskbarAction.StopRecording:
|
case MeetingTaskbarAction.StopRecording:
|
||||||
await coordinator.StopAsync(CancellationToken.None);
|
await coordinator.StopAsync(CancellationToken.None);
|
||||||
break;
|
break;
|
||||||
|
case MeetingTaskbarAction.PauseTranscription:
|
||||||
|
await coordinator.SetTranscriptionPausedAsync(true, CancellationToken.None);
|
||||||
|
break;
|
||||||
|
case MeetingTaskbarAction.UnpauseTranscription:
|
||||||
|
await coordinator.SetTranscriptionPausedAsync(false, CancellationToken.None);
|
||||||
|
break;
|
||||||
case MeetingTaskbarAction.AbortRecording:
|
case MeetingTaskbarAction.AbortRecording:
|
||||||
await coordinator.AbortAsync(CancellationToken.None);
|
await coordinator.AbortAsync(CancellationToken.None);
|
||||||
break;
|
break;
|
||||||
@@ -290,7 +294,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
|||||||
return string.Join(
|
return string.Join(
|
||||||
"|",
|
"|",
|
||||||
FlattenMenuItems(menu.Items).Select(item =>
|
FlattenMenuItems(menu.Items).Select(item =>
|
||||||
$"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.Text}"));
|
$"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.StartsSection}:{item.Text}"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IEnumerable<MeetingTaskbarMenuItem> FlattenMenuItems(
|
private static IEnumerable<MeetingTaskbarMenuItem> FlattenMenuItems(
|
||||||
|
|||||||
@@ -96,14 +96,18 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions()
|
private IEnumerable<PyannoteRuntimeOptions> GetEnabledDiarizationOptions()
|
||||||
{
|
{
|
||||||
return launchProfiles.GetProfiles()
|
var transcriptionRuntimes = launchProfiles.GetProfiles()
|
||||||
.SelectMany(profile => GetEnabledDiarizationOptions(profile.Options))
|
.SelectMany(profile => GetEnabledTranscriptionDiarizationOptions(profile.Options));
|
||||||
|
var speakerValidationRuntimes = GetEnabledSpeakerValidationDiarizationOptions(
|
||||||
|
launchProfiles.GetRequiredProfile(null).Options);
|
||||||
|
return transcriptionRuntimes
|
||||||
|
.Concat(speakerValidationRuntimes)
|
||||||
.DistinctBy(CreateWarmUpKey);
|
.DistinctBy(CreateWarmUpKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions(
|
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledTranscriptionDiarizationOptions(
|
||||||
MeetingAssistantOptions options)
|
MeetingAssistantOptions options)
|
||||||
{
|
{
|
||||||
if (options.Recording.TranscriptionProvider.Equals("whisper-local", StringComparison.OrdinalIgnoreCase) &&
|
if (options.Recording.TranscriptionProvider.Equals("whisper-local", StringComparison.OrdinalIgnoreCase) &&
|
||||||
@@ -111,15 +115,19 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
|
|||||||
{
|
{
|
||||||
yield return options.WhisperLocal.Diarization;
|
yield return options.WhisperLocal.Diarization;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (options.SpeakerIdentification.PyannoteValidation.Enabled &&
|
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledSpeakerValidationDiarizationOptions(
|
||||||
options.SpeakerIdentification.PyannoteValidation.Diarization.Enabled)
|
MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
if (!options.SpeakerIdentification.Resemblyzer.Enabled &&
|
||||||
|
options.SpeakerIdentification.PyannoteValidation.Enabled)
|
||||||
{
|
{
|
||||||
yield return options.SpeakerIdentification.PyannoteValidation.Diarization;
|
yield return options.SpeakerIdentification.PyannoteValidation.Diarization;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string CreateWarmUpKey(PyannoteDiarizationOptions diarization)
|
private static string CreateWarmUpKey(PyannoteRuntimeOptions diarization)
|
||||||
{
|
{
|
||||||
return string.Join(
|
return string.Join(
|
||||||
'\u001f',
|
'\u001f',
|
||||||
|
|||||||
@@ -28,7 +28,12 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
return await FinalizeAsync(
|
if (!options.WhisperLocal.Diarization.Enabled)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return await FinalizeEnabledAsync(
|
||||||
audioPath,
|
audioPath,
|
||||||
liveSegments,
|
liveSegments,
|
||||||
options.WhisperLocal.Diarization,
|
options.WhisperLocal.Diarization,
|
||||||
@@ -36,14 +41,14 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
|
internal async Task<IReadOnlyList<TranscriptionSegment>> FinalizeEnabledAsync(
|
||||||
string audioPath,
|
string audioPath,
|
||||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (!diarization.Enabled || liveSegments.Count == 0)
|
if (liveSegments.Count == 0)
|
||||||
{
|
{
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
@@ -83,14 +88,9 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async Task WarmUpAsync(
|
public async Task WarmUpAsync(
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (!diarization.Enabled)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var token = ResolveToken(diarization);
|
var token = ResolveToken(diarization);
|
||||||
if (string.IsNullOrWhiteSpace(token))
|
if (string.IsNullOrWhiteSpace(token))
|
||||||
{
|
{
|
||||||
@@ -137,7 +137,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
private async Task<CommandResult> RunDiarizationAsync(
|
private async Task<CommandResult> RunDiarizationAsync(
|
||||||
string fullAudioPath,
|
string fullAudioPath,
|
||||||
string token,
|
string token,
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
@@ -171,7 +171,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async Task EnsureDockerImageAsync(
|
private async Task EnsureDockerImageAsync(
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
string modelsFolder,
|
string modelsFolder,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
@@ -198,7 +198,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
|
private static string BuildDockerfile(PyannoteRuntimeOptions diarization)
|
||||||
{
|
{
|
||||||
return
|
return
|
||||||
$"FROM {diarization.BaseImage}\n"
|
$"FROM {diarization.BaseImage}\n"
|
||||||
@@ -211,7 +211,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
private string[] BuildDockerArguments(
|
private string[] BuildDockerArguments(
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
string fullAudioPath,
|
string fullAudioPath,
|
||||||
string modelsFolder,
|
string modelsFolder,
|
||||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||||
@@ -242,7 +242,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
private string[] BuildWarmUpDockerArguments(
|
private string[] BuildWarmUpDockerArguments(
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
string modelsFolder)
|
string modelsFolder)
|
||||||
{
|
{
|
||||||
return
|
return
|
||||||
@@ -269,7 +269,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static string BuildPythonCommand(
|
private static string BuildPythonCommand(
|
||||||
PyannoteDiarizationOptions diarization,
|
PyannoteRuntimeOptions diarization,
|
||||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||||
{
|
{
|
||||||
var model = diarization.Model;
|
var model = diarization.Model;
|
||||||
@@ -296,7 +296,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
+ "python /tmp/meeting_assistant_pyannote.py";
|
+ "python /tmp/meeting_assistant_pyannote.py";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string BuildWarmUpPythonCommand(PyannoteDiarizationOptions diarization)
|
private static string BuildWarmUpPythonCommand(PyannoteRuntimeOptions diarization)
|
||||||
{
|
{
|
||||||
var model = diarization.Model;
|
var model = diarization.Model;
|
||||||
return
|
return
|
||||||
@@ -475,7 +475,7 @@ public sealed class PyannoteTranscriptFinalizer
|
|||||||
return turns;
|
return turns;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ResolveToken(PyannoteDiarizationOptions options)
|
private static string ResolveToken(PyannoteRuntimeOptions options)
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrWhiteSpace(options.Token))
|
if (!string.IsNullOrWhiteSpace(options.Token))
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ namespace MeetingAssistant.Workflow;
|
|||||||
public interface IWorkflowRulesEditorWindowService
|
public interface IWorkflowRulesEditorWindowService
|
||||||
{
|
{
|
||||||
void Show();
|
void Show();
|
||||||
|
void ShowWithPrompt(string initialPrompt);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class NoopWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
|
public sealed class NoopWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
|
||||||
@@ -18,4 +19,6 @@ public sealed class NoopWorkflowRulesEditorWindowService : IWorkflowRulesEditorW
|
|||||||
{
|
{
|
||||||
logger.LogInformation("Workflow rules editor UI is only available on Windows");
|
logger.LogInformation("Workflow rules editor UI is only available on Windows");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void ShowWithPrompt(string initialPrompt) => Show();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
private readonly IMeetingMetadataProvider meetingMetadataProvider;
|
private readonly IMeetingMetadataProvider meetingMetadataProvider;
|
||||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||||
private readonly ISpeakerIdentityMergeService identityMergeService;
|
private readonly ISpeakerIdentityMergeService identityMergeService;
|
||||||
|
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||||
private readonly AsrDiagnosticService asrDiagnosticService;
|
private readonly AsrDiagnosticService asrDiagnosticService;
|
||||||
private readonly IConfiguration configuration;
|
private readonly IConfiguration configuration;
|
||||||
|
|
||||||
@@ -38,6 +39,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
IMeetingMetadataProvider meetingMetadataProvider,
|
IMeetingMetadataProvider meetingMetadataProvider,
|
||||||
ILaunchProfileOptionsProvider launchProfiles,
|
ILaunchProfileOptionsProvider launchProfiles,
|
||||||
ISpeakerIdentityMergeService identityMergeService,
|
ISpeakerIdentityMergeService identityMergeService,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||||
AsrDiagnosticService asrDiagnosticService,
|
AsrDiagnosticService asrDiagnosticService,
|
||||||
IConfiguration configuration)
|
IConfiguration configuration)
|
||||||
{
|
{
|
||||||
@@ -51,6 +53,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
this.meetingMetadataProvider = meetingMetadataProvider;
|
this.meetingMetadataProvider = meetingMetadataProvider;
|
||||||
this.launchProfiles = launchProfiles;
|
this.launchProfiles = launchProfiles;
|
||||||
this.identityMergeService = identityMergeService;
|
this.identityMergeService = identityMergeService;
|
||||||
|
this.outlierPruner = outlierPruner;
|
||||||
this.asrDiagnosticService = asrDiagnosticService;
|
this.asrDiagnosticService = asrDiagnosticService;
|
||||||
this.configuration = configuration;
|
this.configuration = configuration;
|
||||||
}
|
}
|
||||||
@@ -78,33 +81,24 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
identityMergeService,
|
identityMergeService,
|
||||||
asrDiagnosticService,
|
asrDiagnosticService,
|
||||||
configuration,
|
configuration,
|
||||||
logger: logger);
|
logger: logger,
|
||||||
|
outlierPruner: outlierPruner);
|
||||||
var messages = conversation
|
var messages = conversation
|
||||||
.Select(ToChatMessage)
|
.Select(ToChatMessage)
|
||||||
.Append(new ChatMessage(ChatRole.User, userMessage.Trim()))
|
.Append(new ChatMessage(ChatRole.User, userMessage.Trim()))
|
||||||
.ToList();
|
.ToList();
|
||||||
var instructions = await instructionBuilder.BuildAsync(options, cancellationToken);
|
var instructions = await instructionBuilder.BuildAsync(options, cancellationToken);
|
||||||
|
|
||||||
using var compactionSummaryClient = new LiteLlmResponsesChatClient(
|
using var compactionSummaryClient = LiteLlmResponsesChatClient.Create(
|
||||||
new Uri(agentOptions.Endpoint),
|
agentOptions,
|
||||||
key,
|
key,
|
||||||
agentOptions.Model,
|
|
||||||
agentOptions.EnableThinking,
|
|
||||||
ToReasoningEffortValue(agentOptions.ReasoningEffort),
|
|
||||||
agentOptions.ReconnectionAttempts,
|
|
||||||
agentOptions.ReconnectionDelay,
|
|
||||||
compactionOptions: null,
|
compactionOptions: null,
|
||||||
logger,
|
logger,
|
||||||
firstRequestIsUser: false);
|
firstRequestIsUser: false);
|
||||||
var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient);
|
var compactionOptions = CreateCompactionOptions(agentOptions, compactionSummaryClient);
|
||||||
using var chatClient = new LiteLlmResponsesChatClient(
|
using var chatClient = LiteLlmResponsesChatClient.Create(
|
||||||
new Uri(agentOptions.Endpoint),
|
agentOptions,
|
||||||
key,
|
key,
|
||||||
agentOptions.Model,
|
|
||||||
agentOptions.EnableThinking,
|
|
||||||
ToReasoningEffortValue(agentOptions.ReasoningEffort),
|
|
||||||
agentOptions.ReconnectionAttempts,
|
|
||||||
agentOptions.ReconnectionDelay,
|
|
||||||
compactionOptions,
|
compactionOptions,
|
||||||
logger,
|
logger,
|
||||||
firstRequestIsUser: true,
|
firstRequestIsUser: true,
|
||||||
@@ -116,8 +110,12 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
{
|
{
|
||||||
client.FunctionInvoker = async (context, token) =>
|
client.FunctionInvoker = async (context, token) =>
|
||||||
{
|
{
|
||||||
activityChanged?.Invoke(WorkflowRulesEditorActivityUpdate.ToolCall(context.Function.Name));
|
if (context.CallContent.Exception is null)
|
||||||
return await context.Function.InvokeAsync(context.Arguments, token);
|
{
|
||||||
|
activityChanged?.Invoke(WorkflowRulesEditorActivityUpdate.ToolCall(context.Function.Name));
|
||||||
|
}
|
||||||
|
|
||||||
|
return await FunctionInvocationGuard.InvokeAsync(context, token);
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
.Build();
|
.Build();
|
||||||
@@ -396,18 +394,6 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
|||||||
$"No {agentName} API key configured. Set MeetingAssistant:WorkflowRulesEditor:Key, MeetingAssistant:Agent:Key, or environment variable '{options.KeyEnv}'.");
|
$"No {agentName} API key configured. Set MeetingAssistant:WorkflowRulesEditor:Key, MeetingAssistant:Agent:Key, or environment variable '{options.KeyEnv}'.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string ToReasoningEffortValue(ReasoningEffortOption effort)
|
|
||||||
{
|
|
||||||
return effort switch
|
|
||||||
{
|
|
||||||
ReasoningEffortOption.None => "none",
|
|
||||||
ReasoningEffortOption.Low => "low",
|
|
||||||
ReasoningEffortOption.High => "high",
|
|
||||||
ReasoningEffortOption.ExtraHigh => "xhigh",
|
|
||||||
_ => "medium"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort)
|
private static ReasoningEffort ToReasoningEffort(ReasoningEffortOption effort)
|
||||||
{
|
{
|
||||||
return effort switch
|
return effort switch
|
||||||
|
|||||||
@@ -25,9 +25,11 @@ public sealed class WorkflowRulesEditorChatViewModel
|
|||||||
|
|
||||||
public event EventHandler? Changed;
|
public event EventHandler? Changed;
|
||||||
|
|
||||||
public async Task SendAsync(CancellationToken cancellationToken = default)
|
public Task SendAsync(CancellationToken cancellationToken = default) => SendPromptAsync(Draft, cancellationToken);
|
||||||
|
|
||||||
|
public async Task SendPromptAsync(string prompt, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var prompt = Draft.Trim();
|
prompt = prompt.Trim();
|
||||||
if (string.IsNullOrWhiteSpace(prompt) || IsThinking)
|
if (string.IsNullOrWhiteSpace(prompt) || IsThinking)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor
|
|||||||
"Log tools can read and search the current application-owned log file and four rotated older files under the temp log folder." + Environment.NewLine + Environment.NewLine +
|
"Log tools can read and search the current application-owned log file and four rotated older files under the temp log folder." + Environment.NewLine + Environment.NewLine +
|
||||||
"Spec tools can search and read copied OpenSpec markdown files from openspec/specs." + Environment.NewLine + Environment.NewLine +
|
"Spec tools can search and read copied OpenSpec markdown files from openspec/specs." + Environment.NewLine + Environment.NewLine +
|
||||||
"Project tools can create project folders and read, write, list, and search files in configured projects." + Environment.NewLine + Environment.NewLine +
|
"Project tools can create project folders and read, write, list, and search files in configured projects." + Environment.NewLine + Environment.NewLine +
|
||||||
"Meeting artifact tools can list recent summaries and read/search/write summaries, transcripts, meeting notes, and assistant context files for note post-processing and repair. Prefer frontmatter-specific write tools for metadata-only fixes." + Environment.NewLine + Environment.NewLine +
|
"Meeting artifact tools can list recent summaries and read/search/write summaries, transcripts, meeting notes, and assistant context files for note post-processing and repair. Prefer frontmatter-specific write tools for metadata-only fixes. Treat each assistant context file as meeting-specific memory. When asked to fix or investigate a meeting or summary, read the matching assistant context for clues about problems, missing information, assumptions, prior fixes, and conclusions. After completing repairs, use write_context to append a concise record of your fixes and conclusions to that assistant context for future work on the meeting." + Environment.NewLine + Environment.NewLine +
|
||||||
"Diagnostic tools mirror the local health, recording status, Outlook metadata, speaker identity merge, workflow reload, and ASR diagnostic endpoints without requiring HTTP." + Environment.NewLine + Environment.NewLine +
|
"Diagnostic tools mirror the local health, recording status, Outlook metadata, speaker identity merge, workflow reload, and ASR diagnostic endpoints without requiring HTTP." + Environment.NewLine + Environment.NewLine +
|
||||||
"Speaker identity tools can search/list/read/update/delete/merge identities, refuse sampleless identity creation, list/read/delete identity samples, and queue a sample for local playback. Do not delete the last sample from an identity; delete the identity instead." + Environment.NewLine + Environment.NewLine +
|
"Speaker identity tools can search/list/read/update/delete/merge identities, refuse sampleless identity creation, list/read/delete identity samples, and queue a sample for local playback. Do not delete the last sample from an identity; delete the identity instead." + Environment.NewLine + Environment.NewLine +
|
||||||
"Workflow rules reference documentation:" + Environment.NewLine +
|
"Workflow rules reference documentation:" + Environment.NewLine +
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ public interface IWorkflowRulesEditorSamplePlaybackQueue
|
|||||||
Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default);
|
Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if WINDOWS
|
||||||
public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue, IDisposable
|
public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue, IDisposable
|
||||||
{
|
{
|
||||||
private readonly Channel<QueuedSample> channel = Channel.CreateUnbounded<QueuedSample>();
|
private readonly Channel<QueuedSample> channel = Channel.CreateUnbounded<QueuedSample>();
|
||||||
@@ -98,3 +99,13 @@ public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEdito
|
|||||||
|
|
||||||
private sealed record QueuedSample(int SampleId, byte[] WavBytes);
|
private sealed record QueuedSample(int SampleId, byte[] WavBytes);
|
||||||
}
|
}
|
||||||
|
#else
|
||||||
|
public sealed class UnavailableWorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue
|
||||||
|
{
|
||||||
|
public Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
return Task.FromResult("Speaker sample playback requires the Windows build of Meeting Assistant.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ using MeetingAssistant.Transcription;
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
using YamlDotNet.Core;
|
using YamlDotNet.Core;
|
||||||
using YamlDotNet.Serialization;
|
using YamlDotNet.Serialization;
|
||||||
|
|
||||||
@@ -42,6 +43,7 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
private readonly IMeetingMetadataProvider? meetingMetadataProvider;
|
private readonly IMeetingMetadataProvider? meetingMetadataProvider;
|
||||||
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
||||||
private readonly ISpeakerIdentityMergeService? identityMergeService;
|
private readonly ISpeakerIdentityMergeService? identityMergeService;
|
||||||
|
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||||
private readonly AsrDiagnosticService? asrDiagnosticService;
|
private readonly AsrDiagnosticService? asrDiagnosticService;
|
||||||
private readonly IConfiguration? configuration;
|
private readonly IConfiguration? configuration;
|
||||||
private readonly ILogger? logger;
|
private readonly ILogger? logger;
|
||||||
@@ -61,7 +63,8 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
string? logDirectory = null,
|
string? logDirectory = null,
|
||||||
string? specRootPath = null,
|
string? specRootPath = null,
|
||||||
string? projectAgentsTemplatePath = null,
|
string? projectAgentsTemplatePath = null,
|
||||||
ILogger? logger = null)
|
ILogger? logger = null,
|
||||||
|
ResemblyzerVoiceVectorOutlierPruner? outlierPruner = null)
|
||||||
{
|
{
|
||||||
this.options = options;
|
this.options = options;
|
||||||
rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath);
|
rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath);
|
||||||
@@ -82,6 +85,9 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
this.meetingMetadataProvider = meetingMetadataProvider;
|
this.meetingMetadataProvider = meetingMetadataProvider;
|
||||||
this.launchProfiles = launchProfiles;
|
this.launchProfiles = launchProfiles;
|
||||||
this.identityMergeService = identityMergeService;
|
this.identityMergeService = identityMergeService;
|
||||||
|
this.outlierPruner = outlierPruner ?? new ResemblyzerVoiceVectorOutlierPruner(
|
||||||
|
speakerOptions.Resemblyzer,
|
||||||
|
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||||
this.asrDiagnosticService = asrDiagnosticService;
|
this.asrDiagnosticService = asrDiagnosticService;
|
||||||
this.configuration = configuration;
|
this.configuration = configuration;
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
@@ -743,8 +749,8 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
string[]? candidateNames = null)
|
string[]? candidateNames = null)
|
||||||
{
|
{
|
||||||
return Task.FromResult(
|
return Task.FromResult(
|
||||||
"Refused: speaker identities require at least one audio sample. " +
|
"Refused: speaker identities require audio evidence (a WAV sample or voice vector). " +
|
||||||
"Use a speaker override from an existing transcript speaker sample, or update/merge an existing sampled identity.");
|
"Use a speaker override from an existing transcript speaker, or update/merge an existing identity.");
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<string> UpdateIdentity(
|
public async Task<string> UpdateIdentity(
|
||||||
@@ -826,10 +832,12 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
return $"Could not find target {targetIdentityId} or source {sourceIdentityId}.";
|
return $"Could not find target {targetIdentityId} or source {sourceIdentityId}.";
|
||||||
}
|
}
|
||||||
|
|
||||||
SpeakerIdentityMerger.MergeInto(
|
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||||
target,
|
target,
|
||||||
source,
|
source,
|
||||||
speakerOptions.MaxSnippetsPerSpeaker);
|
speakerOptions.MaxSnippetsPerSpeaker,
|
||||||
|
speakerOptions.Resemblyzer.MaxVectorsPerIdentity,
|
||||||
|
outlierPruner);
|
||||||
context.SpeakerIdentities.Remove(source);
|
context.SpeakerIdentities.Remove(source);
|
||||||
await context.SaveChangesAsync();
|
await context.SaveChangesAsync();
|
||||||
return ToJson(ToIdentityDetail(target));
|
return ToJson(ToIdentityDetail(target));
|
||||||
@@ -1567,9 +1575,11 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||||
{
|
{
|
||||||
return context.SpeakerIdentities
|
return context.SpeakerIdentities
|
||||||
|
.AsSplitQuery()
|
||||||
.Include(identity => identity.Aliases)
|
.Include(identity => identity.Aliases)
|
||||||
.Include(identity => identity.CandidateNames)
|
.Include(identity => identity.CandidateNames)
|
||||||
.Include(identity => identity.Snippets)
|
.Include(identity => identity.Snippets)
|
||||||
|
.Include(identity => identity.VoiceVectors)
|
||||||
.Include(identity => identity.References);
|
.Include(identity => identity.References);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1591,6 +1601,7 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
identity.Aliases.Select(alias => alias.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
identity.Aliases.Select(alias => alias.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
||||||
identity.CandidateNames.Select(candidate => candidate.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
identity.CandidateNames.Select(candidate => candidate.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
||||||
identity.Snippets.Count,
|
identity.Snippets.Count,
|
||||||
|
identity.VoiceVectors.Count,
|
||||||
identity.References.Count,
|
identity.References.Count,
|
||||||
identity.UpdatedAt);
|
identity.UpdatedAt);
|
||||||
}
|
}
|
||||||
@@ -1650,6 +1661,7 @@ public sealed class WorkflowRulesEditorTools
|
|||||||
IReadOnlyList<string> Aliases,
|
IReadOnlyList<string> Aliases,
|
||||||
IReadOnlyList<string> CandidateNames,
|
IReadOnlyList<string> CandidateNames,
|
||||||
int SampleCount,
|
int SampleCount,
|
||||||
|
int VoiceVectorCount,
|
||||||
int ReferenceCount,
|
int ReferenceCount,
|
||||||
DateTimeOffset UpdatedAt);
|
DateTimeOffset UpdatedAt);
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,26 @@ internal sealed class WpfWorkflowRulesEditorWindowService : IWorkflowRulesEditor
|
|||||||
dispatcher?.BeginInvoke(ShowOnUiThread);
|
dispatcher?.BeginInvoke(ShowOnUiThread);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void ShowWithPrompt(string initialPrompt)
|
||||||
|
{
|
||||||
|
EnsureUiThread();
|
||||||
|
dispatcher?.BeginInvoke(new Action(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var viewModel = services.GetRequiredService<WorkflowRulesEditorChatViewModel>();
|
||||||
|
var requestedWindow = new WorkflowRulesEditorWpfWindow(viewModel, linkResolver, () => { });
|
||||||
|
requestedWindow.Show();
|
||||||
|
requestedWindow.Activate();
|
||||||
|
await viewModel.SendPromptAsync(initialPrompt);
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(exception, "Could not start settings agent for an approved workflow request");
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
private void EnsureUiThread()
|
private void EnsureUiThread()
|
||||||
{
|
{
|
||||||
lock (sync)
|
lock (sync)
|
||||||
|
|||||||
@@ -36,6 +36,7 @@
|
|||||||
"00:10:00"
|
"00:10:00"
|
||||||
],
|
],
|
||||||
"AutoStopAfter": "00:30:00",
|
"AutoStopAfter": "00:30:00",
|
||||||
|
"MaximumPauseDuration": "04:00:00",
|
||||||
"InferredEndPadding": "00:01:00",
|
"InferredEndPadding": "00:01:00",
|
||||||
"CheckInterval": "00:00:15"
|
"CheckInterval": "00:00:15"
|
||||||
}
|
}
|
||||||
@@ -119,18 +120,17 @@
|
|||||||
"MaxMatchCandidates": 100,
|
"MaxMatchCandidates": 100,
|
||||||
"MatchIdentityActiveAge": "365.00:00:00",
|
"MatchIdentityActiveAge": "365.00:00:00",
|
||||||
"MaxSnippetsPerSpeaker": 3,
|
"MaxSnippetsPerSpeaker": 3,
|
||||||
"MinimumSampleSpeechDuration": "00:00:30",
|
"MinimumSampleSpeechDuration": "00:00:10",
|
||||||
"MaximumSampleSegmentGap": "00:00:01",
|
"MaximumSampleDuration": "00:01:00",
|
||||||
"SilenceBetweenSnippetsSeconds": 1,
|
"SilenceBetweenSnippetsSeconds": 1,
|
||||||
"LiveSampleBufferDuration": "00:10:00",
|
"LiveSampleBufferDuration": "00:10:00",
|
||||||
"MergeRecentIdentityAge": "14.00:00:00",
|
"MergeRecentIdentityAge": "14.00:00:00",
|
||||||
"MatchTimeout": "00:03:00",
|
"MatchTimeout": "00:03:00",
|
||||||
"PyannoteValidation": {
|
"PyannoteValidation": {
|
||||||
"Enabled": true,
|
"Enabled": false,
|
||||||
"MinimumSingleSpeakerCoverage": 0.9,
|
"MinimumSingleSpeakerCoverage": 0.9,
|
||||||
"MinimumMatchingKnownSnippetRatio": 0.5,
|
"MinimumMatchingKnownSnippetRatio": 0.5,
|
||||||
"Diarization": {
|
"Diarization": {
|
||||||
"Enabled": false,
|
|
||||||
"DockerCommand": "docker",
|
"DockerCommand": "docker",
|
||||||
"BaseImage": "python:3.11-slim",
|
"BaseImage": "python:3.11-slim",
|
||||||
"Image": "meeting-assistant-pyannote:local",
|
"Image": "meeting-assistant-pyannote:local",
|
||||||
@@ -142,6 +142,26 @@
|
|||||||
"TokenEnv": "HF_TOKEN",
|
"TokenEnv": "HF_TOKEN",
|
||||||
"CommandTimeout": "01:00:00"
|
"CommandTimeout": "01:00:00"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"Resemblyzer": {
|
||||||
|
"Enabled": false,
|
||||||
|
"RequiredVectorsPerSpeaker": 5,
|
||||||
|
"MaxVectorsPerIdentity": 1000,
|
||||||
|
"OutlierPruningMinimumVectors": 20,
|
||||||
|
"OutlierPruningNeighborSimilarity": 0.75,
|
||||||
|
"OutlierPruningMinimumNeighbors": 3,
|
||||||
|
"OutlierPruningMinimumClusterRatio": 0.60,
|
||||||
|
"MinimumClusterCohesion": 0.75,
|
||||||
|
"MinimumIdentitySimilarity": 0.75,
|
||||||
|
"MinimumSimilarityMargin": 0.05,
|
||||||
|
"ModelId": "resemblyzer-0.1.4-pretrained",
|
||||||
|
"PackageVersion": "0.1.4",
|
||||||
|
"PythonCommand": "python",
|
||||||
|
"TorchVersion": "2.14.0+cpu",
|
||||||
|
"TorchIndexUrl": "https://download.pytorch.org/whl/cpu",
|
||||||
|
"WebRtcVadVersion": "2.0.14",
|
||||||
|
"RuntimeFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Resemblyzer",
|
||||||
|
"CommandTimeout": "00:15:00"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"Automation": {
|
"Automation": {
|
||||||
@@ -172,6 +192,7 @@
|
|||||||
"Endpoint": "http://127.0.0.1:4021",
|
"Endpoint": "http://127.0.0.1:4021",
|
||||||
"KeyEnv": "LITELLM_API_KEY",
|
"KeyEnv": "LITELLM_API_KEY",
|
||||||
"Model": "chatgpt/gpt-5.5",
|
"Model": "chatgpt/gpt-5.5",
|
||||||
|
"UseStreaming": true,
|
||||||
"EnableThinking": true,
|
"EnableThinking": true,
|
||||||
"ReasoningEffort": "Medium",
|
"ReasoningEffort": "Medium",
|
||||||
"ReconnectionAttempts": 2,
|
"ReconnectionAttempts": 2,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user