forked from Manuel/meeting-assistant
Compare commits
77
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
723298e720 | ||
|
|
e011e36ef7 | ||
|
|
e17fb8b225 | ||
|
|
c01213278e | ||
|
|
7814f1c7c4 | ||
|
|
88c3b21b2a | ||
|
|
b249e5d977 | ||
|
|
3e4bf88931 | ||
|
|
5e7c9f64d0 | ||
|
|
fbd5a0852f | ||
|
|
568bedb77a | ||
|
|
f21c56b1c7 | ||
|
|
c56de2b7ab | ||
|
|
40d10d634a | ||
|
|
264a3a823f | ||
|
|
072ebf38a3 | ||
|
|
4d66de943e | ||
|
|
150f11b6a7 | ||
|
|
d7db64bec7 | ||
|
|
cdeef1967e | ||
|
|
aeb0cfe42f | ||
|
|
ef0c0faba8 | ||
|
|
93eb3d4e98 | ||
|
|
d93d758a8f | ||
|
|
48ad4fb930 | ||
|
|
cb5a300b44 | ||
|
|
b6acba8402 | ||
|
|
a82e2fff03 | ||
|
|
336d940acd | ||
|
|
cbf854fb2a | ||
|
|
bacef060fe | ||
|
|
a928382e51 | ||
|
|
e23d04a38c | ||
|
|
c6a50d307e | ||
|
|
436fa6c149 | ||
|
|
5dce5fd350 | ||
|
|
c37654a7d9 | ||
|
|
cc9044e091 | ||
|
|
f86af983e8 | ||
|
|
43fc8aaec0 | ||
|
|
8b543a2161 | ||
|
|
58896073c0 | ||
|
|
945543c078 | ||
|
|
3e1f447b20 | ||
|
|
497390c2ec | ||
|
|
b1365b202d | ||
|
|
70ffaa7357 | ||
|
|
84dc696ce9 | ||
|
|
714da456df | ||
|
|
fa67f30c49 | ||
|
|
68db6955a2 | ||
|
|
dc37719d46 | ||
|
|
ee453e170e | ||
|
|
d51463321b | ||
|
|
41ae47fbfe | ||
|
|
f7ad2ce41a | ||
|
|
bc76151926 | ||
|
|
af9f5b34dc | ||
|
|
f3489f8e95 | ||
|
|
39591eef12 | ||
|
|
2af7933a1b | ||
|
|
9542c05ea4 | ||
|
|
5f39923c27 | ||
|
|
13a70c2375 | ||
|
|
59ba70ca11 | ||
|
|
b3aeef24eb | ||
|
|
c91d956beb | ||
|
|
4c858170d9 | ||
|
|
0b6643d87f | ||
|
|
44ce419c49 | ||
|
|
a087ff0cbf | ||
|
|
8dc1b70f8a | ||
|
|
aaee123b9a | ||
|
|
e886fac311 | ||
|
|
e35ccf661f | ||
|
|
20eed38477 | ||
|
|
ada4a40f62 |
@@ -18,7 +18,7 @@ jobs:
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v5
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: "10.0.x"
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
@@ -41,6 +42,30 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
|
||||
Assert.Equal("ok", body?.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStartupRegistersDetectedHostOperatingSystemOnce()
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
using var response = await client.GetAsync("/health");
|
||||
var first = factory.Services.GetRequiredService<HostOperatingSystem>();
|
||||
var second = factory.Services.GetRequiredService<HostOperatingSystem>();
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Same(first, second);
|
||||
Assert.Equal(HostOperatingSystem.DetectCurrent(), first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
|
||||
{
|
||||
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
|
||||
var result = await playback.QueueAsync(1, [1, 2]);
|
||||
|
||||
Assert.Contains("Windows build", result);
|
||||
Assert.DoesNotContain("Queued", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
|
||||
{
|
||||
|
||||
@@ -58,6 +58,64 @@ public sealed class LaunchProfileOptionsProviderTests
|
||||
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
|
||||
{
|
||||
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
|
||||
|
||||
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
|
||||
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
|
||||
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
|
||||
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
|
||||
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
|
||||
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
|
||||
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContradictorySpeakerSampleDurationsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
|
||||
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
|
||||
Assert.Contains("MaximumSampleDuration", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DuplicateProfileHotkeysAreRejected()
|
||||
{
|
||||
|
||||
@@ -2,8 +2,6 @@ using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
@@ -284,15 +282,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
return PngTestFixture.Solid(width, height);
|
||||
}
|
||||
|
||||
private static string CreateNonStreamingTextResponse(string text)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsDesktopControlManifestTests
|
||||
{
|
||||
[Fact]
|
||||
public void MacOsApplicationRegistersARealInteractiveAgentWindow()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var factory = new WebApplicationFactory<Program>();
|
||||
|
||||
var service = factory.Services.GetRequiredService<IWorkflowRulesEditorWindowService>();
|
||||
|
||||
Assert.IsType<MacOsWorkflowRulesEditorWindowService>(service);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManifestPreservesConfiguredWindowsEquivalentShortcutsAndControlEndpoints()
|
||||
{
|
||||
var profiles = new[]
|
||||
{
|
||||
Profile("default", "Ctrl+Alt+M", "Ctrl+Alt+Z", "Ctrl+Alt+S"),
|
||||
Profile("english", "Ctrl+Alt+L", "", "")
|
||||
};
|
||||
|
||||
var hotkeys = new[]
|
||||
{
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+M", LaunchProfileHotkeyAction.ToggleRecording),
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+Z", LaunchProfileHotkeyAction.AbortRecording),
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+S", LaunchProfileHotkeyAction.CaptureScreenshot),
|
||||
new LaunchProfileHotkey("english", "Ctrl+Alt+L", LaunchProfileHotkeyAction.ToggleRecording)
|
||||
};
|
||||
|
||||
var manifest = MacOsDesktopControlManifest.Create(
|
||||
"http://127.0.0.1:5090",
|
||||
profiles,
|
||||
hotkeys);
|
||||
|
||||
Assert.Collection(
|
||||
manifest.Hotkeys,
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+M", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/recording/toggle", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+Z", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/recording/abort", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+S", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/meetings/screenshot/capture", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+L", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/english/recording/toggle", hotkey.Endpoint);
|
||||
});
|
||||
Assert.Equal("http://127.0.0.1:5090/recording/status", manifest.StatusUrl);
|
||||
Assert.Equal("http://127.0.0.1:5090/diagnostics/settings-and-logs", manifest.OpenAgentUrl);
|
||||
}
|
||||
|
||||
private static LaunchProfile Profile(string name, string toggle, string abort, string screenshot)
|
||||
{
|
||||
return new LaunchProfile(name, new MeetingAssistantOptions
|
||||
{
|
||||
Hotkey = new HotkeyOptions
|
||||
{
|
||||
Toggle = toggle,
|
||||
Abort = abort
|
||||
},
|
||||
Screenshots = new ScreenshotOptions
|
||||
{
|
||||
Hotkey = screenshot
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsMeetingAudioSourceTests
|
||||
{
|
||||
[Fact]
|
||||
public void NativeMicrophoneHelperDeclaresMacOsPrivacyMetadata()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var configuration = new DirectoryInfo(AppContext.BaseDirectory)
|
||||
.Parent?.Name ?? "Debug";
|
||||
var helperPath = Path.GetFullPath(Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"MeetingAssistant",
|
||||
"bin",
|
||||
configuration,
|
||||
"net10.0",
|
||||
"Native",
|
||||
"macos-meeting-audio-capture"));
|
||||
var helperImage = System.Text.Encoding.UTF8.GetString(File.ReadAllBytes(helperPath));
|
||||
|
||||
Assert.Contains("cloud.schweigert.meeting-assistant.audio-capture", helperImage, StringComparison.Ordinal);
|
||||
Assert.Contains("NSMicrophoneUsageDescription", helperImage, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsMicrophoneStreamsNativePcmUsingRunFormat()
|
||||
{
|
||||
var pcm = Pcm16(1_000, -2_000, 3_000, -4_000);
|
||||
var process = new StubMacOsAudioCaptureProcess(pcm, exitCode: 0);
|
||||
var processFactory = new StubMacOsAudioCaptureProcessFactory(process);
|
||||
IMeetingAudioSource source = new MacOsMicrophoneAudioSource(
|
||||
processFactory,
|
||||
NullLogger<MacOsMicrophoneAudioSource>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Recording =
|
||||
{
|
||||
SampleRate = 24_000,
|
||||
Channels = 2
|
||||
}
|
||||
};
|
||||
|
||||
var chunks = await ReadChunksAsync(source, options);
|
||||
|
||||
var chunk = Assert.Single(chunks);
|
||||
Assert.Equal(pcm, chunk.Pcm);
|
||||
Assert.Equal(24_000, chunk.SampleRate);
|
||||
Assert.Equal(2, chunk.Channels);
|
||||
Assert.Equal(
|
||||
new MacOsAudioCaptureRequest(MacOsAudioCaptureKind.Microphone, 24_000, 2),
|
||||
processFactory.Request);
|
||||
Assert.True(process.Disposed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsSystemAudioReportsNativePermissionFailure()
|
||||
{
|
||||
var process = new StubMacOsAudioCaptureProcess(
|
||||
[],
|
||||
exitCode: 77,
|
||||
standardError: "Screen Recording/System Audio permission is required.");
|
||||
IMeetingAudioSource source = new MacOsSystemAudioSource(
|
||||
new StubMacOsAudioCaptureProcessFactory(process),
|
||||
NullLogger<MacOsSystemAudioSource>.Instance);
|
||||
|
||||
var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
await ReadChunksAsync(source, new MeetingAssistantOptions()));
|
||||
|
||||
Assert.Contains("Screen Recording/System Audio permission", exception.Message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PortableRegistrationUsesMacOsCompositeOnlyForMacOs()
|
||||
{
|
||||
var macServices = new ServiceCollection().AddLogging();
|
||||
var unsupportedServices = new ServiceCollection().AddLogging();
|
||||
|
||||
PortableMeetingAudioRegistration.Add(macServices, isMacOs: true);
|
||||
PortableMeetingAudioRegistration.Add(unsupportedServices, isMacOs: false);
|
||||
|
||||
using var macProvider = macServices.BuildServiceProvider();
|
||||
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
|
||||
Assert.IsType<CompositeMeetingAudioSource>(macProvider.GetRequiredService<IMeetingAudioSource>());
|
||||
Assert.IsType<UnavailableMeetingAudioSource>(unsupportedProvider.GetRequiredService<IMeetingAudioSource>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses()
|
||||
{
|
||||
var testRoot = Path.Combine(Path.GetTempPath(), $"meeting-assistant-macos-endpoint-{Guid.NewGuid():N}");
|
||||
var nativeProcesses = new EndpointMacOsAudioCaptureProcessFactory(
|
||||
microphonePcm: Pcm16(2_000),
|
||||
systemPcm: Pcm16(10_000));
|
||||
var speechPipelines = new CapturingSpeechRecognitionPipelineFactory();
|
||||
var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:Vault:BaseFolder"] = testRoot,
|
||||
["MeetingAssistant:Vault:TranscriptsFolder"] = "Transcripts",
|
||||
["MeetingAssistant:Vault:MeetingNotesFolder"] = "Notes",
|
||||
["MeetingAssistant:Vault:AssistantContextFolder"] = "Context",
|
||||
["MeetingAssistant:Vault:SummariesFolder"] = "Summaries",
|
||||
["MeetingAssistant:Vault:ProjectsFolder"] = "Projects",
|
||||
["MeetingAssistant:Recording:TemporaryRecordingsFolder"] = Path.Combine(testRoot, "Recordings"),
|
||||
["MeetingAssistant:Recording:InactivitySafeguard:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(testRoot, "speakers.db"),
|
||||
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false"
|
||||
});
|
||||
});
|
||||
builder.ConfigureTestServices(services =>
|
||||
{
|
||||
services.RemoveAll<IMeetingAudioSource>();
|
||||
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||
PortableMeetingAudioRegistration.Add(services, isMacOs: true);
|
||||
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||
services.AddSingleton<IMacOsAudioCaptureProcessFactory>(nativeProcesses);
|
||||
services.RemoveAll<ISpeechRecognitionPipelineFactory>();
|
||||
services.AddSingleton<ISpeechRecognitionPipelineFactory>(speechPipelines);
|
||||
services.RemoveAll<IMeetingNoteOpener>();
|
||||
services.AddSingleton<IMeetingNoteOpener, NoopMeetingNoteOpener>();
|
||||
services.RemoveAll<IMeetingSummaryPipeline>();
|
||||
services.AddSingleton<IMeetingSummaryPipeline, NoopMeetingSummaryPipeline>();
|
||||
});
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
using var startResponse = await client.PostAsync("/recording/start", content: null);
|
||||
var started = await startResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||
var mixedChunk = await speechPipelines.FirstChunk.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.True(startResponse.IsSuccessStatusCode);
|
||||
Assert.True(started?.IsRecording);
|
||||
Assert.Equal(12_000, BitConverter.ToInt16(mixedChunk.Pcm));
|
||||
|
||||
using var stopResponse = await client.PostAsync("/recording/stop", content: null);
|
||||
var stopped = await stopResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||
|
||||
Assert.True(stopResponse.IsSuccessStatusCode);
|
||||
Assert.False(stopped?.IsRecording);
|
||||
Assert.All(nativeProcesses.Processes, process => Assert.True(process.Terminated));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await factory.DisposeAsync();
|
||||
using var connection = new SqliteConnection($"Data Source={Path.Combine(testRoot, "speakers.db")}");
|
||||
SqliteConnection.ClearPool(connection);
|
||||
if (Directory.Exists(testRoot))
|
||||
{
|
||||
Directory.Delete(testRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<AudioChunk>> ReadChunksAsync(
|
||||
IMeetingAudioSource source,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var chunks = new List<AudioChunk>();
|
||||
await foreach (var chunk in source.CaptureAsync(options, CancellationToken.None))
|
||||
{
|
||||
chunks.Add(chunk);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private static byte[] Pcm16(params short[] samples)
|
||||
{
|
||||
var pcm = new byte[samples.Length * sizeof(short)];
|
||||
Buffer.BlockCopy(samples, 0, pcm, 0, pcm.Length);
|
||||
return pcm;
|
||||
}
|
||||
|
||||
private sealed class StubMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly IMacOsAudioCaptureProcess process;
|
||||
|
||||
public StubMacOsAudioCaptureProcessFactory(IMacOsAudioCaptureProcess process)
|
||||
{
|
||||
this.process = process;
|
||||
}
|
||||
|
||||
public MacOsAudioCaptureRequest? Request { get; private set; }
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
Request = request;
|
||||
return process;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly MemoryStream standardOutput;
|
||||
private readonly string standardError;
|
||||
|
||||
public StubMacOsAudioCaptureProcess(byte[] pcm, int exitCode, string standardError = "")
|
||||
{
|
||||
standardOutput = new MemoryStream(pcm);
|
||||
this.standardError = standardError;
|
||||
ExitCode = exitCode;
|
||||
}
|
||||
|
||||
public Stream StandardOutput => standardOutput;
|
||||
|
||||
public int ExitCode { get; }
|
||||
|
||||
public bool HasExited => true;
|
||||
|
||||
public bool Disposed { get; private set; }
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(standardError);
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Disposed = true;
|
||||
standardOutput.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class EndpointMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly IReadOnlyDictionary<MacOsAudioCaptureKind, byte[]> pcmByKind;
|
||||
|
||||
public EndpointMacOsAudioCaptureProcessFactory(byte[] microphonePcm, byte[] systemPcm)
|
||||
{
|
||||
pcmByKind = new Dictionary<MacOsAudioCaptureKind, byte[]>
|
||||
{
|
||||
[MacOsAudioCaptureKind.Microphone] = microphonePcm,
|
||||
[MacOsAudioCaptureKind.System] = systemPcm
|
||||
};
|
||||
}
|
||||
|
||||
public List<BlockingMacOsAudioCaptureProcess> Processes { get; } = [];
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
var process = new BlockingMacOsAudioCaptureProcess(pcmByKind[request.Kind]);
|
||||
Processes.Add(process);
|
||||
return process;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BlockingMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly TaskCompletionSource terminated = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public BlockingMacOsAudioCaptureProcess(byte[] pcm)
|
||||
{
|
||||
StandardOutput = new InitialPcmThenBlockingStream(pcm, terminated.Task);
|
||||
}
|
||||
|
||||
public Stream StandardOutput { get; }
|
||||
|
||||
public int ExitCode => Terminated ? 137 : throw new InvalidOperationException("Capture is still running.");
|
||||
|
||||
public bool HasExited => Terminated;
|
||||
|
||||
public bool Terminated => terminated.Task.IsCompleted;
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult("");
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return terminated.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
terminated.TrySetResult();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
StandardOutput.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InitialPcmThenBlockingStream : Stream
|
||||
{
|
||||
private readonly byte[] pcm;
|
||||
private readonly Task processExit;
|
||||
private bool read;
|
||||
|
||||
public InitialPcmThenBlockingStream(byte[] pcm, Task processExit)
|
||||
{
|
||||
this.pcm = pcm;
|
||||
this.processExit = processExit;
|
||||
}
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position
|
||||
{
|
||||
get => throw new NotSupportedException();
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public override async ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!read)
|
||||
{
|
||||
read = true;
|
||||
pcm.AsSpan().CopyTo(buffer.Span);
|
||||
return pcm.Length;
|
||||
}
|
||||
|
||||
await processExit.WaitAsync(cancellationToken);
|
||||
return 0;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
public override void Flush()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
|
||||
{
|
||||
public TaskCompletionSource<AudioChunk> FirstChunk { get; } =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public ISpeechRecognitionPipeline Create()
|
||||
{
|
||||
return new CapturingSpeechRecognitionPipeline(FirstChunk);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
|
||||
{
|
||||
private readonly TaskCompletionSource<AudioChunk> firstChunk;
|
||||
private readonly TaskCompletionSource completed = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public CapturingSpeechRecognitionPipeline(TaskCompletionSource<AudioChunk> firstChunk)
|
||||
{
|
||||
this.firstChunk = firstChunk;
|
||||
}
|
||||
|
||||
public Task InitializeAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task InitializeAsync(
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task WaitUntilReadyAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
firstChunk.TrySetResult(chunk);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public Task CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
completed.TrySetResult();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await completed.Task.WaitAsync(cancellationToken);
|
||||
yield break;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<TranscriptionSegment>>([]);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
completed.TrySetResult();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopMeetingNoteOpener : IMeetingNoteOpener
|
||||
{
|
||||
public Task OpenAsync(string notePath, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class NoopMeetingSummaryPipeline : IMeetingSummaryPipeline
|
||||
{
|
||||
public Task<MeetingSummaryRunResult> RunAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingSummaryRunResult(artifacts.SummaryPath, "", Succeeded: true));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
using System.Net.Http.Json;
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsMeetingIntegrationTests
|
||||
{
|
||||
[Fact]
|
||||
public void PortableRegistrationUsesRealMacOsProvidersOnlyOnMacOs()
|
||||
{
|
||||
var macServices = new ServiceCollection().AddLogging();
|
||||
var unsupportedServices = new ServiceCollection().AddLogging();
|
||||
|
||||
MacOsMeetingIntegrationRegistration.Add(macServices, isMacOs: true);
|
||||
MacOsMeetingIntegrationRegistration.Add(unsupportedServices, isMacOs: false);
|
||||
|
||||
using var macProvider = macServices.BuildServiceProvider();
|
||||
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
|
||||
|
||||
Assert.IsType<MacOsMeetingMetadataProvider>(
|
||||
macProvider.GetRequiredService<IMeetingMetadataProvider>());
|
||||
Assert.IsType<MacOsCalendarMeetingProvider>(
|
||||
macProvider.GetRequiredService<ICalendarMeetingProvider>());
|
||||
Assert.IsType<MacOsMeetingStartPromptService>(
|
||||
macProvider.GetRequiredService<IMeetingStartPromptService>());
|
||||
Assert.IsType<MacOsDesktopIntegrationClient>(
|
||||
macProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
|
||||
Assert.IsType<MacOsDesktopIntegrationClient>(
|
||||
macProvider.GetRequiredService<IScreenshotImageCropper>());
|
||||
|
||||
Assert.IsType<NoopMeetingMetadataProvider>(
|
||||
unsupportedProvider.GetRequiredService<IMeetingMetadataProvider>());
|
||||
Assert.IsType<NoopCalendarMeetingProvider>(
|
||||
unsupportedProvider.GetRequiredService<ICalendarMeetingProvider>());
|
||||
Assert.IsType<NoopMeetingStartPromptService>(
|
||||
unsupportedProvider.GetRequiredService<IMeetingStartPromptService>());
|
||||
Assert.IsType<UnavailableActiveWindowScreenshotCapture>(
|
||||
unsupportedProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
|
||||
Assert.IsType<UnavailableScreenshotImageCropper>(
|
||||
unsupportedProvider.GetRequiredService<IScreenshotImageCropper>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsCalendarProvidersReturnTeamsMetadataAndPromptCandidate()
|
||||
{
|
||||
var startedAt = new DateTimeOffset(2026, 7, 25, 10, 5, 0, TimeSpan.FromHours(2));
|
||||
var calendarEvent = new MacOsCalendarEvent(
|
||||
"event-1",
|
||||
"Project sync",
|
||||
startedAt.AddMinutes(-5),
|
||||
startedAt.AddMinutes(55),
|
||||
"Microsoft Teams",
|
||||
"Discuss launch readiness.\nJoin Microsoft Teams Meeting",
|
||||
"https://teams.microsoft.com/l/meetup-join/example",
|
||||
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
|
||||
IsCanceled: false);
|
||||
var client = new StubMacOsDesktopIntegrationClient(calendarEvent);
|
||||
IMeetingMetadataProvider metadataProvider = new MacOsMeetingMetadataProvider(
|
||||
client,
|
||||
NullLogger<MacOsMeetingMetadataProvider>.Instance);
|
||||
ICalendarMeetingProvider calendarProvider = new MacOsCalendarMeetingProvider(
|
||||
client,
|
||||
NullLogger<MacOsCalendarMeetingProvider>.Instance);
|
||||
|
||||
var metadata = await metadataProvider.GetCurrentMeetingAsync(
|
||||
startedAt,
|
||||
CancellationToken.None);
|
||||
var candidates = await calendarProvider.GetRecordingPromptCandidatesForDayAsync(
|
||||
DateOnly.FromDateTime(startedAt.Date),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.NotNull(metadata);
|
||||
Assert.Equal("Project sync", metadata.Title);
|
||||
Assert.Equal(
|
||||
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
|
||||
metadata.Attendees);
|
||||
Assert.Equal("Discuss launch readiness.", metadata.Agenda);
|
||||
Assert.Equal(calendarEvent.End, metadata.ScheduledEnd);
|
||||
|
||||
var candidate = Assert.Single(candidates);
|
||||
Assert.Equal("event-1", candidate.Id);
|
||||
Assert.Equal("Project sync", candidate.Subject);
|
||||
Assert.NotNull(candidate.Metadata);
|
||||
Assert.Equal(metadata.Title, candidate.Metadata.Title);
|
||||
Assert.Equal(metadata.Attendees, candidate.Metadata.Attendees);
|
||||
Assert.Equal(metadata.Agenda, candidate.Metadata.Agenda);
|
||||
Assert.Equal(metadata.ScheduledEnd, candidate.Metadata.ScheduledEnd);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsPromptAndScreenshotUseNativeClientResults()
|
||||
{
|
||||
var screenshot = new ActiveWindowScreenshot([137, 80, 78, 71], "Meeting deck");
|
||||
var client = new StubMacOsDesktopIntegrationClient(
|
||||
screenshot: screenshot,
|
||||
promptResponse: MeetingStartPromptResponse.Record);
|
||||
IMeetingStartPromptService prompt = new MacOsMeetingStartPromptService(
|
||||
client,
|
||||
NullLogger<MacOsMeetingStartPromptService>.Instance);
|
||||
IActiveWindowScreenshotCapture capture = client;
|
||||
IScreenshotImageCropper cropper = client;
|
||||
MeetingStartPromptResponse? callbackResponse = null;
|
||||
var request = new MeetingStartPromptRequest(new CalendarMeeting(
|
||||
"event-1",
|
||||
"Project sync",
|
||||
DateTimeOffset.Now,
|
||||
DateTimeOffset.Now.AddHours(1)));
|
||||
|
||||
await prompt.ShowPromptAsync(
|
||||
request,
|
||||
(response, _) =>
|
||||
{
|
||||
callbackResponse = response;
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
CancellationToken.None);
|
||||
var captured = await capture.CaptureAsync(CancellationToken.None);
|
||||
var croppedPath = await cropper.SaveCroppedScreenshotAsync(
|
||||
"meeting.png",
|
||||
new ScreenshotCropCoordinates(1, 2, 3, 4),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MeetingStartPromptResponse.Record, callbackResponse);
|
||||
Assert.Equal(screenshot, captured);
|
||||
Assert.Equal("meeting.png.cropped.png", croppedPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsCapabilityEndpointReportsEnabledRealProviders()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var factory = new WebApplicationFactory<Program>();
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
var report = await client.GetFromJsonAsync<PlatformCapabilityReport>(
|
||||
"/diagnostics/platform-capabilities");
|
||||
|
||||
Assert.NotNull(report);
|
||||
Assert.Equal("macOS", report.Host);
|
||||
Assert.Equal(nameof(MacOsMeetingMetadataProvider), report.MeetingMetadataProvider);
|
||||
Assert.Equal(nameof(MacOsCalendarMeetingProvider), report.CalendarMeetingProvider);
|
||||
Assert.Equal(nameof(MacOsMeetingStartPromptService), report.MeetingStartPromptProvider);
|
||||
Assert.Equal(nameof(MacOsDesktopIntegrationClient), report.ScreenshotProvider);
|
||||
Assert.Equal(nameof(MacOsWorkflowRulesEditorWindowService), report.WorkflowEditorProvider);
|
||||
Assert.True(report.WorkflowEditorAvailable);
|
||||
Assert.True(report.CalendarRecordingPromptsEnabled);
|
||||
Assert.True(report.SpeakerIdentificationEnabled);
|
||||
Assert.True(report.ScreenshotOcrEnabled);
|
||||
Assert.True(report.FunAsrBackendEnabled);
|
||||
Assert.True(report.FunAsrDiarizationEnabled);
|
||||
Assert.True(report.WhisperLocalDiarizationEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NativeHelperAdvertisesCalendarPromptAndScreenshotFeatures()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var runner = new ProcessCommandRunner(
|
||||
NullLogger<ProcessCommandRunner>.Instance);
|
||||
|
||||
var result = await runner.RunAsync(
|
||||
GetNativeHelperPath(),
|
||||
["self-test"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, result.ExitCode);
|
||||
using var json = System.Text.Json.JsonDocument.Parse(result.StandardOutput);
|
||||
Assert.Equal("ok", json.RootElement.GetProperty("status").GetString());
|
||||
Assert.Equal(
|
||||
["calendar", "prompt", "screenshot", "crop"],
|
||||
json.RootElement.GetProperty("features")
|
||||
.EnumerateArray()
|
||||
.Select(feature => feature.GetString()!)
|
||||
.ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NativeHelperCropsPngUsingOcrPixelCoordinates()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var testRoot = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-native-crop-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(testRoot);
|
||||
var bitmapPath = Path.Combine(testRoot, "input.bmp");
|
||||
var inputPath = Path.Combine(testRoot, "input.png");
|
||||
var outputPath = Path.Combine(testRoot, "output.png");
|
||||
var outputBitmapPath = Path.Combine(testRoot, "output.bmp");
|
||||
try
|
||||
{
|
||||
await File.WriteAllBytesAsync(
|
||||
bitmapPath,
|
||||
CreateTopRedBottomBlueBitmap());
|
||||
var runner = new ProcessCommandRunner(
|
||||
NullLogger<ProcessCommandRunner>.Instance);
|
||||
var inputConversion = await runner.RunAsync(
|
||||
"/usr/bin/sips",
|
||||
["-s", "format", "png", bitmapPath, "--out", inputPath],
|
||||
CancellationToken.None);
|
||||
Assert.Equal(0, inputConversion.ExitCode);
|
||||
|
||||
var result = await runner.RunAsync(
|
||||
GetNativeHelperPath(),
|
||||
["crop", inputPath, outputPath, "0", "0", "2", "1"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, result.ExitCode);
|
||||
var png = await File.ReadAllBytesAsync(outputPath);
|
||||
Assert.Equal((2, 1), PngTestFixture.ReadDimensions(png));
|
||||
var outputConversion = await runner.RunAsync(
|
||||
"/usr/bin/sips",
|
||||
["-s", "format", "bmp", outputPath, "--out", outputBitmapPath],
|
||||
CancellationToken.None);
|
||||
Assert.Equal(0, outputConversion.ExitCode);
|
||||
Assert.Equal(
|
||||
(Red: (byte)255, Green: (byte)0, Blue: (byte)0),
|
||||
ReadFirstBitmapPixel(await File.ReadAllBytesAsync(outputBitmapPath)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(testRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CalendarProviderUsesActualLocalMidnightsAcrossDaylightSavingTransition()
|
||||
{
|
||||
if (TimeZoneInfo.Local.Id != "Europe/Berlin")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var client = new StubMacOsDesktopIntegrationClient();
|
||||
var provider = new MacOsCalendarMeetingProvider(
|
||||
client,
|
||||
NullLogger<MacOsCalendarMeetingProvider>.Instance);
|
||||
|
||||
await provider.GetRecordingPromptCandidatesForDayAsync(
|
||||
new DateOnly(2026, 3, 29),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(
|
||||
DateTimeOffset.Parse("2026-03-29T00:00:00+01:00"),
|
||||
client.CalendarFrom);
|
||||
Assert.Equal(
|
||||
DateTimeOffset.Parse("2026-03-30T00:00:00+02:00"),
|
||||
client.CalendarTo);
|
||||
Assert.Equal(TimeSpan.FromHours(23), client.CalendarTo - client.CalendarFrom);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CalendarClientFallsBackToCalendarAutomationWhenEventKitIsDenied()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var runner = new SequenceCommandRunner(
|
||||
new CommandResult(
|
||||
MacOsIntegrationException.CalendarPermissionDeniedExitCode,
|
||||
"",
|
||||
"Calendar Full Access permission is required."),
|
||||
new CommandResult(
|
||||
0,
|
||||
"""
|
||||
[{"id":"fallback-event","subject":"Project sync","start":"2026-07-25T08:00:00Z","end":"2026-07-25T09:00:00Z","location":"Microsoft Teams","notes":"","url":"https://teams.microsoft.com/example","attendees":[],"isCanceled":false}]
|
||||
""",
|
||||
""));
|
||||
IMacOsCalendarClient client = new MacOsDesktopIntegrationClient(
|
||||
runner,
|
||||
Path.GetDirectoryName(GetNativeHelperPath())!);
|
||||
|
||||
var events = await client.GetCalendarEventsAsync(
|
||||
DateTimeOffset.Parse("2026-07-25T00:00:00Z"),
|
||||
DateTimeOffset.Parse("2026-07-26T00:00:00Z"),
|
||||
CancellationToken.None);
|
||||
|
||||
var calendarEvent = Assert.Single(events);
|
||||
Assert.Equal("fallback-event", calendarEvent.Id);
|
||||
Assert.Collection(
|
||||
runner.Commands,
|
||||
command => Assert.Equal("calendar", command.Arguments[0]),
|
||||
command =>
|
||||
{
|
||||
Assert.Equal("/usr/bin/osascript", command.FileName);
|
||||
Assert.Equal(["-l", "JavaScript"], command.Arguments.Take(2));
|
||||
});
|
||||
}
|
||||
|
||||
private static string GetNativeHelperPath()
|
||||
{
|
||||
var configuration = new DirectoryInfo(AppContext.BaseDirectory)
|
||||
.Parent?.Name ?? "Debug";
|
||||
return Path.GetFullPath(Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"MeetingAssistant",
|
||||
"bin",
|
||||
configuration,
|
||||
"net10.0",
|
||||
"Native",
|
||||
"macos-meeting-integrations"));
|
||||
}
|
||||
|
||||
private static byte[] CreateTopRedBottomBlueBitmap()
|
||||
{
|
||||
var bitmap = new byte[70];
|
||||
bitmap[0] = (byte)'B';
|
||||
bitmap[1] = (byte)'M';
|
||||
BitConverter.GetBytes(bitmap.Length).CopyTo(bitmap, 2);
|
||||
BitConverter.GetBytes(54).CopyTo(bitmap, 10);
|
||||
BitConverter.GetBytes(40).CopyTo(bitmap, 14);
|
||||
BitConverter.GetBytes(2).CopyTo(bitmap, 18);
|
||||
BitConverter.GetBytes(2).CopyTo(bitmap, 22);
|
||||
BitConverter.GetBytes((short)1).CopyTo(bitmap, 26);
|
||||
BitConverter.GetBytes((short)24).CopyTo(bitmap, 28);
|
||||
BitConverter.GetBytes(16).CopyTo(bitmap, 34);
|
||||
|
||||
// BMP rows are bottom-up and padded to four-byte boundaries.
|
||||
bitmap[54] = 255;
|
||||
bitmap[57] = 255;
|
||||
bitmap[64] = 255;
|
||||
bitmap[67] = 255;
|
||||
return bitmap;
|
||||
}
|
||||
|
||||
private static (byte Red, byte Green, byte Blue) ReadFirstBitmapPixel(byte[] bitmap)
|
||||
{
|
||||
var pixelOffset = BitConverter.ToInt32(bitmap, 10);
|
||||
return (
|
||||
Red: bitmap[pixelOffset + 2],
|
||||
Green: bitmap[pixelOffset + 1],
|
||||
Blue: bitmap[pixelOffset]);
|
||||
}
|
||||
|
||||
private sealed class StubMacOsDesktopIntegrationClient :
|
||||
IMacOsCalendarClient,
|
||||
IMacOsMeetingPromptClient,
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private readonly IReadOnlyList<MacOsCalendarEvent> events;
|
||||
private readonly ActiveWindowScreenshot screenshot;
|
||||
private readonly MeetingStartPromptResponse promptResponse;
|
||||
|
||||
public StubMacOsDesktopIntegrationClient(
|
||||
MacOsCalendarEvent? calendarEvent = null,
|
||||
ActiveWindowScreenshot? screenshot = null,
|
||||
MeetingStartPromptResponse promptResponse = MeetingStartPromptResponse.Skip)
|
||||
{
|
||||
events = calendarEvent is null ? [] : [calendarEvent];
|
||||
this.screenshot = screenshot ?? new ActiveWindowScreenshot([], null);
|
||||
this.promptResponse = promptResponse;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
CalendarFrom = from;
|
||||
CalendarTo = to;
|
||||
return Task.FromResult(events);
|
||||
}
|
||||
|
||||
public DateTimeOffset CalendarFrom { get; private set; }
|
||||
|
||||
public DateTimeOffset CalendarTo { get; private set; }
|
||||
|
||||
public Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(promptResponse);
|
||||
}
|
||||
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(screenshot);
|
||||
}
|
||||
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<string?>($"{screenshotPath}.cropped.png");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SequenceCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly Queue<CommandResult> results;
|
||||
|
||||
public SequenceCommandRunner(params CommandResult[] results)
|
||||
{
|
||||
this.results = new Queue<CommandResult>(results);
|
||||
}
|
||||
|
||||
public List<(string FileName, IReadOnlyList<string> Arguments)> Commands { get; } = [];
|
||||
|
||||
public Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
Commands.Add((fileName, arguments));
|
||||
return Task.FromResult(results.Dequeue());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,10 +9,10 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -4,8 +4,6 @@ using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
|
||||
#pragma warning disable CA1416
|
||||
|
||||
@@ -117,11 +115,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
var croppedPath = Assert.Single(Directory.GetFiles(
|
||||
Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"),
|
||||
"*-cropped.png"));
|
||||
using (var cropped = new Bitmap(croppedPath))
|
||||
{
|
||||
Assert.Equal(2, cropped.Width);
|
||||
Assert.Equal(2, cropped.Height);
|
||||
}
|
||||
Assert.Equal((2, 2), PngTestFixture.ReadDimensions(await File.ReadAllBytesAsync(croppedPath)));
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var screenshotIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal);
|
||||
@@ -133,6 +127,38 @@ public sealed class MeetingScreenshotServiceTests
|
||||
Assert.Contains("Shared screen text", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureKeepsSuccessfulOcrTextWhenCropperRejectsCoordinates()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new CapturingScreenshotOcrClient(
|
||||
"Readable text survives.",
|
||||
new ScreenshotCropCoordinates(99, 99, 10, 10));
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture(CreatePngBytes(4, 4)),
|
||||
ocr,
|
||||
screenshotImageCropper: new RejectingScreenshotImageCropper());
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(
|
||||
fixture.Artifacts,
|
||||
TimeSpan.FromSeconds(5),
|
||||
CancellationToken.None);
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("Readable text survives.", context);
|
||||
Assert.DoesNotContain("_OCR failed", context);
|
||||
Assert.DoesNotContain("![Cropped screenshot]", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer()
|
||||
{
|
||||
@@ -454,7 +480,8 @@ public sealed class MeetingScreenshotServiceTests
|
||||
IActiveWindowScreenshotCapture capture,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
|
||||
IScreenshotImageCropper? screenshotImageCropper = null)
|
||||
{
|
||||
return new MeetingScreenshotService(
|
||||
capture,
|
||||
@@ -462,12 +489,28 @@ public sealed class MeetingScreenshotServiceTests
|
||||
NoteStore,
|
||||
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
|
||||
ocrClient,
|
||||
screenshotImageCropper ??
|
||||
capture as IScreenshotImageCropper ??
|
||||
new UnavailableScreenshotImageCropper(),
|
||||
NullLogger<MeetingScreenshotService>.Instance,
|
||||
meetingWorkflowEngine);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
private sealed class RejectingScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new InvalidDataException("The crop is outside image bounds.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedScreenshotCapture :
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private readonly byte[] bytes;
|
||||
|
||||
@@ -480,6 +523,21 @@ public sealed class MeetingScreenshotServiceTests
|
||||
{
|
||||
return Task.FromResult(new ActiveWindowScreenshot(bytes, "Demo Window"));
|
||||
}
|
||||
|
||||
public async Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
await File.WriteAllBytesAsync(
|
||||
croppedPath,
|
||||
PngTestFixture.Solid(2, 2),
|
||||
cancellationToken);
|
||||
return croppedPath;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingScreenshotOcrClient : IScreenshotOcrClient
|
||||
@@ -614,15 +672,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
return PngTestFixture.Solid(width, height);
|
||||
}
|
||||
|
||||
private static string ExtractScreenshotOcrId(string context)
|
||||
|
||||
@@ -189,7 +189,9 @@ public sealed class MeetingSummaryRetryRunnerTests
|
||||
return;
|
||||
}
|
||||
|
||||
await stateChanged.Task.WaitAsync(TimeSpan.FromMilliseconds(100));
|
||||
await Task.WhenAny(
|
||||
stateChanged.Task,
|
||||
Task.Delay(TimeSpan.FromMilliseconds(100)));
|
||||
}
|
||||
|
||||
throw new TimeoutException($"Expected {count} state changes.");
|
||||
|
||||
@@ -25,4 +25,16 @@ public sealed class NotificationActivationArgumentsTests
|
||||
Assert.Equal("abc123", arguments["promptId"]);
|
||||
Assert.Equal("continue", arguments["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
|
||||
{
|
||||
Assert.Contains(
|
||||
MeetingInactivityPromptActions.All,
|
||||
action => action.Content == "Pause transcription" &&
|
||||
action.ResponseArgument == "pause");
|
||||
Assert.Equal(
|
||||
MeetingInactivityPromptResponse.Pause,
|
||||
MeetingInactivityPromptActions.ParseResponse("pause"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
internal static class PngTestFixture
|
||||
{
|
||||
private const string Solid2X2 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAAqADAAQAAAABAAAAAgAAAADO0J6QAAAAEklEQVQIHWP8DwQMQMAEIkAAAD34BACALvQ5AAAAAElFTkSuQmCC";
|
||||
private const string Solid4X4 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAABKADAAQAAAABAAAABAAAAADmpNw4AAAAFUlEQVQIHWP8DwQMSIAJiQ1mEhYAAAZdBAQjjGwcAAAAAElFTkSuQmCC";
|
||||
private const string Solid8X6 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAYAAAD+Bd/7AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAACKADAAQAAAABAAAABgAAAAAJfOcJAAAAFklEQVQIHWP8DwQMeAATHjmw1GBQAAAQiQQIVmhjoQAAAABJRU5ErkJggg==";
|
||||
|
||||
public static byte[] Solid(int width, int height)
|
||||
{
|
||||
var encoded = (width, height) switch
|
||||
{
|
||||
(2, 2) => Solid2X2,
|
||||
(4, 4) => Solid4X4,
|
||||
(8, 6) => Solid8X6,
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(width),
|
||||
$"No deterministic PNG fixture exists for {width}x{height}.")
|
||||
};
|
||||
return Convert.FromBase64String(encoded);
|
||||
}
|
||||
|
||||
public static (int Width, int Height) ReadDimensions(byte[] png)
|
||||
{
|
||||
if (!png.AsSpan(0, 8).SequenceEqual(
|
||||
new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }))
|
||||
{
|
||||
throw new InvalidDataException("The fixture is not a PNG.");
|
||||
}
|
||||
|
||||
return (
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(16, 4)),
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(20, 4)));
|
||||
}
|
||||
}
|
||||
@@ -18,31 +18,8 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote-validation:local",
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
@@ -54,23 +31,109 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
Assert.True(commandRunner.RunCancellationWasObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
|
||||
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
[
|
||||
new LaunchProfile("english", namedOptions),
|
||||
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
|
||||
]),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await commandRunner.WaitForRunAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
|
||||
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(options),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(50));
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
private static MeetingAssistantOptions CreateValidationOptions(string image)
|
||||
{
|
||||
return new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
DockerCommand = "docker",
|
||||
Image = image,
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly IReadOnlyList<LaunchProfile> profiles;
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
|
||||
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
|
||||
{
|
||||
this.profiles = profiles;
|
||||
}
|
||||
|
||||
public LaunchProfile GetRequiredProfile(string? name)
|
||||
{
|
||||
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
|
||||
var profileName = string.IsNullOrWhiteSpace(name)
|
||||
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
|
||||
: name;
|
||||
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfile> GetProfiles()
|
||||
{
|
||||
return [GetRequiredProfile(null)];
|
||||
return profiles;
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
||||
|
||||
@@ -2,6 +2,7 @@ using MeetingAssistant;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
@@ -57,17 +58,49 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner(
|
||||
"""
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
|
||||
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
|
||||
""");
|
||||
var modelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models");
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
|
||||
})
|
||||
.Build();
|
||||
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
|
||||
var validator = CreateValidator(commandRunner, configuredOptions);
|
||||
|
||||
var valid = await validator.ValidateSampleAsync(
|
||||
CreateWav(TimeSpan.FromSeconds(20)),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(valid);
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
bool enabled = true)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
return CreateValidator(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
@@ -76,9 +109,8 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Enabled = enabled,
|
||||
MinimumSingleSpeakerCoverage = 0.90,
|
||||
MinimumMatchingKnownSnippetRatio = 1,
|
||||
Diarization = new PyannoteDiarizationOptions
|
||||
Diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BuildImage = false,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote:local",
|
||||
@@ -94,7 +126,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
MeetingAssistantOptions configuredOptions)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(configuredOptions),
|
||||
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
|
||||
}
|
||||
|
||||
|
||||
@@ -156,9 +156,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var explicitDiarization = new PyannoteDiarizationOptions
|
||||
var explicitDiarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-azure:local",
|
||||
@@ -169,7 +168,7 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
};
|
||||
|
||||
await finalizer.FinalizeAsync(
|
||||
await finalizer.FinalizeEnabledAsync(
|
||||
audioPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
|
||||
explicitDiarization,
|
||||
@@ -186,9 +185,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
|
||||
var diarization = new PyannoteDiarizationOptions
|
||||
var diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-warmup:local",
|
||||
@@ -218,9 +216,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: null);
|
||||
|
||||
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
|
||||
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
Token = "",
|
||||
TokenEnv = "",
|
||||
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,596 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(7, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Single(fixture.Encoder.Requests);
|
||||
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Sabrina", saved.CanonicalName);
|
||||
Assert.Equal(3, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(8, saved.VoiceVectors.Count);
|
||||
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options =>
|
||||
{
|
||||
options.OutlierPruningMinimumVectors = 20;
|
||||
options.OutlierPruningNeighborSimilarity = 0.90;
|
||||
options.OutlierPruningMinimumNeighbors = 3;
|
||||
options.OutlierPruningMinimumClusterRatio = 0.60;
|
||||
});
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(16, saved.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
saved.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
"Chris",
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
var currentRunVectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var liveResult = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
fixture.Encoder.Vectors = currentRunVectors;
|
||||
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
|
||||
{
|
||||
KnownSpeakerMappings = liveResult.SpeakerMappings
|
||||
};
|
||||
if (transcriptAlreadyRelabeled)
|
||||
{
|
||||
finalRequest = finalRequest with { Segments = liveResult.Segments };
|
||||
}
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(10, saved.VoiceVectors.Count);
|
||||
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors =
|
||||
[
|
||||
UnitVector(0),
|
||||
UnitVector(0, 2, 0.01f),
|
||||
UnitVector(0, 3, 0.02f),
|
||||
UnitVector(0, 4, 0.03f),
|
||||
UnitVector(0, 5, 0.04f)
|
||||
];
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
|
||||
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(6, saved.VoiceVectors.Count);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyFinishedSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments[0].Speaker);
|
||||
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
|
||||
Assert.All(
|
||||
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal("John", saved.CanonicalName);
|
||||
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
|
||||
Assert.Equal("John", result.Segments.Single().Speaker);
|
||||
Assert.All(saved.References, reference =>
|
||||
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Manuel", saved.CanonicalName);
|
||||
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(
|
||||
configureSpeaker: options => options.MaxMatchCandidates = 1);
|
||||
await fixture.AddIdentityAsync("Legacy WAV identity", []);
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
|
||||
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(target.Id);
|
||||
Assert.Single(saved.Snippets);
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
|
||||
Assert.Single(await context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private sealed class Fixture : IAsyncDisposable
|
||||
{
|
||||
private readonly string directory;
|
||||
private readonly string databasePath;
|
||||
private readonly SpeakerIdentificationOptions speakerOptions;
|
||||
|
||||
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
|
||||
{
|
||||
this.directory = directory;
|
||||
this.databasePath = databasePath;
|
||||
this.speakerOptions = speakerOptions;
|
||||
}
|
||||
|
||||
public FakeEncoder Encoder { get; } = new();
|
||||
|
||||
public FakeSnippetExtractor SnippetExtractor { get; } = new();
|
||||
|
||||
public string DatabasePath => databasePath;
|
||||
|
||||
public static async Task<Fixture> CreateAsync(
|
||||
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
|
||||
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
|
||||
{
|
||||
var directory = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
var databasePath = Path.Combine(directory, "speaker-identities.db");
|
||||
var speakerOptions = new SpeakerIdentificationOptions
|
||||
{
|
||||
DatabasePath = databasePath,
|
||||
MatchBatchSize = 6,
|
||||
MaxMatchCandidates = 100,
|
||||
MatchIdentityActiveAge = TimeSpan.FromDays(365),
|
||||
MinimumSampleSpeechDuration = TimeSpan.Zero,
|
||||
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
}
|
||||
};
|
||||
configure?.Invoke(speakerOptions.Resemblyzer);
|
||||
configureSpeaker?.Invoke(speakerOptions);
|
||||
await using var context = new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
return new Fixture(directory, databasePath, speakerOptions);
|
||||
}
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService CreateService()
|
||||
{
|
||||
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
|
||||
return new ResemblyzerSpeakerIdentificationService(
|
||||
new TestDbContextFactory(databasePath),
|
||||
SnippetExtractor,
|
||||
Encoder,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(appOptions),
|
||||
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
|
||||
}
|
||||
|
||||
public SpeakerIdentificationRequest CreateRequest(
|
||||
string speaker,
|
||||
int sampleCount,
|
||||
IReadOnlyList<string>? attendees = null,
|
||||
int segmentCount = 1)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, "transcript.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
var segments = Enumerable.Range(0, segmentCount)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 30),
|
||||
TimeSpan.FromSeconds((index + 1) * 30),
|
||||
speaker,
|
||||
"enough useful words for a speaker sample"))
|
||||
.ToList();
|
||||
var segment = segments[0];
|
||||
return new SpeakerIdentificationRequest(
|
||||
Path.Combine(directory, "meeting.wav"),
|
||||
new MeetingNote(
|
||||
Path.Combine(directory, "meeting.md"),
|
||||
new MeetingNoteFrontmatter
|
||||
{
|
||||
Title = "Test",
|
||||
Attendees = attendees?.ToList() ?? ["Chris"],
|
||||
Transcript = transcriptPath,
|
||||
AssistantContext = Path.Combine(directory, "context.md"),
|
||||
Summary = Path.Combine(directory, "summary.md")
|
||||
},
|
||||
""),
|
||||
segments,
|
||||
Enumerable.Range(0, sampleCount)
|
||||
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
|
||||
.ToList());
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> AddIdentityAsync(
|
||||
string? name,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
IReadOnlyList<string>? candidates = null,
|
||||
IReadOnlyList<string>? aliases = null)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
|
||||
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = speakerOptions.Resemblyzer.ModelId,
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"known-{index}",
|
||||
CreatedAt = now.AddMinutes(index)
|
||||
}).ToList()
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
return identity;
|
||||
}
|
||||
|
||||
public async Task AddCurrentRunCandidateAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string candidateName)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
|
||||
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "older-model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(UnitVector(1)),
|
||||
Fingerprint = "older-model-vector",
|
||||
CreatedAt = now
|
||||
}
|
||||
],
|
||||
References =
|
||||
[
|
||||
SpeakerIdentityReferences.Create(
|
||||
request.MeetingNote.Path,
|
||||
request.MeetingNote.Frontmatter.Transcript,
|
||||
now)
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync(identity => identity.Id == id);
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public IReadOnlyList<float[]> Vectors { get; set; } = [];
|
||||
|
||||
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
|
||||
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
|
||||
{
|
||||
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
|
||||
|
||||
public Task<byte[]> ExtractSnippetAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(speakerSegments.ToList());
|
||||
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
var databasePath = Path.Combine(directory, "identities.db");
|
||||
var factory = new TestDbContextFactory(databasePath);
|
||||
await using (var context = factory.CreateDbContext())
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris",
|
||||
DateTimeOffset.UtcNow.AddMonths(-2),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 20, 0.02f))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris duplicate",
|
||||
DateTimeOffset.UtcNow.AddDays(-1),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 40, 0.02f)))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Decoy",
|
||||
DateTimeOffset.UtcNow.AddMonths(-1),
|
||||
[UnitVector(0)],
|
||||
directory));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
};
|
||||
var service = new ResemblyzerSpeakerIdentityMergeService(
|
||||
factory,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
MergeRecentIdentityAge = TimeSpan.FromDays(14),
|
||||
MaxMatchCandidates = 1,
|
||||
MaxSnippetsPerSpeaker = 3,
|
||||
Resemblyzer = resemblyzerOptions
|
||||
}
|
||||
}),
|
||||
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
|
||||
|
||||
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.MatchAttempts);
|
||||
Assert.Equal(1, result.MergedPairs);
|
||||
await using var verification = factory.CreateDbContext();
|
||||
var saved = await verification.SpeakerIdentities
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.SingleAsync(identity => identity.CanonicalName == "Chris");
|
||||
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
|
||||
Assert.Equal("Chris", saved.CanonicalName);
|
||||
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
|
||||
Assert.Equal(20, saved.VoiceVectors.Count);
|
||||
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static SpeakerIdentity CreateIdentity(
|
||||
string name,
|
||||
DateTimeOffset createdAt,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
string directory)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
return new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = createdAt,
|
||||
UpdatedAt = createdAt,
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"{name}-{index}",
|
||||
CreatedAt = createdAt.AddMinutes(index)
|
||||
}).ToList(),
|
||||
References =
|
||||
[
|
||||
new SpeakerIdentityReference
|
||||
{
|
||||
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
|
||||
TranscriptPath = transcriptPath,
|
||||
CreatedAt = createdAt
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primary] = 1;
|
||||
if (secondary is { } index)
|
||||
{
|
||||
vector[index] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static float[] ToVector(byte[] bytes)
|
||||
{
|
||||
var vector = new float[bytes.Length / sizeof(float)];
|
||||
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
|
||||
return vector;
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcherTests
|
||||
{
|
||||
[Fact]
|
||||
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
|
||||
{
|
||||
var matcher = CreateMatcher();
|
||||
var query = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
|
||||
.ToList();
|
||||
var candidates = new[]
|
||||
{
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
|
||||
};
|
||||
|
||||
var result = matcher.Match(query, candidates);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
Assert.True(result.Cohesion >= 0.99);
|
||||
Assert.True(result.BestSimilarity >= 0.99);
|
||||
Assert.True(result.RunnerUpSimilarity < 0.1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FourVectorsDoNotTriggerAutomaticMatching()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 4).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("4/5", result.Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("cohesion", result.Reason);
|
||||
Assert.Null(result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarityBelowThresholdIsRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("best similarity", result.Reason);
|
||||
Assert.Equal(0, result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarCandidatesWithinMarginAreRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
|
||||
]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("margin", result.Reason);
|
||||
Assert.NotNull(result.RunnerUpSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
|
||||
]);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
|
||||
{
|
||||
return new ResemblyzerVoiceClusterMatcher(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
|
||||
{
|
||||
[Fact]
|
||||
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(16, identity.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
identity.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AmbiguousDenseClustersPreserveAllVectors()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(20, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 15)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PruningPreservesOtherModelsAndMalformedStoredRows()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
[UnitVector(2, 60, 0.04f)],
|
||||
"older-model",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = [1],
|
||||
Fingerprint = "malformed",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "zero-magnitude",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
|
||||
{
|
||||
return new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
OutlierPruningMinimumVectors = 20,
|
||||
OutlierPruningNeighborSimilarity = 0.90,
|
||||
OutlierPruningMinimumNeighbors = 3,
|
||||
OutlierPruningMinimumClusterRatio = 0.60
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||
}
|
||||
|
||||
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var identity = new SpeakerIdentity();
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
"resemblyzer-0.1.4-pretrained",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
return identity;
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int secondaryDimension,
|
||||
float secondaryValue)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
vector[secondaryDimension] = secondaryValue;
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(options),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await encoder.WaitForWarmupAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(encoder.CancellationObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisabledWarmupDoesNotInvokeEncoder()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.False(encoder.WarmupStarted);
|
||||
}
|
||||
|
||||
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool WarmupStarted { get; private set; }
|
||||
|
||||
public bool CancellationObserved { get; private set; }
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
WarmupStarted = true;
|
||||
started.TrySetResult();
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
CancellationObserved = true;
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitForWarmupAsync()
|
||||
{
|
||||
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,13 +8,22 @@ namespace MeetingAssistant.Tests;
|
||||
public sealed class SpeakerAudioSampleCollectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
|
||||
public void DefaultSampleDurationsAreTenAndSixtySeconds()
|
||||
{
|
||||
var options = new SpeakerIdentificationOptions();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
|
||||
@@ -37,8 +46,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
|
||||
|
||||
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
|
||||
@@ -48,6 +57,61 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
|
||||
|
||||
var sample = Assert.Single(collector.Snapshot());
|
||||
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
|
||||
|
||||
Assert.Empty(collector.Snapshot());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
|
||||
|
||||
var sample = collector.TryAdd(Segment(
|
||||
0,
|
||||
70,
|
||||
"Guest01",
|
||||
"one two three four five six seven eight nine ten."));
|
||||
|
||||
Assert.NotNull(sample);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
|
||||
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
|
||||
{
|
||||
@@ -55,8 +119,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1),
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
logger: logger);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
@@ -69,6 +133,47 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.Contains("minimum duration", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
|
||||
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
|
||||
|
||||
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
|
||||
|
||||
Assert.Collection(
|
||||
samples,
|
||||
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
|
||||
|
||||
Assert.Single(collector.Snapshot());
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(
|
||||
double start,
|
||||
double end,
|
||||
|
||||
@@ -107,6 +107,9 @@ public sealed class SpeakerIdentityMergeServiceTests
|
||||
return new SpeakerIdentityMergeService(
|
||||
new TestSpeakerIdentityDbContextFactory(dbPath),
|
||||
Matcher,
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions(),
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerIdentityMergerTests
|
||||
{
|
||||
[Fact]
|
||||
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var target = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("oldest", 1, now.AddMinutes(-4)),
|
||||
Vector("duplicate", 2, now.AddMinutes(-3))
|
||||
]
|
||||
};
|
||||
var source = new SpeakerIdentity
|
||||
{
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("duplicate", 2, now.AddMinutes(-2)),
|
||||
Vector("newer", 3, now.AddMinutes(-1)),
|
||||
Vector("newest", 4, now)
|
||||
]
|
||||
};
|
||||
|
||||
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
|
||||
|
||||
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
|
||||
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerRecognitionFeatureSelectionTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
|
||||
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
|
||||
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
|
||||
bool enabled,
|
||||
Type expectedIdentificationType,
|
||||
Type expectedMergeType,
|
||||
int expectedMinimumSamples,
|
||||
bool expectedNonOverlappingSamples)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
await using var factory = new WebApplicationFactory<Program>()
|
||||
.WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
|
||||
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
|
||||
});
|
||||
});
|
||||
builder.ConfigureTestServices(services =>
|
||||
{
|
||||
services.RemoveAll<IResemblyzerVoiceEncoder>();
|
||||
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
|
||||
});
|
||||
});
|
||||
|
||||
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
|
||||
|
||||
Assert.IsType(expectedIdentificationType, service);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
|
||||
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
|
||||
Assert.IsType(expectedMergeType, mergeService);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
|
||||
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
|
||||
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
|
||||
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerSampleSpanSelectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13),
|
||||
Segment(20, 70)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Equal(3, span.Count);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Empty(span);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
|
||||
{
|
||||
var segments = Enumerable.Range(0, 6)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 10),
|
||||
TimeSpan.FromSeconds((index + 1) * 10),
|
||||
"Guest01",
|
||||
$"segment {index} has enough words"))
|
||||
.ToList();
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
segments,
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(5, spans.Count);
|
||||
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
|
||||
Assert.All(
|
||||
spans.Zip(spans.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 10),
|
||||
Segment(9, 19),
|
||||
Segment(20, 30)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(2, spans.Count);
|
||||
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Empty(spans);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[Segment(0, 70)],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(double start, double end)
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(start),
|
||||
TimeSpan.FromSeconds(end),
|
||||
"Guest01",
|
||||
"enough words for this segment");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerVoiceVectorPersistenceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-a",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
var identity = await context.SpeakerIdentities
|
||||
.Include(candidate => candidate.VoiceVectors)
|
||||
.SingleAsync();
|
||||
var vector = Assert.Single(identity.VoiceVectors);
|
||||
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
|
||||
Assert.Equal(256, vector.Dimensions);
|
||||
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
|
||||
|
||||
context.SpeakerIdentities.Remove(identity);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
await using var context = new SpeakerIdentityDbContext(options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
context.SpeakerVoiceVectors.AddRange(
|
||||
CreateVector(identity.Id, "same-vector"),
|
||||
CreateVector(identity.Id, "same-vector"));
|
||||
|
||||
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identityId,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,8 @@ public sealed class TaskbarIconTests
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
@@ -94,10 +95,32 @@ public sealed class TaskbarIconTests
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
|
||||
{
|
||||
var runningMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||
[Profile("default")]);
|
||||
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
|
||||
[Profile("default")]);
|
||||
|
||||
Assert.Contains(runningMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.PauseTranscription &&
|
||||
item.Text == "Pause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
|
||||
item.Text == "Unpause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.StopRecording &&
|
||||
item.Text == "Finish meeting");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessingMenuShowsSummarizingButAllowsStartingNewRecordings()
|
||||
{
|
||||
@@ -200,7 +223,8 @@ public sealed class TaskbarIconTests
|
||||
private static RecordingStatus Status(
|
||||
bool isRecording = false,
|
||||
RecordingProcessState state = RecordingProcessState.Idle,
|
||||
string? profile = null)
|
||||
string? profile = null,
|
||||
bool isPaused = false)
|
||||
{
|
||||
return new RecordingStatus(
|
||||
isRecording,
|
||||
@@ -209,7 +233,8 @@ public sealed class TaskbarIconTests
|
||||
state == RecordingProcessState.Idle ? null : "context.md",
|
||||
state == RecordingProcessState.Idle ? null : "summary.md",
|
||||
state,
|
||||
profile);
|
||||
profile,
|
||||
isPaused);
|
||||
}
|
||||
|
||||
[SupportedOSPlatform("windows")]
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class VenvResemblyzerVoiceEncoderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
|
||||
{
|
||||
using var fixture = new EncoderFixture();
|
||||
|
||||
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName == "python-test"
|
||||
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("torch==2.14.0+cpu")
|
||||
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("Resemblyzer==0.1.4")
|
||||
&& command.Arguments.Contains("--no-deps"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
warmupExitCode: 21,
|
||||
warmupError: "incompatible environment");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
|
||||
|
||||
Assert.Empty(Directory.EnumerateFiles(
|
||||
fixture.RuntimeFolder,
|
||||
".ready",
|
||||
SearchOption.AllDirectories));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
|
||||
{
|
||||
var first = new float[256];
|
||||
first[0] = 2;
|
||||
var second = new float[256];
|
||||
second[1] = 3;
|
||||
using var fixture = new EncoderFixture(VectorOutput([first, second]));
|
||||
|
||||
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, vectors.Count);
|
||||
Assert.Equal(1f, vectors[0][0], 5);
|
||||
Assert.Equal(1f, vectors[1][1], 5);
|
||||
var encodingCommand = Assert.Single(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Any(
|
||||
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
|
||||
Assert.EndsWith(
|
||||
Path.Combine("Scripts", "python.exe"),
|
||||
encodingCommand.FileName,
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal(2, encodingCommand.Arguments.Count);
|
||||
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongVectorDimension()
|
||||
{
|
||||
var vector = new float[255];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("255 dimensions", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsMalformedVectorJson()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsNonFiniteVector()
|
||||
{
|
||||
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
|
||||
using var fixture = new EncoderFixture(
|
||||
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongResultCount()
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
|
||||
{
|
||||
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("zero magnitude", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncReportsFailedLocalCommand()
|
||||
{
|
||||
using var fixture = new EncoderFixture("", 17, "runtime failed");
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("exit code 17", error.Message);
|
||||
Assert.Contains("runtime failed", error.Message);
|
||||
}
|
||||
|
||||
private static string VectorOutput(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
return $"""
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
|
||||
{JsonSerializer.Serialize(vectors)}
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
|
||||
""";
|
||||
}
|
||||
|
||||
private sealed class EncoderFixture : IDisposable
|
||||
{
|
||||
public EncoderFixture(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
RuntimeFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"resemblyzer");
|
||||
Runner = new CapturingCommandRunner(
|
||||
encodingOutput,
|
||||
encodingExitCode,
|
||||
encodingError,
|
||||
warmupExitCode,
|
||||
warmupError);
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
|
||||
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
|
||||
Encoder = new VenvResemblyzerVoiceEncoder(
|
||||
Runner,
|
||||
Options.Create(options),
|
||||
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
|
||||
}
|
||||
|
||||
public CapturingCommandRunner Runner { get; }
|
||||
|
||||
public VenvResemblyzerVoiceEncoder Encoder { get; }
|
||||
|
||||
public string RuntimeFolder { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(RuntimeFolder))
|
||||
{
|
||||
Directory.Delete(RuntimeFolder, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly string encodingOutput;
|
||||
private readonly int encodingExitCode;
|
||||
private readonly string encodingError;
|
||||
private readonly int warmupExitCode;
|
||||
private readonly string warmupError;
|
||||
|
||||
public CapturingCommandRunner(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
this.encodingOutput = encodingOutput;
|
||||
this.encodingExitCode = encodingExitCode;
|
||||
this.encodingError = encodingError;
|
||||
this.warmupExitCode = warmupExitCode;
|
||||
this.warmupError = warmupError;
|
||||
}
|
||||
|
||||
public List<CapturedCommand> Commands { get; } = [];
|
||||
|
||||
public Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
|
||||
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
|
||||
if (isEncoding)
|
||||
{
|
||||
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
|
||||
}
|
||||
|
||||
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
|
||||
return Task.FromResult(isWarmup
|
||||
? new CommandResult(warmupExitCode, string.Empty, warmupError)
|
||||
: new CommandResult(0, string.Empty, string.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
|
||||
}
|
||||
@@ -732,6 +732,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var rulesPath = Path.Combine(root, "rules.yaml");
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
HostOperatingSystem.DetectCurrent(),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions
|
||||
@@ -749,10 +750,61 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Contains("list_recent_summaries", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesWindowsCommandsAndConceptsOnWindows()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.Windows),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: Windows", instructions);
|
||||
Assert.Contains("PowerShell", instructions);
|
||||
Assert.Contains("Windows-style paths", instructions);
|
||||
Assert.Contains("Windows services", instructions);
|
||||
Assert.Contains("Task Manager", instructions);
|
||||
Assert.Contains("Do not use macOS-only commands or concepts", instructions);
|
||||
Assert.Contains("does not make unavailable integrations available", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesMacOSCommandsAndConceptsOnMacOS()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: macOS", instructions);
|
||||
Assert.Contains("zsh", instructions);
|
||||
Assert.Contains("POSIX-style paths", instructions);
|
||||
Assert.Contains("launchd and LaunchAgents", instructions);
|
||||
Assert.Contains("Activity Monitor", instructions);
|
||||
Assert.Contains("Do not use Windows-only commands or concepts", instructions);
|
||||
Assert.Contains("does not make unavailable integrations available", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesPortableConceptsOnUnsupportedHosts()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.Unsupported),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: unsupported", instructions);
|
||||
Assert.Contains("portable .NET concepts", instructions);
|
||||
Assert.Contains("Do not assume Windows or macOS", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
HostOperatingSystem.DetectCurrent(),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
@@ -767,6 +819,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
@@ -784,6 +837,8 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Contains("read the matching assistant context", instructions);
|
||||
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
|
||||
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
|
||||
Assert.Contains("Detected host operating system: macOS", instructions);
|
||||
Assert.Contains("zsh", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -809,6 +864,32 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
|
||||
{
|
||||
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
|
||||
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identity.Id,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-1",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
await fixture.Context.SaveChangesAsync();
|
||||
var tools = fixture.CreateTools();
|
||||
|
||||
var searchResult = await tools.SearchIdentities("Sabrina");
|
||||
var readResult = await tools.ReadIdentity(identity.Id);
|
||||
|
||||
Assert.Contains("\"sampleCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", readResult);
|
||||
Assert.DoesNotContain("vector-1", readResult);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
|
||||
{
|
||||
@@ -817,7 +898,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
|
||||
|
||||
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
|
||||
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
|
||||
@@ -52,14 +52,14 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
cachedMeetingDay = today;
|
||||
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
|
||||
logger.LogInformation(
|
||||
"Synced {MeetingCount} Outlook Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
|
||||
"Synced {MeetingCount} Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
|
||||
cachedMeetings.Count,
|
||||
today,
|
||||
nextSyncAt);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read today's Outlook Teams meetings for recording prompts");
|
||||
logger.LogWarning(exception, "Could not read today's Teams calendar meetings for recording prompts");
|
||||
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace MeetingAssistant;
|
||||
|
||||
public enum HostOperatingSystemKind
|
||||
{
|
||||
Windows,
|
||||
MacOS,
|
||||
Unsupported
|
||||
}
|
||||
|
||||
public sealed record HostOperatingSystem(HostOperatingSystemKind Kind)
|
||||
{
|
||||
public static HostOperatingSystem DetectCurrent()
|
||||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.Windows);
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsMacOS())
|
||||
{
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.MacOS);
|
||||
}
|
||||
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.Unsupported);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.LaunchProfiles;
|
||||
|
||||
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
var options = BindDefaultOptions();
|
||||
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ValidateSpeakerSampleDurations(options, DefaultProfileName);
|
||||
return new LaunchProfile(DefaultProfileName, options);
|
||||
}
|
||||
|
||||
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
|
||||
profileSection.Bind(options);
|
||||
ApplyArrayOverrides(profileSection, options);
|
||||
ValidateSpeakerSampleDurations(options, profileName);
|
||||
return new LaunchProfile(profileName, options);
|
||||
}
|
||||
|
||||
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
: name.Trim();
|
||||
}
|
||||
|
||||
private static void ValidateSpeakerSampleDurations(
|
||||
MeetingAssistantOptions options,
|
||||
string profileName)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification,
|
||||
$"Launch profile '{profileName}' has invalid speaker sample durations");
|
||||
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification.Resemblyzer,
|
||||
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
|
||||
}
|
||||
|
||||
private static void ApplyArrayOverrides(
|
||||
IConfigurationSection profileSection,
|
||||
MeetingAssistantOptions options)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed record MacOsDesktopControlManifest(
|
||||
string BaseUrl,
|
||||
string StatusUrl,
|
||||
string OpenAgentUrl,
|
||||
IReadOnlyList<MacOsDesktopProfile> Profiles,
|
||||
IReadOnlyList<MacOsDesktopHotkey> Hotkeys)
|
||||
{
|
||||
public static MacOsDesktopControlManifest Create(
|
||||
string baseUrl,
|
||||
IReadOnlyList<LaunchProfile> profiles,
|
||||
IReadOnlyList<LaunchProfileHotkey> hotkeys)
|
||||
{
|
||||
var normalizedBaseUrl = baseUrl.TrimEnd('/');
|
||||
return new MacOsDesktopControlManifest(
|
||||
normalizedBaseUrl,
|
||||
$"{normalizedBaseUrl}/recording/status",
|
||||
$"{normalizedBaseUrl}/diagnostics/settings-and-logs",
|
||||
profiles
|
||||
.Select(profile => new MacOsDesktopProfile(
|
||||
profile.Name,
|
||||
profile.Options.Hotkey.Toggle))
|
||||
.ToArray(),
|
||||
hotkeys.Select(CreateHotkey).ToArray());
|
||||
}
|
||||
|
||||
private static MacOsDesktopHotkey CreateHotkey(LaunchProfileHotkey hotkey)
|
||||
{
|
||||
var endpoint = hotkey.Action switch
|
||||
{
|
||||
LaunchProfileHotkeyAction.ToggleRecording => "recording/toggle",
|
||||
LaunchProfileHotkeyAction.AbortRecording => "recording/abort",
|
||||
LaunchProfileHotkeyAction.CaptureScreenshot => "meetings/screenshot/capture",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(hotkey))
|
||||
};
|
||||
return new MacOsDesktopHotkey(
|
||||
hotkey.Hotkey,
|
||||
$"/profiles/{Uri.EscapeDataString(hotkey.ProfileName)}/{endpoint}");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MacOsDesktopProfile(string Name, string ToggleShortcut);
|
||||
|
||||
public sealed record MacOsDesktopHotkey(string Shortcut, string Endpoint);
|
||||
@@ -0,0 +1,107 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed class MacOsDesktopControlService : IHostedService, IDisposable
|
||||
{
|
||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||
private readonly IOptions<MeetingAssistantOptions> options;
|
||||
private readonly IHostApplicationLifetime applicationLifetime;
|
||||
private readonly ILogger<MacOsDesktopControlService> logger;
|
||||
private Process? helper;
|
||||
private string? manifestPath;
|
||||
|
||||
public MacOsDesktopControlService(
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
IHostApplicationLifetime applicationLifetime,
|
||||
ILogger<MacOsDesktopControlService> logger)
|
||||
{
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.options = options;
|
||||
this.applicationLifetime = applicationLifetime;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-desktop-controls");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
logger.LogWarning("macOS desktop controls helper was not found at {HelperPath}", helperPath);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var manifest = MacOsDesktopControlManifest.Create(
|
||||
options.Value.Api.PublicBaseUrl,
|
||||
launchProfiles.GetProfiles(),
|
||||
launchProfiles.GetHotkeys());
|
||||
manifestPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-desktop-{Environment.ProcessId}.json");
|
||||
File.WriteAllText(
|
||||
manifestPath,
|
||||
JsonSerializer.Serialize(
|
||||
manifest,
|
||||
new JsonSerializerOptions(JsonSerializerDefaults.Web)));
|
||||
|
||||
helper = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = helperPath,
|
||||
ArgumentList = { manifestPath },
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
});
|
||||
|
||||
if (helper is null)
|
||||
{
|
||||
throw new InvalidOperationException("Could not start the macOS desktop controls helper.");
|
||||
}
|
||||
|
||||
helper.EnableRaisingEvents = true;
|
||||
helper.Exited += (_, _) =>
|
||||
{
|
||||
if (!applicationLifetime.ApplicationStopping.IsCancellationRequested)
|
||||
{
|
||||
logger.LogWarning("macOS desktop controls helper exited unexpectedly");
|
||||
}
|
||||
};
|
||||
logger.LogInformation("Started macOS menu-bar controls with {HotkeyCount} global hotkeys", manifest.Hotkeys.Count);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (helper is { HasExited: false })
|
||||
{
|
||||
helper.Kill();
|
||||
helper.WaitForExit(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
DeleteManifest();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
helper?.Dispose();
|
||||
DeleteManifest();
|
||||
}
|
||||
|
||||
private void DeleteManifest()
|
||||
{
|
||||
if (manifestPath is not null && File.Exists(manifestPath))
|
||||
{
|
||||
File.Delete(manifestPath);
|
||||
manifestPath = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
using System.Text.Json;
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
internal sealed record MacOsCalendarEvent(
|
||||
string Id,
|
||||
string Subject,
|
||||
DateTimeOffset Start,
|
||||
DateTimeOffset End,
|
||||
string Location,
|
||||
string Notes,
|
||||
string Url,
|
||||
IReadOnlyList<string> Attendees,
|
||||
bool IsCanceled);
|
||||
|
||||
internal interface IMacOsCalendarClient
|
||||
{
|
||||
Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal interface IMacOsMeetingPromptClient
|
||||
{
|
||||
Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class MacOsDesktopIntegrationClient :
|
||||
IMacOsCalendarClient,
|
||||
IMacOsMeetingPromptClient,
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly string nativeDirectory;
|
||||
|
||||
public MacOsDesktopIntegrationClient(ICommandRunner commandRunner)
|
||||
: this(commandRunner, Path.Combine(AppContext.BaseDirectory, "Native"))
|
||||
{
|
||||
}
|
||||
|
||||
internal MacOsDesktopIntegrationClient(
|
||||
ICommandRunner commandRunner,
|
||||
string nativeDirectory)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.nativeDirectory = nativeDirectory;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fromValue = from.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
|
||||
var toValue = to.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
|
||||
CommandResult result;
|
||||
try
|
||||
{
|
||||
result = await RunHelperAsync(
|
||||
["calendar", fromValue, toValue],
|
||||
cancellationToken);
|
||||
}
|
||||
catch (MacOsIntegrationException exception)
|
||||
when (exception.ExitCode == MacOsIntegrationException.CalendarPermissionDeniedExitCode)
|
||||
{
|
||||
result = await RunCalendarAutomationFallbackAsync(
|
||||
fromValue,
|
||||
toValue,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return JsonSerializer.Deserialize<MacOsCalendarEvent[]>(result.StandardOutput, JsonOptions) ?? [];
|
||||
}
|
||||
|
||||
public async Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requestPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-prompt-{Guid.NewGuid():N}.json");
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(
|
||||
requestPath,
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
subject = request.Meeting.Subject,
|
||||
start = request.Meeting.Start
|
||||
}, JsonOptions),
|
||||
cancellationToken);
|
||||
var result = await RunHelperAsync(["prompt", requestPath], cancellationToken);
|
||||
var response = JsonSerializer.Deserialize<MacOsPromptResult>(
|
||||
result.StandardOutput,
|
||||
JsonOptions);
|
||||
return string.Equals(response?.Response, "record", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingStartPromptResponse.Record
|
||||
: MeetingStartPromptResponse.Skip;
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(requestPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<ActiveWindowScreenshot> CaptureAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var outputPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-screenshot-{Guid.NewGuid():N}.png");
|
||||
try
|
||||
{
|
||||
var result = await RunHelperAsync(["screenshot", outputPath], cancellationToken);
|
||||
var response = JsonSerializer.Deserialize<MacOsScreenshotResult>(
|
||||
result.StandardOutput,
|
||||
JsonOptions);
|
||||
return new ActiveWindowScreenshot(
|
||||
await File.ReadAllBytesAsync(outputPath, cancellationToken),
|
||||
response?.WindowTitle);
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
try
|
||||
{
|
||||
await RunHelperAsync(
|
||||
[
|
||||
"crop",
|
||||
screenshotPath,
|
||||
croppedPath,
|
||||
crop.X.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Y.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Width.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Height.ToString(System.Globalization.CultureInfo.InvariantCulture)
|
||||
],
|
||||
cancellationToken);
|
||||
}
|
||||
catch (MacOsIntegrationException exception)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
|
||||
exception);
|
||||
}
|
||||
return croppedPath;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunHelperAsync(
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
throw new PlatformNotSupportedException(
|
||||
"The native macOS desktop integrations can only run on macOS.");
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(nativeDirectory, "macos-meeting-integrations");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS meeting integrations helper was not found at '{helperPath}'. Build or publish Meeting Assistant on macOS.",
|
||||
helperPath);
|
||||
}
|
||||
|
||||
var result = await commandRunner.RunAsync(helperPath, arguments, cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(result.StandardError)
|
||||
? "The helper did not provide an error message."
|
||||
: result.StandardError.Trim();
|
||||
throw new MacOsIntegrationException(
|
||||
result.ExitCode,
|
||||
$"macOS meeting integration '{arguments[0]}' failed with exit code {result.ExitCode}. {detail}");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunCalendarAutomationFallbackAsync(
|
||||
string from,
|
||||
string to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scriptPath = Path.Combine(nativeDirectory, "macos-calendar-events.js");
|
||||
if (!File.Exists(scriptPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS Calendar automation fallback was not found at '{scriptPath}'.",
|
||||
scriptPath);
|
||||
}
|
||||
|
||||
var result = await commandRunner.RunAsync(
|
||||
"/usr/bin/osascript",
|
||||
["-l", "JavaScript", scriptPath, from, to],
|
||||
cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(result.StandardError)
|
||||
? "Calendar automation did not provide an error message."
|
||||
: result.StandardError.Trim();
|
||||
throw new InvalidOperationException(
|
||||
$"macOS calendar access failed through both EventKit and Calendar automation. {detail}");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private sealed record MacOsPromptResult(string Response);
|
||||
|
||||
private sealed record MacOsScreenshotResult(string? WindowTitle);
|
||||
}
|
||||
|
||||
internal sealed class MacOsIntegrationException : InvalidOperationException
|
||||
{
|
||||
public const int CalendarPermissionDeniedExitCode = 77;
|
||||
|
||||
public MacOsIntegrationException(int exitCode, string message)
|
||||
: base(message)
|
||||
{
|
||||
ExitCode = exitCode;
|
||||
}
|
||||
|
||||
public int ExitCode { get; }
|
||||
}
|
||||
|
||||
internal sealed class MacOsMeetingMetadataProvider :
|
||||
IMeetingMetadataProvider,
|
||||
IMeetingMetadataDiagnosticProvider
|
||||
{
|
||||
private readonly IMacOsCalendarClient client;
|
||||
private readonly ILogger<MacOsMeetingMetadataProvider> logger;
|
||||
|
||||
public MacOsMeetingMetadataProvider(
|
||||
IMacOsCalendarClient client,
|
||||
ILogger<MacOsMeetingMetadataProvider> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<MeetingMetadata?> GetCurrentMeetingAsync(
|
||||
DateTimeOffset startedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await GetCurrentMeetingCoreAsync(startedAt, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogDebug(exception, "macOS calendar meeting metadata was not available");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<MeetingMetadataDiagnosticResult> DiagnoseCurrentMeetingAsync(
|
||||
DateTimeOffset startedAt,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await MeetingMetadataDiagnosticRunner.RunAsync(
|
||||
nameof(MacOsMeetingMetadataProvider),
|
||||
startedAt,
|
||||
timeout,
|
||||
token => GetCurrentMeetingCoreAsync(startedAt, token),
|
||||
"No matching Teams calendar event was found.",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<MeetingMetadata?> GetCurrentMeetingCoreAsync(
|
||||
DateTimeOffset startedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var events = await client.GetCalendarEventsAsync(
|
||||
startedAt.AddHours(-4),
|
||||
startedAt.AddMinutes(5),
|
||||
cancellationToken);
|
||||
var candidates = events
|
||||
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
|
||||
.Where(calendarEvent => !calendarEvent.IsCanceled)
|
||||
.ToList();
|
||||
var selected = OutlookMeetingCandidateSelector.Select(
|
||||
candidates,
|
||||
startedAt.LocalDateTime,
|
||||
calendarEvent => calendarEvent.Start.LocalDateTime,
|
||||
calendarEvent => calendarEvent.End.LocalDateTime);
|
||||
return selected is null ? null : MacOsCalendarEventMapper.CreateMetadata(selected);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsCalendarMeetingProvider : ICalendarMeetingProvider
|
||||
{
|
||||
private readonly IMacOsCalendarClient client;
|
||||
private readonly ILogger<MacOsCalendarMeetingProvider> logger;
|
||||
|
||||
public MacOsCalendarMeetingProvider(
|
||||
IMacOsCalendarClient client,
|
||||
ILogger<MacOsCalendarMeetingProvider> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CalendarMeeting>> GetRecordingPromptCandidatesForDayAsync(
|
||||
DateOnly day,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var dayStart = AtLocalMidnight(day);
|
||||
var dayEnd = AtLocalMidnight(day.AddDays(1));
|
||||
try
|
||||
{
|
||||
var events = await client.GetCalendarEventsAsync(dayStart, dayEnd, cancellationToken);
|
||||
return events
|
||||
.Where(calendarEvent => !calendarEvent.IsCanceled)
|
||||
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
|
||||
.Where(calendarEvent => calendarEvent.Start >= dayStart && calendarEvent.Start < dayEnd)
|
||||
.OrderBy(calendarEvent => calendarEvent.Start)
|
||||
.Select(calendarEvent => new CalendarMeeting(
|
||||
calendarEvent.Id,
|
||||
CalendarAppointmentMetadata.ReadSubject(calendarEvent.Subject),
|
||||
calendarEvent.Start,
|
||||
calendarEvent.End,
|
||||
MacOsCalendarEventMapper.CreateMetadata(calendarEvent),
|
||||
calendarEvent.IsCanceled))
|
||||
.ToList();
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogDebug(exception, "macOS calendar meetings were not available");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private static DateTimeOffset AtLocalMidnight(DateOnly day)
|
||||
{
|
||||
var localMidnight = day.ToDateTime(TimeOnly.MinValue);
|
||||
return new DateTimeOffset(
|
||||
localMidnight,
|
||||
TimeZoneInfo.Local.GetUtcOffset(localMidnight));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsMeetingStartPromptService : IMeetingStartPromptService
|
||||
{
|
||||
private readonly IMacOsMeetingPromptClient client;
|
||||
private readonly ILogger<MacOsMeetingStartPromptService> logger;
|
||||
|
||||
public MacOsMeetingStartPromptService(
|
||||
IMacOsMeetingPromptClient client,
|
||||
ILogger<MacOsMeetingStartPromptService> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task ShowPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await client.ShowMeetingStartPromptAsync(request, cancellationToken);
|
||||
await handleResponseAsync(response, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(exception, "Failed to display native macOS meeting-start prompt");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class MacOsMeetingIntegrationRegistration
|
||||
{
|
||||
public static IServiceCollection Add(
|
||||
IServiceCollection services,
|
||||
bool isMacOs)
|
||||
{
|
||||
if (isMacOs)
|
||||
{
|
||||
services.TryAddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||
services.TryAddSingleton<MacOsDesktopIntegrationClient>();
|
||||
services.TryAddSingleton<IMacOsCalendarClient>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.TryAddSingleton<IMacOsMeetingPromptClient>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.AddSingleton<IMeetingMetadataProvider, MacOsMeetingMetadataProvider>();
|
||||
services.AddSingleton<ICalendarMeetingProvider, MacOsCalendarMeetingProvider>();
|
||||
services.AddSingleton<IMeetingStartPromptService, MacOsMeetingStartPromptService>();
|
||||
services.AddSingleton<IActiveWindowScreenshotCapture>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.AddSingleton<IScreenshotImageCropper>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
return services;
|
||||
}
|
||||
|
||||
services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||
services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
|
||||
services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
|
||||
services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
services.AddSingleton<IScreenshotImageCropper, UnavailableScreenshotImageCropper>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class MacOsCalendarEventMapper
|
||||
{
|
||||
public static bool IsTeamsEvent(MacOsCalendarEvent calendarEvent)
|
||||
{
|
||||
return CalendarAppointmentMetadata.IsTeamsAppointment(
|
||||
calendarEvent.Subject,
|
||||
calendarEvent.Location,
|
||||
calendarEvent.Notes,
|
||||
calendarEvent.Url);
|
||||
}
|
||||
|
||||
public static MeetingMetadata CreateMetadata(MacOsCalendarEvent calendarEvent)
|
||||
{
|
||||
return CalendarAppointmentMetadata.Create(
|
||||
calendarEvent.Subject,
|
||||
calendarEvent.Attendees,
|
||||
calendarEvent.Notes,
|
||||
calendarEvent.End);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
using System.Diagnostics;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed class MacOsWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
|
||||
{
|
||||
private readonly IOptions<MeetingAssistantOptions> options;
|
||||
|
||||
public MacOsWorkflowRulesEditorWindowService(IOptions<MeetingAssistantOptions> options)
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void Show()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var url = $"{options.Value.Api.PublicBaseUrl.TrimEnd('/')}/diagnostics/settings-and-logs";
|
||||
Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "/usr/bin/open",
|
||||
ArgumentList = { url },
|
||||
UseShellExecute = false
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MacOsWorkflowRulesEditorSession
|
||||
{
|
||||
private readonly WorkflowRulesEditorChatViewModel viewModel;
|
||||
private readonly SemaphoreSlim sendLock = new(1, 1);
|
||||
|
||||
public MacOsWorkflowRulesEditorSession(WorkflowRulesEditorChatViewModel viewModel)
|
||||
{
|
||||
this.viewModel = viewModel;
|
||||
}
|
||||
|
||||
public async Task<MacOsWorkflowRulesEditorChatResponse> SendAsync(
|
||||
string? message,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(message))
|
||||
{
|
||||
return Snapshot();
|
||||
}
|
||||
|
||||
await sendLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
viewModel.Draft = message;
|
||||
await viewModel.SendAsync(cancellationToken);
|
||||
return Snapshot();
|
||||
}
|
||||
finally
|
||||
{
|
||||
sendLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private MacOsWorkflowRulesEditorChatResponse Snapshot()
|
||||
{
|
||||
return new MacOsWorkflowRulesEditorChatResponse(
|
||||
viewModel.Messages
|
||||
.Where(item => item.Kind == WorkflowRulesEditorConversationItemKind.Message)
|
||||
.Select(item => new MacOsWorkflowRulesEditorMessage(
|
||||
item.Message!.Role.ToString().ToLowerInvariant(),
|
||||
item.Message.Content))
|
||||
.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorChatRequest(string? Message);
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorChatResponse(
|
||||
IReadOnlyList<MacOsWorkflowRulesEditorMessage> Messages);
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorMessage(string Role, string Content);
|
||||
|
||||
public static class MacOsWorkflowRulesEditorPage
|
||||
{
|
||||
public const string Html = """
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Meeting Summary Agent</title>
|
||||
<style>
|
||||
:root { color-scheme: dark; font: 15px -apple-system, BlinkMacSystemFont, sans-serif; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: #171719; color: #f5f5f7; }
|
||||
main { height: 100vh; display: grid; grid-template-rows: auto 1fr auto; padding: 18px; gap: 14px; }
|
||||
h1 { margin: 0; font-size: 18px; }
|
||||
#messages { overflow: auto; display: flex; flex-direction: column; gap: 10px; }
|
||||
.message { max-width: 82%; padding: 10px 12px; border: 1px solid #414147; border-radius: 10px; white-space: pre-wrap; }
|
||||
.user { align-self: flex-end; background: #293650; }
|
||||
.agent { align-self: flex-start; background: #232326; }
|
||||
form { display: grid; grid-template-columns: 1fr auto; gap: 10px; }
|
||||
textarea { min-height: 78px; resize: vertical; border-radius: 9px; border: 1px solid #55555d; background: #222225; color: inherit; padding: 10px; font: inherit; }
|
||||
button { min-width: 96px; border: 0; border-radius: 9px; background: #0a84ff; color: white; font-weight: 600; }
|
||||
button:disabled { opacity: .55; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Meeting Summary Agent</h1>
|
||||
<section id="messages"></section>
|
||||
<form id="chat">
|
||||
<textarea id="prompt" placeholder="ask me what I can do for you"></textarea>
|
||||
<button id="send">Send</button>
|
||||
</form>
|
||||
</main>
|
||||
<script>
|
||||
const form = document.querySelector('#chat');
|
||||
const prompt = document.querySelector('#prompt');
|
||||
const send = document.querySelector('#send');
|
||||
const messages = document.querySelector('#messages');
|
||||
const render = items => {
|
||||
messages.replaceChildren(...items.map(item => {
|
||||
const card = document.createElement('div');
|
||||
card.className = `message ${item.role}`;
|
||||
card.textContent = item.content;
|
||||
return card;
|
||||
}));
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
};
|
||||
form.addEventListener('submit', async event => {
|
||||
event.preventDefault();
|
||||
const message = prompt.value.trim();
|
||||
if (!message) return;
|
||||
prompt.value = '';
|
||||
send.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/diagnostics/settings-and-logs/chat', {
|
||||
method: 'POST',
|
||||
headers: {'content-type': 'application/json'},
|
||||
body: JSON.stringify({message})
|
||||
});
|
||||
const body = await response.json();
|
||||
render(body.messages);
|
||||
} finally {
|
||||
send.disabled = false;
|
||||
prompt.focus();
|
||||
}
|
||||
});
|
||||
prompt.addEventListener('keydown', event => {
|
||||
if (event.key === 'Enter' && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
form.requestSubmit();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
""";
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||
<MicrosoftSpeechVersion>1.50.0</MicrosoftSpeechVersion>
|
||||
<MicrosoftSpeechVersion>1.51.2</MicrosoftSpeechVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
@@ -15,21 +15,37 @@
|
||||
<UseWPF>true</UseWPF>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="$([MSBuild]::IsOSPlatform('OSX')) and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||
<MacOsNativeHelpersEnabled>true</MacOsNativeHelpersEnabled>
|
||||
<MacOsAudioCaptureSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/main.swift</MacOsAudioCaptureSource>
|
||||
<MacOsAudioCaptureInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/Info.plist</MacOsAudioCaptureInfoPlist>
|
||||
<MacOsAudioCaptureOutputPath>Native/macos-meeting-audio-capture</MacOsAudioCaptureOutputPath>
|
||||
<MacOsAudioCaptureRid Condition="'$(RuntimeIdentifier)' != ''">$(RuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||
<MacOsAudioCaptureRid Condition="'$(MacOsAudioCaptureRid)' == ''">$(NETCoreSdkRuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureRid)' == 'osx-x64'">x86_64</MacOsAudioCaptureArchitecture>
|
||||
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureArchitecture)' == ''">arm64</MacOsAudioCaptureArchitecture>
|
||||
<MacOsDesktopControlsSource>$(MSBuildProjectDirectory)/Native/MacOsDesktopControls/main.swift</MacOsDesktopControlsSource>
|
||||
<MacOsDesktopControlsOutputPath>Native/macos-desktop-controls</MacOsDesktopControlsOutputPath>
|
||||
<MacOsMeetingIntegrationsSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/main.swift</MacOsMeetingIntegrationsSource>
|
||||
<MacOsMeetingIntegrationsInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/Info.plist</MacOsMeetingIntegrationsInfoPlist>
|
||||
<MacOsMeetingIntegrationsOutputPath>Native/macos-meeting-integrations</MacOsMeetingIntegrationsOutputPath>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.13.0" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
|
||||
<PackageReference Include="DiffPlex" Version="1.9.0" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
|
||||
<PackageReference Include="NAudio" Version="2.3.0" />
|
||||
<PackageReference Include="NCalcSync" Version="6.4.0" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
|
||||
<PackageReference Include="NAudio" Version="3.1.0" />
|
||||
<PackageReference Include="NCalcSync" Version="7.2.0" />
|
||||
<PackageReference Include="RazorLight" Version="2.3.1" />
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.10" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
|
||||
<PackageReference Include="Whisper.net" Version="1.9.1" />
|
||||
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
|
||||
<PackageReference Include="YamlDotNet" Version="18.1.0" />
|
||||
@@ -47,11 +63,11 @@
|
||||
<None Include="..\docs\meeting-assistant-configuration.md" Link="docs\meeting-assistant-configuration.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="Content\Project-AGENTS.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="..\openspec\specs\**\*.md" Link="openspec\specs\%(RecursiveDir)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="Native\MacOsMeetingIntegrations\calendar-events.js" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" TargetPath="Native\macos-calendar-events.js" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\aec_v1.fpie" Link="runtimes\win-x64\native\MASmodels\aec_v1.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\pns_avg4.fpie" Link="runtimes\win-x64\native\MASmodels\pns_avg4.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\*.fpie" Link="runtimes\win-x64\native\MASmodels\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -62,6 +78,26 @@
|
||||
<Copy SourceFiles="@(MeetingAssistantRootConfig)" DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="BuildMacOsNativeHelpers"
|
||||
AfterTargets="Build"
|
||||
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
|
||||
<MakeDir Directories="$(TargetDir)Native" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AVFoundation -framework CoreMedia -framework ScreenCaptureKit -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$(MacOsAudioCaptureInfoPlist)" "$(MacOsAudioCaptureSource)" -o "$(TargetDir)$(MacOsAudioCaptureOutputPath)"" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework Carbon -framework WebKit "$(MacOsDesktopControlsSource)" -o "$(TargetDir)$(MacOsDesktopControlsOutputPath)"" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework CoreGraphics -framework EventKit -framework ImageIO -framework UniformTypeIdentifiers -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$(MacOsMeetingIntegrationsInfoPlist)" "$(MacOsMeetingIntegrationsSource)" -o "$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)"" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CopyMacOsNativeHelpersToPublish"
|
||||
AfterTargets="Publish"
|
||||
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
|
||||
<MakeDir Directories="$(PublishDir)Native" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsAudioCaptureOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsDesktopControlsOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||
</Target>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||
|
||||
@@ -165,6 +165,8 @@ public sealed class RecordingInactivitySafeguardOptions
|
||||
|
||||
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
|
||||
|
||||
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
|
||||
|
||||
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
|
||||
@@ -181,10 +183,8 @@ public sealed class WhisperLocalOptions
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions
|
||||
public class PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string DockerCommand { get; set; } = "docker";
|
||||
|
||||
public string BaseImage { get; set; } = "python:3.11-slim";
|
||||
@@ -208,6 +208,11 @@ public sealed class PyannoteDiarizationOptions
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
|
||||
public sealed class AzureSpeechOptions
|
||||
{
|
||||
public string Endpoint { get; set; } = "";
|
||||
@@ -340,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
|
||||
|
||||
public int MaxSnippetsPerSpeaker { get; set; } = 3;
|
||||
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
|
||||
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
|
||||
|
||||
@@ -355,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
|
||||
public AzureSpeechOptions AzureSpeech { get; set; } = new();
|
||||
|
||||
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
|
||||
|
||||
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public int RequiredVectorsPerSpeaker { get; set; } = 5;
|
||||
|
||||
public int MaxVectorsPerIdentity { get; set; } = 1000;
|
||||
|
||||
public int OutlierPruningMinimumVectors { get; set; } = 20;
|
||||
|
||||
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
|
||||
|
||||
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
|
||||
|
||||
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
|
||||
|
||||
public double MinimumClusterCohesion { get; set; } = 0.75;
|
||||
|
||||
public double MinimumIdentitySimilarity { get; set; } = 0.75;
|
||||
|
||||
public double MinimumSimilarityMargin { get; set; } = 0.05;
|
||||
|
||||
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
|
||||
|
||||
public string PackageVersion { get; set; } = "0.1.4";
|
||||
|
||||
public string PythonCommand { get; set; } = "python";
|
||||
|
||||
public string TorchVersion { get; set; } = "2.14.0+cpu";
|
||||
|
||||
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
|
||||
|
||||
public string WebRtcVadVersion { get; set; } = "2.0.14";
|
||||
|
||||
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
|
||||
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
|
||||
}
|
||||
|
||||
public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
@@ -365,9 +411,8 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
|
||||
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
|
||||
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new()
|
||||
public PyannoteRuntimeOptions Diarization { get; set; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
CommandTimeout = TimeSpan.FromHours(1),
|
||||
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
|
||||
};
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
using MeetingAssistant.Calendar;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
|
||||
internal static class CalendarAppointmentMetadata
|
||||
{
|
||||
public static MeetingMetadata Create(
|
||||
string? subject,
|
||||
IEnumerable<string> attendees,
|
||||
string? body,
|
||||
DateTimeOffset end)
|
||||
{
|
||||
return new MeetingMetadata(
|
||||
ReadSubject(subject),
|
||||
NormalizeAttendees(attendees),
|
||||
ExtractAgenda(body ?? ""),
|
||||
end);
|
||||
}
|
||||
|
||||
public static string ReadSubject(string? subject)
|
||||
{
|
||||
var trimmed = subject?.Trim();
|
||||
return string.IsNullOrWhiteSpace(trimmed) ? "Teams meeting" : trimmed;
|
||||
}
|
||||
|
||||
public static bool IsTeamsAppointment(
|
||||
string? subject,
|
||||
string? location,
|
||||
string? body,
|
||||
string? url = null)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(
|
||||
subject ?? "",
|
||||
location ?? "",
|
||||
string.Join(Environment.NewLine, body, url));
|
||||
}
|
||||
|
||||
public static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || TeamsMeetingMarkerDetector.ContainsTeamsMarker(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim()
|
||||
.Replace("mailto:", "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
return emailStart > 0 && emailEnd > emailStart
|
||||
? attendee[(emailStart + 1)..emailEnd].Trim()
|
||||
: attendee.Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
|
||||
internal static class MeetingMetadataDiagnosticRunner
|
||||
{
|
||||
public static async Task<MeetingMetadataDiagnosticResult> RunAsync(
|
||||
string provider,
|
||||
DateTimeOffset startedAt,
|
||||
TimeSpan timeout,
|
||||
Func<CancellationToken, Task<MeetingMetadata?>> lookupAsync,
|
||||
string noMatchError,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var lookup = lookupAsync(cancellationToken);
|
||||
var metadata = timeout > TimeSpan.Zero
|
||||
? await lookup.WaitAsync(timeout, cancellationToken)
|
||||
: await lookup.WaitAsync(cancellationToken);
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
metadata is not null,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
metadata,
|
||||
metadata is null ? noMatchError : null);
|
||||
}
|
||||
catch (TimeoutException exception)
|
||||
{
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
false,
|
||||
true,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
false,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,18 +7,17 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
{
|
||||
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
|
||||
{
|
||||
var title = ReadSubject(appointment);
|
||||
return new MeetingMetadata(
|
||||
title,
|
||||
return CalendarAppointmentMetadata.Create(
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")),
|
||||
ReadAttendees(appointment),
|
||||
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")),
|
||||
OutlookClassicCom.ToLocalOffset(end));
|
||||
}
|
||||
|
||||
public static string ReadSubject(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
|
||||
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
|
||||
return CalendarAppointmentMetadata.ReadSubject(
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")));
|
||||
}
|
||||
|
||||
public static bool IsTeamsAppointment(object appointment)
|
||||
@@ -26,29 +25,12 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
|
||||
return CalendarAppointmentMetadata.IsTeamsAppointment(subject, location, body);
|
||||
}
|
||||
|
||||
public static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
return CalendarAppointmentMetadata.ExtractAgenda(body);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> ReadAttendees(object appointment)
|
||||
@@ -93,66 +75,7 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static bool IsTeamsJoinLine(string line)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim();
|
||||
var mailtoPrefix = "mailto:";
|
||||
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
return attendee[(emailStart + 1)..emailEnd].Trim();
|
||||
}
|
||||
|
||||
return attendee.Trim();
|
||||
return CalendarAppointmentMetadata.NormalizeAttendees(attendees);
|
||||
}
|
||||
|
||||
private static string FormatRecipient(object recipient)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Diagnostics;
|
||||
using MeetingAssistant.Calendar;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
@@ -36,54 +35,16 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var lookup = OutlookClassicCom.RunStaAsync(
|
||||
return await MeetingMetadataDiagnosticRunner.RunAsync(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
timeout,
|
||||
token => OutlookClassicCom.RunStaAsync(
|
||||
() => GetCurrentMeeting(startedAt),
|
||||
cancellationToken,
|
||||
"Meeting Assistant Outlook COM Lookup");
|
||||
var metadata = timeout > TimeSpan.Zero
|
||||
? await lookup.WaitAsync(timeout, cancellationToken)
|
||||
: await lookup.WaitAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
metadata is not null,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
metadata,
|
||||
metadata is null ? "No matching Teams appointment was found." : null);
|
||||
}
|
||||
catch (TimeoutException exception)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
false,
|
||||
true,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
false,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
token,
|
||||
"Meeting Assistant Outlook COM Lookup"),
|
||||
"No matching Teams appointment was found.",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt)
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import AppKit
|
||||
import Carbon
|
||||
import Foundation
|
||||
import WebKit
|
||||
|
||||
struct Manifest: Decodable {
|
||||
let baseUrl: String
|
||||
let statusUrl: String
|
||||
let openAgentUrl: String
|
||||
let profiles: [Profile]
|
||||
let hotkeys: [Hotkey]
|
||||
}
|
||||
|
||||
struct Profile: Decodable {
|
||||
let name: String
|
||||
let toggleShortcut: String
|
||||
}
|
||||
|
||||
struct Hotkey: Decodable {
|
||||
let shortcut: String
|
||||
let endpoint: String
|
||||
}
|
||||
|
||||
struct RecordingStatus: Decodable {
|
||||
let isRecording: Bool
|
||||
let state: String
|
||||
let launchProfile: String?
|
||||
}
|
||||
|
||||
@main
|
||||
final class DesktopControls: NSObject, NSApplicationDelegate, NSMenuDelegate {
|
||||
private let manifest: Manifest
|
||||
private var status = RecordingStatus(isRecording: false, state: "Idle", launchProfile: nil)
|
||||
private var statusItem: NSStatusItem!
|
||||
private var registeredHotkeys: [UInt32: Hotkey] = [:]
|
||||
private var timer: Timer?
|
||||
private var agentWindow: NSWindow?
|
||||
|
||||
override init() {
|
||||
guard CommandLine.arguments.count == 2,
|
||||
let data = try? Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[1])),
|
||||
let decoded = try? JSONDecoder().decode(Manifest.self, from: data) else {
|
||||
fputs("Usage: macos-desktop-controls <manifest.json>\n", stderr)
|
||||
exit(2)
|
||||
}
|
||||
manifest = decoded
|
||||
super.init()
|
||||
}
|
||||
|
||||
static func main() {
|
||||
let application = NSApplication.shared
|
||||
let delegate = DesktopControls()
|
||||
application.delegate = delegate
|
||||
application.setActivationPolicy(.accessory)
|
||||
application.run()
|
||||
}
|
||||
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.squareLength)
|
||||
statusItem.button?.image = NSImage(
|
||||
systemSymbolName: "waveform.circle",
|
||||
accessibilityDescription: "Meeting Assistant")
|
||||
let menu = NSMenu()
|
||||
menu.delegate = self
|
||||
statusItem.menu = menu
|
||||
registerHotkeys()
|
||||
refreshStatus()
|
||||
timer = Timer.scheduledTimer(withTimeInterval: 2, repeats: true) { [weak self] _ in
|
||||
self?.refreshStatus()
|
||||
}
|
||||
}
|
||||
|
||||
func applicationWillTerminate(_ notification: Notification) {
|
||||
timer?.invalidate()
|
||||
}
|
||||
|
||||
func menuWillOpen(_ menu: NSMenu) {
|
||||
rebuildMenu(menu)
|
||||
refreshStatus()
|
||||
}
|
||||
|
||||
private func rebuildMenu(_ menu: NSMenu) {
|
||||
menu.removeAllItems()
|
||||
let stateItem = NSMenuItem(title: tooltip, action: nil, keyEquivalent: "")
|
||||
stateItem.isEnabled = false
|
||||
menu.addItem(stateItem)
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(item("Open agent", #selector(openAgent)))
|
||||
|
||||
if status.isRecording {
|
||||
menu.addItem(item("Stop meeting recording and transcribe", #selector(stopRecording)))
|
||||
menu.addItem(item("Cancel meeting recording and discard", #selector(abortRecording)))
|
||||
} else {
|
||||
for (index, profile) in manifest.profiles.enumerated() {
|
||||
let entry = item(
|
||||
"Start meeting recording (\(profile.name))",
|
||||
#selector(startProfile(_:)))
|
||||
entry.tag = index
|
||||
entry.keyEquivalent = menuKey(from: profile.toggleShortcut)
|
||||
entry.keyEquivalentModifierMask = menuModifiers(from: profile.toggleShortcut)
|
||||
menu.addItem(entry)
|
||||
}
|
||||
}
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(item("Exit", #selector(exitApplication)))
|
||||
}
|
||||
|
||||
private var tooltip: String {
|
||||
if status.isRecording {
|
||||
return status.launchProfile.map { "Meeting Assistant — recording (\($0))" }
|
||||
?? "Meeting Assistant — recording"
|
||||
}
|
||||
return status.state.lowercased() == "summarizing"
|
||||
? "Meeting Assistant — summarizing"
|
||||
: "Meeting Assistant — idle"
|
||||
}
|
||||
|
||||
private func item(_ title: String, _ action: Selector) -> NSMenuItem {
|
||||
let result = NSMenuItem(title: title, action: action, keyEquivalent: "")
|
||||
result.target = self
|
||||
return result
|
||||
}
|
||||
|
||||
@objc private func openAgent() {
|
||||
if agentWindow == nil {
|
||||
let webView = WKWebView(frame: .zero)
|
||||
let window = NSWindow(
|
||||
contentRect: NSRect(x: 0, y: 0, width: 840, height: 760),
|
||||
styleMask: [.titled, .closable, .miniaturizable, .resizable],
|
||||
backing: .buffered,
|
||||
defer: false)
|
||||
window.title = "Meeting Summary Agent"
|
||||
window.contentView = webView
|
||||
window.center()
|
||||
window.isReleasedWhenClosed = false
|
||||
webView.load(URLRequest(url: URL(string: manifest.openAgentUrl)!))
|
||||
agentWindow = window
|
||||
}
|
||||
NSApplication.shared.activate(ignoringOtherApps: true)
|
||||
agentWindow?.makeKeyAndOrderFront(nil)
|
||||
}
|
||||
|
||||
@objc private func startProfile(_ sender: NSMenuItem) {
|
||||
let profile = manifest.profiles[sender.tag]
|
||||
post(path: "/profiles/\(escaped(profile.name))/recording/start")
|
||||
}
|
||||
|
||||
@objc private func stopRecording() {
|
||||
post(path: "/recording/stop")
|
||||
}
|
||||
|
||||
@objc private func abortRecording() {
|
||||
post(path: "/recording/abort")
|
||||
}
|
||||
|
||||
@objc private func exitApplication() {
|
||||
post(path: "/application/exit")
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
|
||||
NSApplication.shared.terminate(nil)
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshStatus() {
|
||||
guard let url = URL(string: manifest.statusUrl) else { return }
|
||||
URLSession.shared.dataTask(with: url) { [weak self] data, _, _ in
|
||||
guard let data,
|
||||
let next = try? JSONDecoder().decode(RecordingStatus.self, from: data) else { return }
|
||||
DispatchQueue.main.async {
|
||||
self?.status = next
|
||||
self?.statusItem.button?.image = NSImage(
|
||||
systemSymbolName: next.isRecording ? "record.circle.fill" : "waveform.circle",
|
||||
accessibilityDescription: self?.tooltip)
|
||||
}
|
||||
}.resume()
|
||||
}
|
||||
|
||||
private func post(path: String) {
|
||||
post(url: manifest.baseUrl + path)
|
||||
}
|
||||
|
||||
private func post(url value: String) {
|
||||
guard let url = URL(string: value) else { return }
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
URLSession.shared.dataTask(with: request) { [weak self] _, _, _ in
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) {
|
||||
self?.refreshStatus()
|
||||
}
|
||||
}.resume()
|
||||
}
|
||||
|
||||
private func registerHotkeys() {
|
||||
var eventType = EventTypeSpec(
|
||||
eventClass: OSType(kEventClassKeyboard),
|
||||
eventKind: OSType(kEventHotKeyPressed))
|
||||
InstallEventHandler(
|
||||
GetApplicationEventTarget(),
|
||||
{ _, event, context in
|
||||
guard let event, let context else { return noErr }
|
||||
var identifier = EventHotKeyID()
|
||||
GetEventParameter(
|
||||
event,
|
||||
EventParamName(kEventParamDirectObject),
|
||||
EventParamType(typeEventHotKeyID),
|
||||
nil,
|
||||
MemoryLayout<EventHotKeyID>.size,
|
||||
nil,
|
||||
&identifier)
|
||||
let owner = Unmanaged<DesktopControls>.fromOpaque(context).takeUnretainedValue()
|
||||
if let hotkey = owner.registeredHotkeys[identifier.id] {
|
||||
owner.post(path: hotkey.endpoint)
|
||||
}
|
||||
return noErr
|
||||
},
|
||||
1,
|
||||
&eventType,
|
||||
Unmanaged.passUnretained(self).toOpaque(),
|
||||
nil)
|
||||
|
||||
for (index, hotkey) in manifest.hotkeys.enumerated() {
|
||||
guard let keyCode = carbonKeyCode(from: hotkey.shortcut) else { continue }
|
||||
let identifier = EventHotKeyID(signature: fourCC("MTGA"), id: UInt32(index + 1))
|
||||
var reference: EventHotKeyRef?
|
||||
let result = RegisterEventHotKey(
|
||||
keyCode,
|
||||
carbonModifiers(from: hotkey.shortcut),
|
||||
identifier,
|
||||
GetApplicationEventTarget(),
|
||||
0,
|
||||
&reference)
|
||||
if result == noErr {
|
||||
registeredHotkeys[identifier.id] = hotkey
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func escaped(_ value: String) -> String {
|
||||
value.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? value
|
||||
}
|
||||
}
|
||||
|
||||
private func tokens(_ shortcut: String) -> [String] {
|
||||
shortcut.uppercased().split(separator: "+").map(String.init)
|
||||
}
|
||||
|
||||
private func carbonModifiers(from shortcut: String) -> UInt32 {
|
||||
let parts = tokens(shortcut)
|
||||
var result: UInt32 = 0
|
||||
if parts.contains("CTRL") || parts.contains("CONTROL") { result |= UInt32(controlKey) }
|
||||
if parts.contains("ALT") || parts.contains("OPTION") { result |= UInt32(optionKey) }
|
||||
if parts.contains("SHIFT") { result |= UInt32(shiftKey) }
|
||||
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
|
||||
result |= UInt32(cmdKey)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private func menuModifiers(from shortcut: String) -> NSEvent.ModifierFlags {
|
||||
let parts = tokens(shortcut)
|
||||
var result: NSEvent.ModifierFlags = []
|
||||
if parts.contains("CTRL") || parts.contains("CONTROL") { result.insert(.control) }
|
||||
if parts.contains("ALT") || parts.contains("OPTION") { result.insert(.option) }
|
||||
if parts.contains("SHIFT") { result.insert(.shift) }
|
||||
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
|
||||
result.insert(.command)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private func menuKey(from shortcut: String) -> String {
|
||||
tokens(shortcut).last?.lowercased() ?? ""
|
||||
}
|
||||
|
||||
private func carbonKeyCode(from shortcut: String) -> UInt32? {
|
||||
guard let key = tokens(shortcut).last, key.count == 1, let scalar = key.unicodeScalars.first else {
|
||||
return nil
|
||||
}
|
||||
let codes: [Character: UInt32] = [
|
||||
"A": 0, "S": 1, "D": 2, "F": 3, "H": 4, "G": 5, "Z": 6, "X": 7,
|
||||
"C": 8, "V": 9, "B": 11, "Q": 12, "W": 13, "E": 14, "R": 15,
|
||||
"Y": 16, "T": 17, "M": 46, "L": 37, "N": 45, "P": 35, "O": 31,
|
||||
"I": 34, "U": 32, "J": 38, "K": 40
|
||||
]
|
||||
return codes[Character(String(scalar))]
|
||||
}
|
||||
|
||||
private func fourCC(_ value: String) -> OSType {
|
||||
value.utf8.reduce(0) { ($0 << 8) + OSType($1) }
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>cloud.schweigert.meeting-assistant.audio-capture</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>Meeting Assistant Audio Capture</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Meeting Assistant records microphone audio for live meeting transcription.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,328 @@
|
||||
import AVFoundation
|
||||
import CoreMedia
|
||||
import Darwin
|
||||
import Foundation
|
||||
import ScreenCaptureKit
|
||||
|
||||
private enum CaptureFailure: Error, CustomStringConvertible {
|
||||
case usage
|
||||
case invalidFormat
|
||||
case microphonePermission
|
||||
case noDisplay
|
||||
case audioConverter(String)
|
||||
case systemAudio(String)
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .usage:
|
||||
return "Usage: macos-meeting-audio-capture <microphone|system> <sample-rate> <channels>"
|
||||
case .invalidFormat:
|
||||
return "Sample rate must be positive and channels must be 1 or 2."
|
||||
case .microphonePermission:
|
||||
return "Microphone permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Microphone."
|
||||
case .noDisplay:
|
||||
return "ScreenCaptureKit could not find a display for system-audio capture."
|
||||
case .audioConverter(let detail):
|
||||
return "Native PCM conversion failed: \(detail)"
|
||||
case .systemAudio(let detail):
|
||||
return "Screen Recording/System Audio permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Screen & System Audio Recording. \(detail)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final class ErrorReporter: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
|
||||
func write(_ message: String) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
let line = message.hasSuffix("\n") ? message : message + "\n"
|
||||
if let data = line.data(using: .utf8) {
|
||||
try? FileHandle.standardError.write(contentsOf: data)
|
||||
}
|
||||
}
|
||||
|
||||
func exit(with failure: CaptureFailure, code: Int32) -> Never {
|
||||
write(failure.description)
|
||||
Darwin.exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
private final class PcmWriter: @unchecked Sendable {
|
||||
private let targetFormat: AVAudioFormat
|
||||
private let output = FileHandle.standardOutput
|
||||
private let reporter: ErrorReporter
|
||||
private let lock = NSLock()
|
||||
private var converter: AVAudioConverter?
|
||||
|
||||
init(sampleRate: Int, channels: Int, reporter: ErrorReporter) throws {
|
||||
guard let format = AVAudioFormat(
|
||||
commonFormat: .pcmFormatInt16,
|
||||
sampleRate: Double(sampleRate),
|
||||
channels: AVAudioChannelCount(channels),
|
||||
interleaved: true
|
||||
) else {
|
||||
throw CaptureFailure.invalidFormat
|
||||
}
|
||||
|
||||
targetFormat = format
|
||||
self.reporter = reporter
|
||||
}
|
||||
|
||||
func write(_ input: AVAudioPCMBuffer) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
do {
|
||||
let outputBuffer = try convert(input)
|
||||
let buffers = UnsafeMutableAudioBufferListPointer(outputBuffer.mutableAudioBufferList)
|
||||
guard let audioBuffer = buffers.first, let bytes = audioBuffer.mData else {
|
||||
return
|
||||
}
|
||||
|
||||
let data = Data(bytes: bytes, count: Int(audioBuffer.mDataByteSize))
|
||||
try output.write(contentsOf: data)
|
||||
} catch {
|
||||
reporter.exit(with: .audioConverter(error.localizedDescription), code: 70)
|
||||
}
|
||||
}
|
||||
|
||||
private func convert(_ input: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer {
|
||||
let sourceFormat = input.format
|
||||
let activeConverter: AVAudioConverter
|
||||
if let converter {
|
||||
activeConverter = converter
|
||||
} else {
|
||||
guard let created = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
|
||||
throw CaptureFailure.audioConverter("AVAudioConverter could not convert \(sourceFormat) to \(targetFormat).")
|
||||
}
|
||||
converter = created
|
||||
activeConverter = created
|
||||
}
|
||||
|
||||
let ratio = targetFormat.sampleRate / sourceFormat.sampleRate
|
||||
let capacity = max(1, AVAudioFrameCount(ceil(Double(input.frameLength) * ratio)) + 32)
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
|
||||
throw CaptureFailure.audioConverter("Could not allocate the target PCM buffer.")
|
||||
}
|
||||
|
||||
var suppliedInput = false
|
||||
var conversionError: NSError?
|
||||
let status = activeConverter.convert(to: converted, error: &conversionError) { _, inputStatus in
|
||||
if suppliedInput {
|
||||
inputStatus.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
|
||||
suppliedInput = true
|
||||
inputStatus.pointee = .haveData
|
||||
return input
|
||||
}
|
||||
|
||||
if status == .error {
|
||||
throw conversionError ?? CaptureFailure.audioConverter("AVAudioConverter returned an unknown error.")
|
||||
}
|
||||
|
||||
return converted
|
||||
}
|
||||
}
|
||||
|
||||
private final class MicrophoneCapture: @unchecked Sendable {
|
||||
private let engine = AVAudioEngine()
|
||||
private let writer: PcmWriter
|
||||
|
||||
init(writer: PcmWriter) {
|
||||
self.writer = writer
|
||||
}
|
||||
|
||||
func run() async throws -> Never {
|
||||
let authorized = await withCheckedContinuation { continuation in
|
||||
AVCaptureDevice.requestAccess(for: .audio) { granted in
|
||||
continuation.resume(returning: granted)
|
||||
}
|
||||
}
|
||||
guard authorized else {
|
||||
throw CaptureFailure.microphonePermission
|
||||
}
|
||||
|
||||
let input = engine.inputNode
|
||||
let inputFormat = input.outputFormat(forBus: 0)
|
||||
input.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [writer] buffer, _ in
|
||||
writer.write(buffer)
|
||||
}
|
||||
engine.prepare()
|
||||
try engine.start()
|
||||
|
||||
while true {
|
||||
try await Task.sleep(for: .seconds(3_600))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final class SystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
|
||||
private let writer: PcmWriter
|
||||
private let reporter: ErrorReporter
|
||||
private let sampleQueue = DispatchQueue(label: "meeting-assistant.system-audio")
|
||||
private var stream: SCStream?
|
||||
|
||||
init(writer: PcmWriter, reporter: ErrorReporter) {
|
||||
self.writer = writer
|
||||
self.reporter = reporter
|
||||
}
|
||||
|
||||
func run(sampleRate: Int, channels: Int) async throws -> Never {
|
||||
let content: SCShareableContent
|
||||
do {
|
||||
content = try await SCShareableContent.excludingDesktopWindows(
|
||||
false,
|
||||
onScreenWindowsOnly: false
|
||||
)
|
||||
} catch {
|
||||
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||
}
|
||||
|
||||
guard let display = content.displays.first else {
|
||||
throw CaptureFailure.noDisplay
|
||||
}
|
||||
|
||||
let filter = SCContentFilter(
|
||||
display: display,
|
||||
excludingApplications: [],
|
||||
exceptingWindows: []
|
||||
)
|
||||
let configuration = SCStreamConfiguration()
|
||||
configuration.capturesAudio = true
|
||||
configuration.excludesCurrentProcessAudio = false
|
||||
configuration.sampleRate = sampleRate
|
||||
configuration.channelCount = channels
|
||||
configuration.width = 2
|
||||
configuration.height = 2
|
||||
configuration.showsCursor = false
|
||||
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
|
||||
|
||||
let captureStream = SCStream(filter: filter, configuration: configuration, delegate: self)
|
||||
try captureStream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
|
||||
do {
|
||||
try await captureStream.startCapture()
|
||||
} catch {
|
||||
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||
}
|
||||
stream = captureStream
|
||||
|
||||
while true {
|
||||
try await Task.sleep(for: .seconds(3_600))
|
||||
}
|
||||
}
|
||||
|
||||
func stream(
|
||||
_ stream: SCStream,
|
||||
didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
|
||||
of outputType: SCStreamOutputType
|
||||
) {
|
||||
guard outputType == .audio,
|
||||
sampleBuffer.isValid,
|
||||
CMSampleBufferDataIsReady(sampleBuffer),
|
||||
let description = CMSampleBufferGetFormatDescription(sampleBuffer),
|
||||
let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description)
|
||||
else {
|
||||
return
|
||||
}
|
||||
|
||||
let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
|
||||
guard frameCount > 0,
|
||||
let sourceFormat = AVAudioFormat(streamDescription: streamDescription)
|
||||
else {
|
||||
return
|
||||
}
|
||||
|
||||
var bufferListSize = 0
|
||||
var retainedBlockBuffer: CMBlockBuffer?
|
||||
var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||
sampleBuffer,
|
||||
bufferListSizeNeededOut: &bufferListSize,
|
||||
bufferListOut: nil,
|
||||
bufferListSize: 0,
|
||||
blockBufferAllocator: nil,
|
||||
blockBufferMemoryAllocator: nil,
|
||||
flags: 0,
|
||||
blockBufferOut: &retainedBlockBuffer
|
||||
)
|
||||
guard status == noErr else {
|
||||
reporter.exit(
|
||||
with: .audioConverter("CoreMedia could not describe system audio (OSStatus \(status))."),
|
||||
code: 71
|
||||
)
|
||||
}
|
||||
|
||||
let rawBufferList = UnsafeMutableRawPointer.allocate(
|
||||
byteCount: bufferListSize,
|
||||
alignment: MemoryLayout<AudioBufferList>.alignment
|
||||
)
|
||||
defer { rawBufferList.deallocate() }
|
||||
let audioBufferList = rawBufferList.bindMemory(to: AudioBufferList.self, capacity: 1)
|
||||
status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||
sampleBuffer,
|
||||
bufferListSizeNeededOut: nil,
|
||||
bufferListOut: audioBufferList,
|
||||
bufferListSize: bufferListSize,
|
||||
blockBufferAllocator: nil,
|
||||
blockBufferMemoryAllocator: nil,
|
||||
flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment),
|
||||
blockBufferOut: &retainedBlockBuffer
|
||||
)
|
||||
guard status == noErr,
|
||||
let buffer = AVAudioPCMBuffer(
|
||||
pcmFormat: sourceFormat,
|
||||
bufferListNoCopy: audioBufferList,
|
||||
deallocator: nil
|
||||
)
|
||||
else {
|
||||
reporter.exit(
|
||||
with: .audioConverter("CoreMedia could not expose system audio (OSStatus \(status))."),
|
||||
code: 71
|
||||
)
|
||||
}
|
||||
|
||||
buffer.frameLength = frameCount
|
||||
writer.write(buffer)
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
||||
reporter.exit(with: .systemAudio(error.localizedDescription), code: 72)
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
private struct MacOsMeetingAudioCapture {
|
||||
static func main() async {
|
||||
let reporter = ErrorReporter()
|
||||
do {
|
||||
let arguments = CommandLine.arguments
|
||||
guard arguments.count == 4,
|
||||
let sampleRate = Int(arguments[2]),
|
||||
let channels = Int(arguments[3])
|
||||
else {
|
||||
throw CaptureFailure.usage
|
||||
}
|
||||
guard sampleRate > 0, channels == 1 || channels == 2 else {
|
||||
throw CaptureFailure.invalidFormat
|
||||
}
|
||||
|
||||
let writer = try PcmWriter(sampleRate: sampleRate, channels: channels, reporter: reporter)
|
||||
switch arguments[1] {
|
||||
case "microphone":
|
||||
try await MicrophoneCapture(writer: writer).run()
|
||||
case "system":
|
||||
try await SystemAudioCapture(writer: writer, reporter: reporter)
|
||||
.run(sampleRate: sampleRate, channels: channels)
|
||||
default:
|
||||
throw CaptureFailure.usage
|
||||
}
|
||||
} catch let failure as CaptureFailure {
|
||||
reporter.exit(with: failure, code: 64)
|
||||
} catch {
|
||||
reporter.write(error.localizedDescription)
|
||||
Darwin.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>cloud.schweigert.meeting-assistant.integrations</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>Meeting Assistant Integrations</string>
|
||||
<key>NSCalendarsUsageDescription</key>
|
||||
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,68 @@
|
||||
function safe(read, fallback) {
|
||||
try {
|
||||
const value = read();
|
||||
return value === null || value === undefined ? fallback : value;
|
||||
} catch (_) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function participantText(participant) {
|
||||
const name = String(safe(() => participant.displayName(), "")).trim();
|
||||
const email = String(safe(() => participant.email(), "")).trim();
|
||||
if (!name) {
|
||||
return email;
|
||||
}
|
||||
if (!email || name.toLowerCase() === email.toLowerCase()) {
|
||||
return name;
|
||||
}
|
||||
return `${name} <${email}>`;
|
||||
}
|
||||
|
||||
function run(argv) {
|
||||
if (argv.length !== 2) {
|
||||
throw new Error("Usage: calendar-events.js <from-iso8601> <to-iso8601>");
|
||||
}
|
||||
|
||||
const from = new Date(argv[0]);
|
||||
const to = new Date(argv[1]);
|
||||
const calendar = Application("Calendar");
|
||||
const result = [];
|
||||
|
||||
for (const sourceCalendar of calendar.calendars()) {
|
||||
const events = sourceCalendar.events.whose({
|
||||
_and: [
|
||||
{ startDate: { _lessThan: to } },
|
||||
{ endDate: { _greaterThan: from } }
|
||||
]
|
||||
})();
|
||||
for (const event of events) {
|
||||
const start = event.startDate();
|
||||
const end = event.endDate();
|
||||
const attendees = safe(() => event.attendees(), [])
|
||||
.map(participantText)
|
||||
.filter(value => value.length > 0);
|
||||
const organizer = String(safe(() => event.organizer(), "")).trim();
|
||||
if (organizer && !attendees.includes(organizer)) {
|
||||
attendees.unshift(organizer);
|
||||
}
|
||||
|
||||
result.push({
|
||||
id: String(safe(() => event.uid(), `${start.toISOString()}|${event.summary()}`)),
|
||||
subject: String(safe(() => event.summary(), "")),
|
||||
start: start.toISOString(),
|
||||
end: end.toISOString(),
|
||||
location: String(safe(() => event.location(), "")),
|
||||
notes: String(safe(() => event.description(), "")),
|
||||
url: String(safe(() => event.url(), "")),
|
||||
attendees: attendees,
|
||||
isCanceled: String(safe(() => event.status(), ""))
|
||||
.toLowerCase()
|
||||
.includes("cancel")
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
result.sort((left, right) => left.start.localeCompare(right.start));
|
||||
return JSON.stringify(result);
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import EventKit
|
||||
import Foundation
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
private struct CalendarEventOutput: Encodable {
|
||||
let id: String
|
||||
let subject: String
|
||||
let start: Date
|
||||
let end: Date
|
||||
let location: String
|
||||
let notes: String
|
||||
let url: String
|
||||
let attendees: [String]
|
||||
let isCanceled: Bool
|
||||
}
|
||||
|
||||
private struct PromptRequest: Decodable {
|
||||
let subject: String
|
||||
let start: Date
|
||||
}
|
||||
|
||||
private struct PromptOutput: Encodable {
|
||||
let response: String
|
||||
}
|
||||
|
||||
private struct ScreenshotOutput: Encodable {
|
||||
let windowTitle: String?
|
||||
}
|
||||
|
||||
private struct SelfTestOutput: Encodable {
|
||||
let status: String
|
||||
let features: [String]
|
||||
}
|
||||
|
||||
private enum IntegrationError: LocalizedError {
|
||||
case usage
|
||||
case invalidDate(String)
|
||||
case calendarPermission
|
||||
case screenRecordingPermission
|
||||
case noFrontmostApplication
|
||||
case noActiveWindow
|
||||
case screenshotCapture
|
||||
case screenshotWrite
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .usage:
|
||||
return "Usage: macos-meeting-integrations self-test | calendar <from-iso8601> <to-iso8601> | prompt <request.json> | screenshot <output.png> | crop <input.png> <output.png> <x> <y> <width> <height>"
|
||||
case .invalidDate(let value):
|
||||
return "Invalid ISO-8601 date: \(value)"
|
||||
case .calendarPermission:
|
||||
return "Calendar Full Access permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Calendars."
|
||||
case .screenRecordingPermission:
|
||||
return "Screen & System Audio Recording permission is required. Enable Meeting Assistant under System Settings > Privacy & Security."
|
||||
case .noFrontmostApplication:
|
||||
return "Could not determine the frontmost macOS application."
|
||||
case .noActiveWindow:
|
||||
return "Could not find an on-screen window for the frontmost macOS application."
|
||||
case .screenshotCapture:
|
||||
return "macOS could not capture the active application window."
|
||||
case .screenshotWrite:
|
||||
return "macOS could not write the active-window screenshot as PNG."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
private enum MacOsMeetingIntegrations {
|
||||
@MainActor
|
||||
static func main() async {
|
||||
do {
|
||||
let arguments = Array(CommandLine.arguments.dropFirst())
|
||||
guard let command = arguments.first else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
|
||||
switch command {
|
||||
case "self-test":
|
||||
try writeJson(SelfTestOutput(
|
||||
status: "ok",
|
||||
features: ["calendar", "prompt", "screenshot", "crop"]))
|
||||
case "calendar":
|
||||
guard arguments.count == 3 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
let from = try parseDate(arguments[1])
|
||||
let to = try parseDate(arguments[2])
|
||||
try await writeCalendar(from: from, to: to)
|
||||
case "prompt":
|
||||
guard arguments.count == 2 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try showPrompt(requestPath: arguments[1])
|
||||
case "screenshot":
|
||||
guard arguments.count == 2 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try captureScreenshot(outputPath: arguments[1])
|
||||
case "crop":
|
||||
guard arguments.count == 7,
|
||||
let x = Int(arguments[3]),
|
||||
let y = Int(arguments[4]),
|
||||
let width = Int(arguments[5]),
|
||||
let height = Int(arguments[6]) else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try cropScreenshot(
|
||||
inputPath: arguments[1],
|
||||
outputPath: arguments[2],
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height)
|
||||
default:
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
} catch {
|
||||
fputs("\(error.localizedDescription)\n", stderr)
|
||||
if let integrationError = error as? IntegrationError,
|
||||
case .calendarPermission = integrationError {
|
||||
exit(77)
|
||||
}
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
private static func parseDate(_ value: String) throws -> Date {
|
||||
let formatter = ISO8601DateFormatter()
|
||||
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
|
||||
if let date = formatter.date(from: value) {
|
||||
return date
|
||||
}
|
||||
|
||||
formatter.formatOptions = [.withInternetDateTime]
|
||||
guard let date = formatter.date(from: value) else {
|
||||
throw IntegrationError.invalidDate(value)
|
||||
}
|
||||
return date
|
||||
}
|
||||
|
||||
private static func writeCalendar(from: Date, to: Date) async throws {
|
||||
let store = EKEventStore()
|
||||
guard try await requestCalendarAccess(store) else {
|
||||
throw IntegrationError.calendarPermission
|
||||
}
|
||||
|
||||
let predicate = store.predicateForEvents(withStart: from, end: to, calendars: nil)
|
||||
let events = store.events(matching: predicate)
|
||||
.map { event in
|
||||
CalendarEventOutput(
|
||||
id: event.eventIdentifier ?? "\(event.startDate.timeIntervalSince1970)|\(event.title ?? "")",
|
||||
subject: event.title ?? "",
|
||||
start: event.startDate,
|
||||
end: event.endDate,
|
||||
location: event.location ?? "",
|
||||
notes: event.notes ?? "",
|
||||
url: event.url?.absoluteString ?? "",
|
||||
attendees: attendees(from: event),
|
||||
isCanceled: event.status == .canceled)
|
||||
}
|
||||
.sorted { $0.start < $1.start }
|
||||
try writeJson(events)
|
||||
}
|
||||
|
||||
private static func requestCalendarAccess(_ store: EKEventStore) async throws -> Bool {
|
||||
if #available(macOS 14.0, *) {
|
||||
return try await store.requestFullAccessToEvents()
|
||||
}
|
||||
|
||||
return try await withCheckedThrowingContinuation { continuation in
|
||||
store.requestAccess(to: .event) { granted, error in
|
||||
if let error {
|
||||
continuation.resume(throwing: error)
|
||||
} else {
|
||||
continuation.resume(returning: granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func attendees(from event: EKEvent) -> [String] {
|
||||
var values: [String] = []
|
||||
if let organizer = event.organizer,
|
||||
let value = participantText(organizer) {
|
||||
values.append(value)
|
||||
}
|
||||
for participant in event.attendees ?? [] {
|
||||
if let value = participantText(participant) {
|
||||
values.append(value)
|
||||
}
|
||||
}
|
||||
return Array(NSOrderedSet(array: values)) as? [String] ?? values
|
||||
}
|
||||
|
||||
private static func participantText(_ participant: EKParticipant) -> String? {
|
||||
let name = participant.name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let email = participant.url
|
||||
.absoluteString
|
||||
.replacingOccurrences(of: "mailto:", with: "", options: .caseInsensitive)
|
||||
if name.isEmpty {
|
||||
return email.isEmpty ? nil : email
|
||||
}
|
||||
if email.isEmpty || name.caseInsensitiveCompare(email) == .orderedSame {
|
||||
return name
|
||||
}
|
||||
return "\(name) <\(email)>"
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private static func showPrompt(requestPath: String) throws {
|
||||
let data = try Data(contentsOf: URL(fileURLWithPath: requestPath))
|
||||
let decoder = JSONDecoder()
|
||||
decoder.dateDecodingStrategy = .iso8601
|
||||
let request = try decoder.decode(PromptRequest.self, from: data)
|
||||
|
||||
let application = NSApplication.shared
|
||||
application.setActivationPolicy(.accessory)
|
||||
application.activate(ignoringOtherApps: true)
|
||||
|
||||
let alert = NSAlert()
|
||||
alert.messageText = "Record meeting?"
|
||||
alert.informativeText = "\(request.subject) starts at \(request.start.formatted(date: .omitted, time: .shortened))."
|
||||
alert.alertStyle = .informational
|
||||
alert.addButton(withTitle: "Record")
|
||||
alert.addButton(withTitle: "Skip")
|
||||
let response = alert.runModal() == .alertFirstButtonReturn ? "record" : "skip"
|
||||
try writeJson(PromptOutput(response: response))
|
||||
}
|
||||
|
||||
private static func captureScreenshot(outputPath: String) throws {
|
||||
guard CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() else {
|
||||
throw IntegrationError.screenRecordingPermission
|
||||
}
|
||||
guard let frontmost = NSWorkspace.shared.frontmostApplication else {
|
||||
throw IntegrationError.noFrontmostApplication
|
||||
}
|
||||
guard let windows = CGWindowListCopyWindowInfo(
|
||||
[.optionOnScreenOnly, .excludeDesktopElements],
|
||||
kCGNullWindowID) as? [[String: Any]] else {
|
||||
throw IntegrationError.noActiveWindow
|
||||
}
|
||||
|
||||
let candidate = windows.first { window in
|
||||
let ownerPid = window[kCGWindowOwnerPID as String] as? Int32
|
||||
let layer = window[kCGWindowLayer as String] as? Int
|
||||
let alpha = window[kCGWindowAlpha as String] as? Double
|
||||
guard ownerPid == frontmost.processIdentifier,
|
||||
layer == 0,
|
||||
(alpha ?? 1) > 0,
|
||||
let bounds = window[kCGWindowBounds as String] as? [String: Any],
|
||||
let width = bounds["Width"] as? Double,
|
||||
let height = bounds["Height"] as? Double else {
|
||||
return false
|
||||
}
|
||||
return width > 1 && height > 1
|
||||
}
|
||||
guard let candidate,
|
||||
let windowNumber = candidate[kCGWindowNumber as String] as? UInt32 else {
|
||||
throw IntegrationError.noActiveWindow
|
||||
}
|
||||
guard let image = CGWindowListCreateImage(
|
||||
.null,
|
||||
.optionIncludingWindow,
|
||||
CGWindowID(windowNumber),
|
||||
[.boundsIgnoreFraming, .bestResolution]) else {
|
||||
throw IntegrationError.screenshotCapture
|
||||
}
|
||||
|
||||
try writePng(image, to: outputPath)
|
||||
|
||||
try writeJson(ScreenshotOutput(
|
||||
windowTitle: candidate[kCGWindowName as String] as? String))
|
||||
}
|
||||
|
||||
private static func cropScreenshot(
|
||||
inputPath: String,
|
||||
outputPath: String,
|
||||
x: Int,
|
||||
y: Int,
|
||||
width: Int,
|
||||
height: Int
|
||||
) throws {
|
||||
guard x >= 0,
|
||||
y >= 0,
|
||||
width > 0,
|
||||
height > 0,
|
||||
let source = CGImageSourceCreateWithURL(
|
||||
URL(fileURLWithPath: inputPath) as CFURL,
|
||||
nil),
|
||||
let image = CGImageSourceCreateImageAtIndex(source, 0, nil),
|
||||
x + width <= image.width,
|
||||
y + height <= image.height,
|
||||
let cropped = image.cropping(to: CGRect(
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height)) else {
|
||||
throw IntegrationError.screenshotCapture
|
||||
}
|
||||
|
||||
try writePng(cropped, to: outputPath)
|
||||
try writeJson(["status": "ok"])
|
||||
}
|
||||
|
||||
private static func writePng(_ image: CGImage, to outputPath: String) throws {
|
||||
let outputUrl = URL(fileURLWithPath: outputPath) as CFURL
|
||||
guard let destination = CGImageDestinationCreateWithURL(
|
||||
outputUrl,
|
||||
UTType.png.identifier as CFString,
|
||||
1,
|
||||
nil) else {
|
||||
throw IntegrationError.screenshotWrite
|
||||
}
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
guard CGImageDestinationFinalize(destination) else {
|
||||
throw IntegrationError.screenshotWrite
|
||||
}
|
||||
}
|
||||
|
||||
private static func writeJson<T: Encodable>(_ value: T) throws {
|
||||
let encoder = JSONEncoder()
|
||||
encoder.dateEncodingStrategy = .iso8601
|
||||
encoder.outputFormatting = [.sortedKeys]
|
||||
FileHandle.standardOutput.write(try encoder.encode(value))
|
||||
FileHandle.standardOutput.write(Data([0x0A]))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant;
|
||||
|
||||
public sealed record PlatformCapabilityReport(
|
||||
string Host,
|
||||
string MeetingMetadataProvider,
|
||||
string CalendarMeetingProvider,
|
||||
string MeetingStartPromptProvider,
|
||||
string ScreenshotProvider,
|
||||
string WorkflowEditorProvider,
|
||||
bool WorkflowEditorAvailable,
|
||||
bool CalendarRecordingPromptsEnabled,
|
||||
bool SpeakerIdentificationEnabled,
|
||||
bool ScreenshotOcrEnabled,
|
||||
bool FunAsrBackendEnabled,
|
||||
bool FunAsrDiarizationEnabled,
|
||||
bool WhisperLocalDiarizationEnabled)
|
||||
{
|
||||
public static PlatformCapabilityReport Create(
|
||||
HostOperatingSystem host,
|
||||
MeetingAssistantOptions options,
|
||||
IMeetingMetadataProvider meetingMetadataProvider,
|
||||
ICalendarMeetingProvider calendarMeetingProvider,
|
||||
IMeetingStartPromptService meetingStartPromptProvider,
|
||||
IActiveWindowScreenshotCapture screenshotProvider,
|
||||
IWorkflowRulesEditorWindowService workflowEditorProvider)
|
||||
{
|
||||
return new PlatformCapabilityReport(
|
||||
host.Kind switch
|
||||
{
|
||||
HostOperatingSystemKind.Windows => "Windows",
|
||||
HostOperatingSystemKind.MacOS => "macOS",
|
||||
_ => "Unsupported"
|
||||
},
|
||||
meetingMetadataProvider.GetType().Name,
|
||||
calendarMeetingProvider.GetType().Name,
|
||||
meetingStartPromptProvider.GetType().Name,
|
||||
screenshotProvider.GetType().Name,
|
||||
workflowEditorProvider.GetType().Name,
|
||||
workflowEditorProvider is not NoopWorkflowRulesEditorWindowService,
|
||||
options.CalendarRecordingPrompts.Enabled,
|
||||
options.SpeakerIdentification.Enabled,
|
||||
options.Screenshots.Ocr.Enabled,
|
||||
options.FunAsr.Backend.Enabled,
|
||||
options.FunAsr.Diarization.Enabled,
|
||||
options.WhisperLocal.Diarization.Enabled);
|
||||
}
|
||||
}
|
||||
+81
-13
@@ -3,6 +3,7 @@ using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.Hotkeys;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Logging;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Screenshots;
|
||||
@@ -16,7 +17,11 @@ using Microsoft.Extensions.Options;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
|
||||
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
|
||||
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
|
||||
builder.Services.AddOptions<MeetingAssistantOptions>()
|
||||
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton(HostOperatingSystem.DetectCurrent());
|
||||
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
|
||||
@@ -35,8 +40,9 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
|
||||
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||
PortableMeetingAudioRegistration.Add(builder.Services);
|
||||
MacOsMeetingIntegrationRegistration.Add(builder.Services, OperatingSystem.IsMacOS());
|
||||
builder.Services.AddHostedService<MacOsDesktopControlService>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
|
||||
@@ -66,14 +72,8 @@ builder.Services.AddSingleton<IMeetingStartPromptService>(services =>
|
||||
services.GetRequiredService<WindowsMeetingStartPromptService>());
|
||||
builder.Services.AddHostedService(services =>
|
||||
services.GetRequiredService<WindowsMeetingStartPromptService>());
|
||||
#else
|
||||
builder.Services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
|
||||
builder.Services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
|
||||
#endif
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
builder.Services.AddSingleton<IScreenshotImageCropper, WindowsScreenshotImageCropper>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
|
||||
builder.Services.AddSingleton<MeetingScreenshotService>();
|
||||
@@ -91,8 +91,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
|
||||
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
|
||||
builder.Services.AddSingleton(services =>
|
||||
{
|
||||
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer;
|
||||
return resemblyzer.Enabled
|
||||
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
|
||||
: SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
});
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
|
||||
: services.GetRequiredService<SpeakerIdentityService>());
|
||||
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
|
||||
: services.GetRequiredService<SpeakerIdentityMergeService>());
|
||||
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
|
||||
@@ -102,7 +129,11 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
|
||||
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
|
||||
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
|
||||
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
|
||||
#if WINDOWS
|
||||
@@ -110,7 +141,8 @@ builder.Services.AddSingleton(services => WorkflowRulesEditorMarkdownLinkResolve
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value));
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRulesEditorWindowService>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
|
||||
builder.Services.AddSingleton<MacOsWorkflowRulesEditorSession>();
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, MacOsWorkflowRulesEditorWindowService>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<AsrDiagnosticService>();
|
||||
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||
@@ -134,6 +166,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
|
||||
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
||||
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
||||
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
||||
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddHostedService<GlobalHotkeyService>();
|
||||
builder.Services.AddHostedService<UnoTaskbarIconService>();
|
||||
@@ -146,11 +179,32 @@ app.MapGet("/health", () => Results.Ok(new
|
||||
service = "meeting-assistant",
|
||||
status = "ok"
|
||||
}));
|
||||
app.MapGet("/diagnostics/platform-capabilities", (
|
||||
HostOperatingSystem host,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
IMeetingMetadataProvider meetingMetadataProvider,
|
||||
ICalendarMeetingProvider calendarMeetingProvider,
|
||||
IMeetingStartPromptService meetingStartPromptProvider,
|
||||
IActiveWindowScreenshotCapture screenshotProvider,
|
||||
IWorkflowRulesEditorWindowService workflowEditorProvider) =>
|
||||
Results.Ok(PlatformCapabilityReport.Create(
|
||||
host,
|
||||
options.Value,
|
||||
meetingMetadataProvider,
|
||||
calendarMeetingProvider,
|
||||
meetingStartPromptProvider,
|
||||
screenshotProvider,
|
||||
workflowEditorProvider)));
|
||||
|
||||
app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
|
||||
app.MapGet("/profiles/{launchProfile}/recording/status", (
|
||||
string launchProfile,
|
||||
MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
|
||||
app.MapPost("/application/exit", (IHostApplicationLifetime lifetime) =>
|
||||
{
|
||||
lifetime.StopApplication();
|
||||
return Results.Accepted();
|
||||
});
|
||||
app.MapGet("/diagnostics/outlook/current-meeting", async (
|
||||
IMeetingMetadataProvider metadataProvider,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
@@ -280,6 +334,15 @@ app.MapPost("/diagnostics/settings-and-logs/show", (
|
||||
rulesEditorWindow.Show();
|
||||
return Results.Accepted();
|
||||
});
|
||||
#if !WINDOWS
|
||||
app.MapGet("/diagnostics/settings-and-logs", () =>
|
||||
Results.Content(MacOsWorkflowRulesEditorPage.Html, "text/html"));
|
||||
app.MapPost("/diagnostics/settings-and-logs/chat", async (
|
||||
MacOsWorkflowRulesEditorChatRequest request,
|
||||
MacOsWorkflowRulesEditorSession session,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await session.SendAsync(request.Message, cancellationToken)));
|
||||
#endif
|
||||
|
||||
static WorkflowConfigurationReloadResponse ReloadWorkflowConfiguration(IConfiguration configuration)
|
||||
{
|
||||
@@ -322,6 +385,11 @@ app.MapPost("/profiles/{launchProfile}/recording/abort", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
|
||||
app.MapPost("/profiles/{launchProfile}/meetings/screenshot/capture", async (
|
||||
string launchProfile,
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.CaptureScreenshotAsync(launchProfile, cancellationToken)));
|
||||
app.MapPost("/meetings/current/summary/run", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
IMeetingSummaryPipeline summaryPipeline,
|
||||
|
||||
@@ -6,6 +6,8 @@ public interface IMeetingInactivityPromptService
|
||||
MeetingInactivityPromptRequest request,
|
||||
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task DismissAllAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingInactivityPromptRequest(
|
||||
@@ -18,9 +20,32 @@ public enum MeetingInactivityPromptResponse
|
||||
{
|
||||
Dismissed,
|
||||
Continue,
|
||||
Pause,
|
||||
Stop
|
||||
}
|
||||
|
||||
internal sealed record MeetingInactivityPromptAction(
|
||||
string Content,
|
||||
string ResponseArgument,
|
||||
MeetingInactivityPromptResponse Response);
|
||||
|
||||
internal static class MeetingInactivityPromptActions
|
||||
{
|
||||
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
|
||||
[
|
||||
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
|
||||
new("No", "continue", MeetingInactivityPromptResponse.Continue),
|
||||
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
|
||||
];
|
||||
|
||||
public static MeetingInactivityPromptResponse ParseResponse(string? value)
|
||||
{
|
||||
return All.FirstOrDefault(action =>
|
||||
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||
?.Response ?? MeetingInactivityPromptResponse.Continue;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
|
||||
{
|
||||
public Task ShowStopPromptAsync(
|
||||
@@ -30,4 +55,9 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
internal enum MacOsAudioCaptureKind
|
||||
{
|
||||
Microphone,
|
||||
System
|
||||
}
|
||||
|
||||
internal sealed record MacOsAudioCaptureRequest(
|
||||
MacOsAudioCaptureKind Kind,
|
||||
int SampleRate,
|
||||
int Channels);
|
||||
|
||||
internal interface IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request);
|
||||
}
|
||||
|
||||
internal interface IMacOsAudioCaptureProcess : IAsyncDisposable
|
||||
{
|
||||
Stream StandardOutput { get; }
|
||||
|
||||
int ExitCode { get; }
|
||||
|
||||
bool HasExited { get; }
|
||||
|
||||
Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task WaitForExitAsync(CancellationToken cancellationToken);
|
||||
|
||||
void Terminate();
|
||||
}
|
||||
|
||||
internal abstract class MacOsMeetingAudioSource : IMeetingAudioSource
|
||||
{
|
||||
private const int ReadBufferSize = 8 * 1024;
|
||||
private readonly MacOsAudioCaptureKind kind;
|
||||
private readonly IMacOsAudioCaptureProcessFactory processFactory;
|
||||
private readonly ILogger logger;
|
||||
|
||||
protected MacOsMeetingAudioSource(
|
||||
MacOsAudioCaptureKind kind,
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger logger)
|
||||
{
|
||||
this.kind = kind;
|
||||
this.processFactory = processFactory;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
MeetingAssistantOptions options,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new MacOsAudioCaptureRequest(
|
||||
kind,
|
||||
options.Recording.SampleRate,
|
||||
options.Recording.Channels);
|
||||
await using var process = processFactory.Start(request);
|
||||
var standardErrorTask = process.ReadStandardErrorAsync(CancellationToken.None);
|
||||
var buffer = new byte[ReadBufferSize];
|
||||
byte? pendingByte = null;
|
||||
var hasLoggedFirstChunk = false;
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting macOS {CaptureKind} audio capture at {SampleRate} Hz/{Channels} channel(s)",
|
||||
kind,
|
||||
request.SampleRate,
|
||||
request.Channels);
|
||||
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var bytesRead = await process.StandardOutput.ReadAsync(buffer, cancellationToken);
|
||||
if (bytesRead == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var pcm = CopyCompleteSamples(buffer, bytesRead, ref pendingByte);
|
||||
if (pcm.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hasLoggedFirstChunk)
|
||||
{
|
||||
hasLoggedFirstChunk = true;
|
||||
logger.LogInformation(
|
||||
"macOS {CaptureKind} audio produced its first PCM chunk: {ByteCount} bytes",
|
||||
kind,
|
||||
pcm.Length);
|
||||
}
|
||||
|
||||
yield return new AudioChunk(pcm, request.SampleRate, request.Channels);
|
||||
}
|
||||
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
var standardError = await standardErrorTask;
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
throw CreateCaptureException(kind, process.ExitCode, standardError);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Terminate();
|
||||
await process.WaitForExitAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] CopyCompleteSamples(byte[] buffer, int bytesRead, ref byte? pendingByte)
|
||||
{
|
||||
var totalBytes = bytesRead + (pendingByte.HasValue ? 1 : 0);
|
||||
var completeBytes = totalBytes - totalBytes % sizeof(short);
|
||||
var pcm = new byte[completeBytes];
|
||||
var targetOffset = 0;
|
||||
|
||||
if (pendingByte is { } firstByte)
|
||||
{
|
||||
pcm[0] = firstByte;
|
||||
targetOffset = 1;
|
||||
pendingByte = null;
|
||||
}
|
||||
|
||||
var copyCount = completeBytes - targetOffset;
|
||||
if (copyCount > 0)
|
||||
{
|
||||
Buffer.BlockCopy(buffer, 0, pcm, targetOffset, copyCount);
|
||||
}
|
||||
|
||||
var consumedFromBuffer = copyCount;
|
||||
if (consumedFromBuffer < bytesRead)
|
||||
{
|
||||
pendingByte = buffer[consumedFromBuffer];
|
||||
}
|
||||
|
||||
return pcm;
|
||||
}
|
||||
|
||||
private static InvalidOperationException CreateCaptureException(
|
||||
MacOsAudioCaptureKind kind,
|
||||
int exitCode,
|
||||
string standardError)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(standardError)
|
||||
? "The native capture helper did not provide an error message."
|
||||
: standardError.Trim();
|
||||
return new InvalidOperationException(
|
||||
$"macOS {kind.ToString().ToLowerInvariant()} audio capture exited with code {exitCode}. {detail}");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsMicrophoneAudioSource : MacOsMeetingAudioSource
|
||||
{
|
||||
public MacOsMicrophoneAudioSource(
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger<MacOsMicrophoneAudioSource> logger)
|
||||
: base(MacOsAudioCaptureKind.Microphone, processFactory, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsSystemAudioSource : MacOsMeetingAudioSource
|
||||
{
|
||||
public MacOsSystemAudioSource(
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger<MacOsSystemAudioSource> logger)
|
||||
: base(MacOsAudioCaptureKind.System, processFactory, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly ILogger<MacOsAudioCaptureProcessFactory> logger;
|
||||
|
||||
public MacOsAudioCaptureProcessFactory(ILogger<MacOsAudioCaptureProcessFactory> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
throw new PlatformNotSupportedException("The macOS audio capture helper can only run on macOS.");
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-meeting-audio-capture");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS audio capture helper was not found at '{helperPath}'. Build or publish the application on macOS so the Swift helper is compiled into the Native folder.",
|
||||
helperPath);
|
||||
}
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = helperPath,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
startInfo.ArgumentList.Add(request.Kind == MacOsAudioCaptureKind.Microphone ? "microphone" : "system");
|
||||
startInfo.ArgumentList.Add(request.SampleRate.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
startInfo.ArgumentList.Add(request.Channels.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
|
||||
logger.LogDebug("Starting native macOS audio helper {HelperPath} for {CaptureKind}", helperPath, request.Kind);
|
||||
var process = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException($"Failed to start the macOS audio capture helper '{helperPath}'.");
|
||||
return new MacOsAudioCaptureProcess(process);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed class MacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly Process process;
|
||||
|
||||
public MacOsAudioCaptureProcess(Process process)
|
||||
{
|
||||
this.process = process;
|
||||
}
|
||||
|
||||
public Stream StandardOutput => process.StandardOutput.BaseStream;
|
||||
|
||||
public int ExitCode => process.ExitCode;
|
||||
|
||||
public bool HasExited => process.HasExited;
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return process.WaitForExitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
process.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly IMeetingInactivityClock inactivityClock;
|
||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
|
||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
private RecordingRun? currentRun;
|
||||
@@ -63,7 +64,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
||||
IMeetingInactivityPromptService? inactivityPromptService = null,
|
||||
IMeetingInactivityClock? inactivityClock = null,
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
|
||||
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
|
||||
{
|
||||
this.audioSource = audioSource;
|
||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||
@@ -86,6 +88,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||
this.options = options.Value;
|
||||
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
|
||||
?? SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -96,7 +100,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
currentArtifacts?.AssistantContextPath,
|
||||
currentArtifacts?.SummaryPath,
|
||||
GetProcessState(currentRun),
|
||||
currentRun?.LaunchProfileName);
|
||||
currentRun?.LaunchProfileName,
|
||||
IsRecording && currentRun?.IsTranscriptionPaused == true);
|
||||
|
||||
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
|
||||
|
||||
@@ -289,10 +294,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
startedAt,
|
||||
launchProfile.Name,
|
||||
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
|
||||
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
|
||||
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
|
||||
logger);
|
||||
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
|
||||
if (runOptions.Recording.InactivitySafeguard.Enabled)
|
||||
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
|
||||
{
|
||||
_ = Task.Run(
|
||||
() => RunInactivitySafeguardAsync(run),
|
||||
@@ -334,11 +341,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return await StopAsync(null, cancellationToken);
|
||||
return await StopAsync(null, null, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> StopAsync(
|
||||
DateTimeOffset? inferredEndTime,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
DateTimeOffset? expectedPauseStartedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -347,13 +357,30 @@ public sealed class MeetingRecordingCoordinator
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (currentRun is null || currentRun.IsCaptureStopping)
|
||||
if (currentRun is null ||
|
||||
currentRun.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
run = currentRun;
|
||||
if (inferredEndTime is not null)
|
||||
var inactivityStopClaimed = expectedActivityVersion is not null;
|
||||
if (expectedActivityVersion is not null &&
|
||||
(inferredEndTime is null ||
|
||||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
|
||||
if (expectedPauseStartedAt is not null &&
|
||||
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
|
||||
{
|
||||
run.SetInferredEndTime(inferredEndTime.Value);
|
||||
}
|
||||
@@ -363,7 +390,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
{
|
||||
run.Abort();
|
||||
}
|
||||
else
|
||||
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
|
||||
{
|
||||
run.StopCapture();
|
||||
}
|
||||
@@ -409,6 +436,65 @@ public sealed class MeetingRecordingCoordinator
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var run = currentRun;
|
||||
if (run is null ||
|
||||
run.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var changed = false;
|
||||
await run.PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var changedAt = inactivityClock.Now;
|
||||
changed = run.TrySetTranscriptionPaused(
|
||||
isPaused,
|
||||
changedAt,
|
||||
expectedActivityVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
run.PipelineGate.Release();
|
||||
}
|
||||
|
||||
if (changed && isPaused)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (changed)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Meeting transcription {PauseState}",
|
||||
isPaused ? "paused" : "unpaused");
|
||||
}
|
||||
|
||||
return CurrentStatus;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -624,6 +710,24 @@ public sealed class MeetingRecordingCoordinator
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Meeting recording failed");
|
||||
if (IsAzureSpeechRun(run) && !run.IsQueuedForOfflineTranscription)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Azure Speech transcription failed; retaining recorded audio and queueing offline transcription backlog item");
|
||||
run.MarkQueuedForOfflineTranscription();
|
||||
try
|
||||
{
|
||||
await offlineTranscriptionBacklog.EnqueueAsync(
|
||||
CreateOfflineBacklogItem(run),
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception backlogException)
|
||||
{
|
||||
logger.LogError(
|
||||
backlogException,
|
||||
"Could not queue offline transcription backlog item; retaining recorded audio for manual recovery");
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -661,8 +765,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
chunk.Channels);
|
||||
}
|
||||
|
||||
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
|
||||
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
|
||||
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -701,7 +804,6 @@ public sealed class MeetingRecordingCoordinator
|
||||
run.ResetSpeakerIdentification();
|
||||
}
|
||||
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
var sample = run.TryAddSpeakerSample(segment);
|
||||
if (sample is not null)
|
||||
{
|
||||
@@ -718,6 +820,56 @@ public sealed class MeetingRecordingCoordinator
|
||||
var relabeledSegment = run.Relabel(segment);
|
||||
run.AddLiveSegment(relabeledSegment);
|
||||
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
if (!string.IsNullOrWhiteSpace(segment.Text))
|
||||
{
|
||||
// Switching profiles holds the coordinator gate while draining this reader.
|
||||
// Notification cleanup must not make transcript draining wait for that gate.
|
||||
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsForRunAsync(
|
||||
RecordingRun run,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Pending cleanup is no longer needed after this run stops capturing.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await inactivityPromptService.DismissAllAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -845,7 +997,10 @@ public sealed class MeetingRecordingCoordinator
|
||||
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
||||
{
|
||||
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
|
||||
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
|
||||
var promptThresholds = safeguardOptions.Enabled
|
||||
? GetInactivityPromptThresholds(safeguardOptions)
|
||||
: [];
|
||||
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
|
||||
var promptedThresholds = new HashSet<TimeSpan>();
|
||||
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
|
||||
|
||||
@@ -853,7 +1008,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
|
||||
string.Join(", ", promptThresholds),
|
||||
safeguardOptions.AutoStopAfter,
|
||||
safeguardOptions.CheckInterval);
|
||||
checkInterval);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -869,9 +1024,38 @@ public sealed class MeetingRecordingCoordinator
|
||||
snapshot.LastTranscriptActivityAt);
|
||||
}
|
||||
|
||||
var pauseStartedAt = snapshot.TranscriptionPausedAt;
|
||||
if (pauseStartedAt is not null)
|
||||
{
|
||||
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
|
||||
if (maximumPauseDuration > TimeSpan.Zero &&
|
||||
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
|
||||
maximumPauseDuration);
|
||||
var stopStatus = await StopAsync(
|
||||
null,
|
||||
run,
|
||||
null,
|
||||
pauseStartedAt,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = inactivityClock.Now;
|
||||
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
|
||||
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
if (safeguardOptions.Enabled &&
|
||||
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
inactivityDuration >= safeguardOptions.AutoStopAfter)
|
||||
{
|
||||
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
|
||||
@@ -879,8 +1063,16 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
|
||||
inactivityDuration,
|
||||
inferredEndTime);
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
return;
|
||||
var stopStatus = await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
snapshot.ActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var threshold in promptThresholds)
|
||||
@@ -897,6 +1089,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
inactivityDuration,
|
||||
threshold,
|
||||
inferredEndTime);
|
||||
var promptActivityVersion = snapshot.ActivityVersion;
|
||||
await inactivityPromptService.ShowStopPromptAsync(
|
||||
new MeetingInactivityPromptRequest(
|
||||
inactivityDuration,
|
||||
@@ -912,16 +1105,36 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
if (response == MeetingInactivityPromptResponse.Stop)
|
||||
{
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
}
|
||||
else if (response == MeetingInactivityPromptResponse.Pause)
|
||||
{
|
||||
await SetTranscriptionPausedAsync(
|
||||
true,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
CancellationToken.None);
|
||||
}
|
||||
},
|
||||
run.CaptureCancellation);
|
||||
|
||||
var currentActivity = run.GetTranscriptActivitySnapshot();
|
||||
if (currentActivity.IsTranscriptionPaused ||
|
||||
currentActivity.ActivityVersion != promptActivityVersion)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(run.CaptureCancellation);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
GetInactivitySafeguardCheckInterval(safeguardOptions),
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
@@ -989,11 +1202,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
try
|
||||
{
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
|
||||
if (checkpoint is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
|
||||
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1821,6 +2034,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
|
||||
}
|
||||
|
||||
private static TimeSpan GetInactivitySafeguardCheckInterval(
|
||||
RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
@@ -1877,6 +2095,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
private TimeSpan? lastTranscriptSegmentEnd;
|
||||
private long transcriptActivityVersion;
|
||||
private DateTimeOffset? inferredEndTime;
|
||||
private bool isTranscriptionPaused;
|
||||
private DateTimeOffset? transcriptionPausedAt;
|
||||
|
||||
public RecordingRun(
|
||||
CancellationTokenSource captureCancellation,
|
||||
@@ -1892,6 +2112,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
string launchProfileName,
|
||||
TimeSpan liveSampleBufferDuration,
|
||||
int maxSpeakerSamples,
|
||||
bool requireNonOverlappingSpeakerSamples,
|
||||
ILogger logger)
|
||||
{
|
||||
CaptureCancellationSource = captureCancellation;
|
||||
@@ -1912,8 +2133,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
liveSampleBufferDuration,
|
||||
maxSpeakerSamples,
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleSegmentGap,
|
||||
logger);
|
||||
options.SpeakerIdentification.MaximumSampleDuration,
|
||||
logger,
|
||||
requireNonOverlappingSpeakerSamples);
|
||||
}
|
||||
|
||||
public CancellationTokenSource CaptureCancellationSource { get; }
|
||||
@@ -1960,6 +2182,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public bool HasAttachedPromptMetadata { get; private set; }
|
||||
|
||||
public bool IsTranscriptionPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
return isTranscriptionPaused;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||
|
||||
public DateTimeOffset? InferredEndTime
|
||||
@@ -1983,6 +2216,37 @@ public sealed class MeetingRecordingCoordinator
|
||||
HasAttachedPromptMetadata = true;
|
||||
}
|
||||
|
||||
public bool TrySetTranscriptionPaused(
|
||||
bool isPaused,
|
||||
DateTimeOffset changedAt,
|
||||
long? expectedActivityVersion)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (expectedActivityVersion is not null &&
|
||||
transcriptActivityVersion != expectedActivityVersion.Value)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isTranscriptionPaused == isPaused)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
isTranscriptionPaused = isPaused;
|
||||
transcriptionPausedAt = isPaused ? changedAt : null;
|
||||
|
||||
if (!isPaused)
|
||||
{
|
||||
lastTranscriptActivityAt = changedAt;
|
||||
}
|
||||
|
||||
transcriptActivityVersion++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Abort()
|
||||
{
|
||||
IsAborted = true;
|
||||
@@ -2011,13 +2275,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
await PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
AppendAudio(chunk);
|
||||
await Pipeline.WriteAsync(chunk, cancellationToken);
|
||||
var routedChunk = IsTranscriptionPaused
|
||||
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
|
||||
: chunk;
|
||||
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
|
||||
AppendAudio(routedChunk);
|
||||
await Pipeline.WriteAsync(routedChunk, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -2201,7 +2469,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
return new TranscriptActivitySnapshot(
|
||||
lastTranscriptActivityAt,
|
||||
lastTranscriptSegmentEnd,
|
||||
transcriptActivityVersion);
|
||||
transcriptActivityVersion,
|
||||
isTranscriptionPaused,
|
||||
transcriptionPausedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2221,6 +2491,35 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
inferredEndTime = endTime;
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void AppendAudio(AudioChunk chunk)
|
||||
{
|
||||
speakerSampleCollector.AppendAudio(chunk);
|
||||
@@ -2244,9 +2543,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
return speakerSampleCollector.Snapshot();
|
||||
}
|
||||
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
var speakers = GetUnmappedSampleSpeakers();
|
||||
var speakers = GetUnmappedSampleSpeakers(samples);
|
||||
if (speakers.Count == 0)
|
||||
{
|
||||
return null;
|
||||
@@ -2254,7 +2555,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
var checkpoint = new LiveIdentificationCheckpoint(
|
||||
speakers,
|
||||
BuildAttendeeSignature(meetingNote));
|
||||
BuildAttendeeSignature(meetingNote),
|
||||
Options.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? speakers.Select(speaker => samples.Count(sample =>
|
||||
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
|
||||
: []);
|
||||
lock (liveIdentificationGate)
|
||||
{
|
||||
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
|
||||
@@ -2312,9 +2617,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
return GetSpeakerSamplesSnapshot()
|
||||
return samples
|
||||
.Select(sample => sample.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -2350,7 +2655,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
public sealed record TranscriptActivitySnapshot(
|
||||
DateTimeOffset LastTranscriptActivityAt,
|
||||
TimeSpan? LastTranscriptSegmentEnd,
|
||||
long ActivityVersion);
|
||||
long ActivityVersion,
|
||||
bool IsTranscriptionPaused,
|
||||
DateTimeOffset? TranscriptionPausedAt);
|
||||
|
||||
private static int StateRank(AssistantContextState state)
|
||||
{
|
||||
@@ -2368,11 +2675,13 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public sealed record LiveIdentificationCheckpoint(
|
||||
IReadOnlyList<string> Speakers,
|
||||
string AttendeeSignature)
|
||||
string AttendeeSignature,
|
||||
IReadOnlyList<int> SampleCounts)
|
||||
{
|
||||
public bool Matches(LiveIdentificationCheckpoint other)
|
||||
{
|
||||
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
|
||||
SampleCounts.SequenceEqual(other.SampleCounts) &&
|
||||
Speakers.Count == other.Speakers.Count &&
|
||||
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -2405,7 +2714,8 @@ public sealed record RecordingStatus(
|
||||
string? AssistantContextPath,
|
||||
string? SummaryPath,
|
||||
RecordingProcessState State = RecordingProcessState.Idle,
|
||||
string? LaunchProfile = null);
|
||||
string? LaunchProfile = null,
|
||||
bool IsPaused = false);
|
||||
|
||||
public enum RecordingProcessState
|
||||
{
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
internal static class PortableMeetingAudioRegistration
|
||||
{
|
||||
public static void Add(IServiceCollection services)
|
||||
{
|
||||
Add(services, OperatingSystem.IsMacOS());
|
||||
}
|
||||
|
||||
internal static void Add(IServiceCollection services, bool isMacOs)
|
||||
{
|
||||
if (!isMacOs)
|
||||
{
|
||||
services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||
return;
|
||||
}
|
||||
|
||||
services.AddSingleton<IMacOsAudioCaptureProcessFactory, MacOsAudioCaptureProcessFactory>();
|
||||
services.AddSingleton<MacOsMicrophoneAudioSource>();
|
||||
services.AddSingleton<MacOsSystemAudioSource>();
|
||||
services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||
services.AddSingleton<IMeetingAudioSource>(serviceProvider => new CompositeMeetingAudioSource(
|
||||
serviceProvider.GetRequiredService<MacOsMicrophoneAudioSource>(),
|
||||
serviceProvider.GetRequiredService<MacOsSystemAudioSource>(),
|
||||
serviceProvider.GetRequiredService<IAcousticEchoCancellerFactory>(),
|
||||
serviceProvider.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||
}
|
||||
}
|
||||
@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
private readonly object gate = new();
|
||||
private readonly RollingAudioBuffer audioBuffer;
|
||||
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly int maxSamplesPerSpeaker;
|
||||
private readonly TimeSpan minimumUninterruptedSpeechDuration;
|
||||
private readonly TimeSpan maximumSegmentGap;
|
||||
private readonly TimeSpan minimumSampleSpeechDuration;
|
||||
private readonly TimeSpan maximumSampleDuration;
|
||||
private readonly bool requireNonOverlappingSamples;
|
||||
private readonly ILogger? logger;
|
||||
private PendingSpeakerSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
|
||||
: this(
|
||||
bufferDuration,
|
||||
maxSamplesPerSpeaker,
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(1),
|
||||
logger: null)
|
||||
{
|
||||
}
|
||||
private SpeakerSampleSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(
|
||||
TimeSpan bufferDuration,
|
||||
int maxSamplesPerSpeaker,
|
||||
TimeSpan minimumUninterruptedSpeechDuration,
|
||||
TimeSpan maximumSegmentGap,
|
||||
ILogger? logger = null)
|
||||
TimeSpan minimumSampleSpeechDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
ILogger? logger = null,
|
||||
bool requireNonOverlappingSamples = false)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
minimumSampleSpeechDuration,
|
||||
maximumSampleDuration,
|
||||
"Speaker sample collector configuration is invalid");
|
||||
audioBuffer = new RollingAudioBuffer(bufferDuration);
|
||||
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
|
||||
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
|
||||
? minimumUninterruptedSpeechDuration
|
||||
: TimeSpan.Zero;
|
||||
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
|
||||
? maximumSegmentGap
|
||||
: TimeSpan.Zero;
|
||||
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
|
||||
this.maximumSampleDuration = maximumSampleDuration;
|
||||
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
lock (gate)
|
||||
{
|
||||
samplesBySpeaker.Clear();
|
||||
lastAcceptedEndBySpeaker.Clear();
|
||||
pendingSpan = null;
|
||||
audioBuffer.Reset();
|
||||
}
|
||||
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return null;
|
||||
}
|
||||
|
||||
TranscriptionSegment sampleSegment;
|
||||
PendingSpanReset? reset;
|
||||
SpeakerSampleSpan? sampleSpan;
|
||||
SpeakerSampleSpan? previousSpan;
|
||||
lock (gate)
|
||||
{
|
||||
(sampleSegment, reset) = ExtendPendingSpan(segment);
|
||||
if (requireNonOverlappingSamples &&
|
||||
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
|
||||
segment.Start < lastAcceptedEnd)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
|
||||
segment.Speaker,
|
||||
lastAcceptedEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
|
||||
}
|
||||
|
||||
if (reset is not null)
|
||||
if (sampleSpan is null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
|
||||
reset.PreviousSpeaker,
|
||||
segment.Speaker,
|
||||
reset.PreviousEnd,
|
||||
segment.Start,
|
||||
reset.Gap,
|
||||
maximumSegmentGap);
|
||||
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
|
||||
segment.Speaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
|
||||
var sampleSegment = sampleSpan.ToSegment();
|
||||
|
||||
if (previousSpan is not null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
|
||||
previousSpan.Speaker,
|
||||
segment.Speaker,
|
||||
previousSpan.End,
|
||||
segment.Start);
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
|
||||
if (!score.Accepted)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
sampleSegment.Speaker,
|
||||
score.Reason,
|
||||
sampleSpan.SpeechDuration,
|
||||
sampleSegment.End - sampleSegment.Start,
|
||||
minimumUninterruptedSpeechDuration,
|
||||
minimumSampleSpeechDuration,
|
||||
score.WordCount);
|
||||
return null;
|
||||
}
|
||||
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
|
||||
lock (gate)
|
||||
{
|
||||
if (requireNonOverlappingSamples &&
|
||||
ReferenceEquals(pendingSpan, sampleSpan))
|
||||
{
|
||||
pendingSpan = null;
|
||||
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
|
||||
}
|
||||
|
||||
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
|
||||
{
|
||||
samples = [];
|
||||
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
{
|
||||
if (pendingSpan is null ||
|
||||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
|
||||
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
|
||||
{
|
||||
var reset = pendingSpan is null
|
||||
? null
|
||||
: new PendingSpanReset(
|
||||
pendingSpan.Speaker,
|
||||
pendingSpan.End,
|
||||
segment.Start - pendingSpan.End);
|
||||
pendingSpan = new PendingSpeakerSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
segment.End,
|
||||
[segment.Text]);
|
||||
return (pendingSpan.ToSegment(), reset);
|
||||
pendingSpan = extended;
|
||||
return (pendingSpan, null);
|
||||
}
|
||||
|
||||
pendingSpan = pendingSpan.Extend(segment);
|
||||
return (pendingSpan.ToSegment(), null);
|
||||
var previousSpan = pendingSpan;
|
||||
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
return (pendingSpan, previousSpan);
|
||||
}
|
||||
|
||||
private static SampleScore Score(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan minimumUninterruptedSpeechDuration)
|
||||
TimeSpan speechDuration,
|
||||
TimeSpan minimumSampleSpeechDuration)
|
||||
{
|
||||
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
|
||||
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
|
||||
var durationSeconds = speechDuration.TotalSeconds;
|
||||
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
|
||||
{
|
||||
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
|
||||
}
|
||||
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
|
||||
}
|
||||
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
|
||||
var wordScore = Math.Min(words / 60.0, 1);
|
||||
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
|
||||
segment.Text.TrimEnd().EndsWith('?') ||
|
||||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
|
||||
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
|
||||
|
||||
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
|
||||
|
||||
private sealed record PendingSpeakerSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
IReadOnlyList<string> TextParts)
|
||||
{
|
||||
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
|
||||
{
|
||||
return this with
|
||||
{
|
||||
End = segment.End > End ? segment.End : End,
|
||||
TextParts = TextParts.Append(segment.Text).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,6 @@ public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
throw new PlatformNotSupportedException(
|
||||
"Meeting audio capture is only implemented for the Windows target in this version.");
|
||||
"Meeting audio capture is supported on Windows and macOS, but is unavailable on this host.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
private const string NotificationGroup = "meeting-inactivity";
|
||||
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
|
||||
private readonly object registrationGate = new();
|
||||
private readonly object notificationGate = new();
|
||||
private bool registered;
|
||||
|
||||
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
|
||||
@@ -37,24 +37,27 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
try
|
||||
{
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
lock (notificationGate)
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -76,6 +79,36 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (pendingPrompts.IsEmpty)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
pendingPrompts.Clear();
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
|
||||
logger.LogInformation("Dismissed all native Windows inactivity notifications");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
@@ -83,29 +116,28 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (!registered)
|
||||
lock (notificationGate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (!registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
registered = false;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void EnsureRegistered()
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (registrationGate)
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
@@ -122,29 +154,25 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
string promptId,
|
||||
MeetingInactivityPromptRequest request)
|
||||
{
|
||||
var yesButton = new ToastButton()
|
||||
.SetContent("Yes")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "stop")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
var noButton = new ToastButton()
|
||||
.SetContent("No")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "continue")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
return new ToastContentBuilder()
|
||||
var builder = new ToastContentBuilder()
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText("Stop meeting?")
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
|
||||
.AddButton(yesButton)
|
||||
.AddButton(noButton);
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
|
||||
|
||||
foreach (var action in MeetingInactivityPromptActions.All)
|
||||
{
|
||||
builder.AddButton(new ToastButton()
|
||||
.SetContent(action.Content)
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", action.ResponseArgument)
|
||||
.SetBackgroundActivation());
|
||||
}
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||
@@ -177,10 +205,9 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return;
|
||||
}
|
||||
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
||||
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingInactivityPromptResponse.Stop
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||
? MeetingInactivityPromptActions.ParseResponse(responseValue)
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
@@ -46,6 +43,25 @@ public sealed record ScreenshotOcrResult
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
public interface IScreenshotImageCropper
|
||||
{
|
||||
Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class UnavailableScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
public interface IMeetingScreenshotService
|
||||
{
|
||||
Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
@@ -147,6 +163,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
IMeetingNoteImageOcrService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IScreenshotImageCropper screenshotImageCropper;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IMeetingNoteStore meetingNoteStore;
|
||||
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
|
||||
@@ -162,6 +179,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
IMeetingNoteStore meetingNoteStore,
|
||||
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
IScreenshotImageCropper screenshotImageCropper,
|
||||
ILogger<MeetingScreenshotService> logger,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
{
|
||||
@@ -171,6 +189,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
this.attendeeCanonicalizer = attendeeCanonicalizer;
|
||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||
this.ocrClient = ocrClient;
|
||||
this.screenshotImageCropper = screenshotImageCropper;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -639,17 +658,20 @@ public sealed partial class MeetingScreenshotService :
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
|
||||
var croppedPath = await screenshotImageCropper.SaveCroppedScreenshotAsync(
|
||||
screenshotPath,
|
||||
crop,
|
||||
cancellationToken);
|
||||
if (croppedPath is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because no image cropper is available on this host",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
croppedPath));
|
||||
@@ -657,7 +679,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
Environment.NewLine +
|
||||
Environment.NewLine;
|
||||
}
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
@@ -667,39 +689,6 @@ public sealed partial class MeetingScreenshotService :
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CA1416
|
||||
private static Task<string> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult(croppedPath);
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackOcr(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationTokenSource cancellation,
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#if WINDOWS
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
internal sealed class WindowsScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
#pragma warning disable CA1416
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult<string?>(croppedPath);
|
||||
}
|
||||
catch (ExternalException exception)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
|
||||
exception);
|
||||
}
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(
|
||||
ScreenshotCropCoordinates crop,
|
||||
int imageWidth,
|
||||
int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
return [];
|
||||
}
|
||||
|
||||
return await finalizer.FinalizeAsync(
|
||||
return await finalizer.FinalizeEnabledAsync(
|
||||
wavPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
||||
options.Diarization,
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
ResemblyzerVoiceClusterMatcher clusterMatcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentificationService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.encoder = encoder;
|
||||
this.clusterMatcher = clusterMatcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ApplySpeakerOverrideAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string sourceSpeaker,
|
||||
string targetSpeaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled ||
|
||||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
|
||||
string.IsNullOrWhiteSpace(targetSpeaker) ||
|
||||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = sourceSpeaker.Trim();
|
||||
var targetName = targetSpeaker.Trim();
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var sourceCandidate = identities
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
|
||||
if (target is null && sourceCandidate is null && vectors.Count == 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
|
||||
sourceLabel,
|
||||
targetName);
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (target is null)
|
||||
{
|
||||
target = sourceCandidate ?? new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now
|
||||
};
|
||||
if (target.Id == 0)
|
||||
{
|
||||
context.SpeakerIdentities.Add(target);
|
||||
}
|
||||
}
|
||||
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||
{
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
sourceCandidate,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||
}
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
|
||||
outlierPruner.AddAndPrune(
|
||||
target,
|
||||
vectors,
|
||||
now);
|
||||
target.UpdatedAt = now;
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
target.References,
|
||||
sourceLabel,
|
||||
targetName,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteSpeakerIdentityAsync(
|
||||
string identity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities.FirstOrDefault(candidate =>
|
||||
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
|
||||
if (target is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
context.SpeakerIdentities.Remove(target);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
bool final,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return EmptyResult(request.Segments);
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var knownMappings = request.KnownSpeakerMappings ??
|
||||
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
if (final && knownMappings.Count > 0)
|
||||
{
|
||||
await PersistMappedSpeakerEvidenceAsync(
|
||||
context,
|
||||
request,
|
||||
knownMappings,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
var identifiedNames = knownMappings.Values
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var segmentSpeaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
identifiedNames.Add(segmentSpeaker.Trim());
|
||||
}
|
||||
|
||||
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var pendingAudits = new List<PendingIdentificationAudit>();
|
||||
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAutomaticVectorsAsync(
|
||||
request,
|
||||
speaker,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
|
||||
speaker,
|
||||
vectors.Count,
|
||||
resemblyzerOptions.RequiredVectorsPerSpeaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (identity, decision) = await FindMatchAsync(
|
||||
context,
|
||||
attendees,
|
||||
identifiedNames,
|
||||
vectors,
|
||||
cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, vectors));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousCanonicalName = identity.CanonicalName;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees);
|
||||
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var displayName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(displayName))
|
||||
{
|
||||
mappings[speaker] = displayName;
|
||||
identifiedNames.Add(displayName);
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
identifiedNames.Add(acceptedName);
|
||||
}
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
displayName,
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
LearnUnmatchedSpeakers(
|
||||
context,
|
||||
request.MeetingNote,
|
||||
attendees,
|
||||
matchedAcceptedNames,
|
||||
unmatchedSpeakers,
|
||||
pendingAudits);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await AppendAuditsAsync(pendingAudits, cancellationToken);
|
||||
var relabeled = request.Segments
|
||||
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
|
||||
? segment with { Speaker = name }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> identifiedNames,
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var identities = await LoadIdentities(context)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var candidates = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
|
||||
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
|
||||
candidate.Identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
candidate.Identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
var match = clusterMatcher.Match(queryVectors, candidates);
|
||||
return (
|
||||
match.IdentityId is { } identityId
|
||||
? identities.Single(identity => identity.Id == identityId)
|
||||
: null,
|
||||
match);
|
||||
}
|
||||
|
||||
private void LearnUnmatchedSpeakers(
|
||||
SpeakerIdentityDbContext context,
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> matchedAcceptedNames,
|
||||
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
|
||||
ICollection<PendingIdentificationAudit> pendingAudits)
|
||||
{
|
||||
var candidates = attendees
|
||||
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (speaker, vectors) in unmatchedSpeakers)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates
|
||||
.Select(name => new SpeakerCandidateName { Name = name })
|
||||
.ToList(),
|
||||
References = [CreateReference(meetingNote, now)]
|
||||
};
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
logger.LogInformation(
|
||||
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
|
||||
speaker,
|
||||
identity.VoiceVectors.Count,
|
||||
string.Join(", ", candidates));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AppendAuditsAsync(
|
||||
IEnumerable<PendingIdentificationAudit> pendingAudits,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
audit.Identity.References,
|
||||
audit.Speaker,
|
||||
audit.Name,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (samples.Count < requiredCount)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task PersistMappedSpeakerEvidenceAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
SpeakerIdentificationRequest request,
|
||||
IReadOnlyDictionary<string, string> knownMappings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
foreach (var (speaker, mappedName) in knownMappings)
|
||||
{
|
||||
var identity = identities
|
||||
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
|
||||
.OrderBy(candidate =>
|
||||
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(candidate => candidate.Id)
|
||||
.FirstOrDefault();
|
||||
if (identity is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
|
||||
speaker,
|
||||
mappedName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
|
||||
if (vectors.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
var vectorUpdate = outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
|
||||
vectorUpdate.AddedCount,
|
||||
speaker,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback: true,
|
||||
cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
int maxSamples,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var suppliedSamples = request.Samples?
|
||||
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(sample => sample.WavBytes.Length > 0)
|
||||
.OrderByDescending(sample => sample.Score)
|
||||
.Take(maxSamples)
|
||||
.ToList()
|
||||
?? [];
|
||||
var wavSamples = suppliedSamples
|
||||
.Select(sample => sample.WavBytes)
|
||||
.ToList();
|
||||
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
|
||||
{
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
request.Segments.Count);
|
||||
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
|
||||
{
|
||||
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
span,
|
||||
cancellationToken);
|
||||
if (wavBytes.Length > 0)
|
||||
{
|
||||
wavSamples.Add(wavBytes);
|
||||
}
|
||||
|
||||
if (wavSamples.Count >= maxSamples)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
|
||||
wavSamples.Count,
|
||||
maxSamples,
|
||||
speaker,
|
||||
suppliedSamples.Count,
|
||||
wavSamples.Count - suppliedSamples.Count);
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
private static bool OverlapsSuppliedSample(
|
||||
IReadOnlyList<TranscriptionSegment> span,
|
||||
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
|
||||
{
|
||||
if (span.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var start = span[0].Start;
|
||||
var end = span[^1].End;
|
||||
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
|
||||
}
|
||||
|
||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
|
||||
}
|
||||
|
||||
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
private sealed record PendingIdentificationAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string Speaker,
|
||||
string Name);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ResemblyzerVoiceClusterMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ResemblyzerVoiceClusterMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||
TimeSpan? recentIdentityAge,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var recentIds = identities
|
||||
.Where(identity => identity.CreatedAt >= cutoff)
|
||||
.Select(identity => identity.Id)
|
||||
.ToList();
|
||||
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var attempts = 0;
|
||||
var mergedPairs = 0;
|
||||
var pendingAudits = new List<PendingMergeAudit>();
|
||||
|
||||
foreach (var sourceId in recentIds)
|
||||
{
|
||||
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
|
||||
source,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)
|
||||
.Take(required * 2)
|
||||
.ToList();
|
||||
if (sourceVectors.Count < required * 2)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
|
||||
source.Id,
|
||||
sourceVectors.Count,
|
||||
required * 2);
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidates = identities
|
||||
.Where(identity => identity.Id != source.Id)
|
||||
.Where(identity => identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(identity => new ResemblyzerVoiceVectorCandidate(
|
||||
identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
|
||||
if (!first.Accepted || first.IdentityId is not { } targetId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
|
||||
if (!second.Accepted || second.IdentityId != targetId)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
|
||||
source.Id,
|
||||
first.IdentityId,
|
||||
second.IdentityId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
|
||||
if (target is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
pendingAudits.Add(new PendingMergeAudit(
|
||||
target,
|
||||
targetName,
|
||||
sourceName));
|
||||
context.SpeakerIdentities.Remove(source);
|
||||
identities.Remove(source);
|
||||
mergedPairs++;
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
|
||||
audit.Identity.References,
|
||||
audit.TargetName,
|
||||
audit.SourceName,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return new SpeakerIdentityMergeResult(
|
||||
recentIds.Count,
|
||||
identities.Count,
|
||||
attempts,
|
||||
mergedPairs);
|
||||
}
|
||||
|
||||
private sealed record PendingMergeAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string TargetName,
|
||||
string SourceName);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorCandidate(
|
||||
int IdentityId,
|
||||
IReadOnlyList<float[]> Vectors);
|
||||
|
||||
public sealed record ResemblyzerVoiceClusterMatchResult(
|
||||
int? IdentityId,
|
||||
string Reason,
|
||||
double Cohesion,
|
||||
double? BestSimilarity,
|
||||
double? RunnerUpSimilarity)
|
||||
{
|
||||
public bool Accepted => IdentityId.HasValue;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcher
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
|
||||
|
||||
public ResemblyzerVoiceClusterMatcher(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceClusterMatcher> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceClusterMatchResult Match(
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
|
||||
{
|
||||
var requiredCount = options.RequiredVectorsPerSpeaker;
|
||||
if (queryVectors.Count < requiredCount)
|
||||
{
|
||||
return Reject(
|
||||
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
|
||||
cohesion: 0,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var normalizedQuery = queryVectors
|
||||
.Take(requiredCount)
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var cohesion = MeanPairwiseCosine(normalizedQuery);
|
||||
if (cohesion < options.MinimumClusterCohesion)
|
||||
{
|
||||
return Reject(
|
||||
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
|
||||
cohesion,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var scored = candidates
|
||||
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
|
||||
.Where(candidate => candidate is not null)
|
||||
.Select(candidate => candidate!.Value)
|
||||
.OrderByDescending(candidate => candidate.Similarity)
|
||||
.ThenBy(candidate => candidate.IdentityId)
|
||||
.ToList();
|
||||
if (scored.Count == 0)
|
||||
{
|
||||
return Reject("no compatible identity vectors were available", cohesion, null, null);
|
||||
}
|
||||
|
||||
var best = scored[0];
|
||||
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
|
||||
if (best.Similarity < options.MinimumIdentitySimilarity)
|
||||
{
|
||||
return Reject(
|
||||
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
if (runnerUp is { } runnerUpSimilarity &&
|
||||
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
|
||||
{
|
||||
return Reject(
|
||||
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
|
||||
best.IdentityId,
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
best.IdentityId,
|
||||
"accepted",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
private ResemblyzerVoiceClusterMatchResult Reject(
|
||||
string reason,
|
||||
double cohesion,
|
||||
double? bestSimilarity,
|
||||
double? runnerUpSimilarity)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
null,
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
private (int IdentityId, double Similarity)? ScoreCandidate(
|
||||
IReadOnlyList<float[]> normalizedQuery,
|
||||
ResemblyzerVoiceVectorCandidate candidate)
|
||||
{
|
||||
if (candidate.Vectors.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var normalizedCandidate = candidate.Vectors
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
|
||||
var similarities = normalizedQuery
|
||||
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
|
||||
.Order()
|
||||
.ToList();
|
||||
return (candidate.IdentityId, Median(similarities));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
|
||||
candidate.IdentityId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] Centroid(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var centroid = new float[ResemblyzerVectorContract.Dimensions];
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] += vector[index];
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] /= vectors.Count;
|
||||
}
|
||||
|
||||
return centroid;
|
||||
}
|
||||
|
||||
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
if (vectors.Count < 2)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
double total = 0;
|
||||
var pairs = 0;
|
||||
for (var first = 0; first < vectors.Count - 1; first++)
|
||||
{
|
||||
for (var second = first + 1; second < vectors.Count; second++)
|
||||
{
|
||||
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
|
||||
pairs++;
|
||||
}
|
||||
}
|
||||
|
||||
return total / pairs;
|
||||
}
|
||||
|
||||
private static double Median(IReadOnlyList<double> sortedValues)
|
||||
{
|
||||
var middle = sortedValues.Count / 2;
|
||||
return sortedValues.Count % 2 == 0
|
||||
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
|
||||
: sortedValues[middle];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorUpdateResult(
|
||||
int AddedCount,
|
||||
int RemovedCount)
|
||||
{
|
||||
public bool Changed => AddedCount > 0 || RemovedCount > 0;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPruner
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
|
||||
|
||||
public ResemblyzerVoiceVectorOutlierPruner(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public int Prune(SpeakerIdentity identity)
|
||||
{
|
||||
var compatible = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
|
||||
identity,
|
||||
options.ModelId,
|
||||
logger))
|
||||
{
|
||||
try
|
||||
{
|
||||
compatible.Add(new DecodedSpeakerVoiceVector(
|
||||
entry.Stored,
|
||||
SpeakerVoiceVectors.Normalize(
|
||||
entry.Vector,
|
||||
$"Stored voice vector {entry.Stored.Id}")));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
|
||||
entry.Stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
var minimumVectors = options.OutlierPruningMinimumVectors;
|
||||
if (compatible.Count < minimumVectors)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
|
||||
var ranked = clusters
|
||||
.GroupBy(cluster => cluster)
|
||||
.Where(group => group.Key > 0)
|
||||
.Select(group => new { Id = group.Key, Count = group.Count() })
|
||||
.OrderByDescending(group => group.Count)
|
||||
.ThenBy(group => group.Id)
|
||||
.ToList();
|
||||
if (ranked.Count == 0)
|
||||
{
|
||||
return Skip(identity, "no dense cluster was found");
|
||||
}
|
||||
|
||||
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
|
||||
{
|
||||
return Skip(identity, "no uniquely largest dense cluster was found");
|
||||
}
|
||||
|
||||
var dominant = ranked[0];
|
||||
var ratio = (double)dominant.Count / compatible.Count;
|
||||
if (ratio < options.OutlierPruningMinimumClusterRatio)
|
||||
{
|
||||
return Skip(
|
||||
identity,
|
||||
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
|
||||
}
|
||||
|
||||
var removed = compatible
|
||||
.Where((_, index) => clusters[index] != dominant.Id)
|
||||
.Select(item => item.Stored)
|
||||
.ToList();
|
||||
foreach (var vector in removed)
|
||||
{
|
||||
identity.VoiceVectors.Remove(vector);
|
||||
}
|
||||
|
||||
if (removed.Count > 0)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
|
||||
removed.Count,
|
||||
identity.Id,
|
||||
dominant.Count,
|
||||
compatible.Count);
|
||||
return removed.Count;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var removedBeforeAdding = Prune(identity);
|
||||
var added = SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
options.ModelId,
|
||||
options.MaxVectorsPerIdentity,
|
||||
createdAt);
|
||||
var removedAfterAdding = Prune(identity);
|
||||
var result = new ResemblyzerVoiceVectorUpdateResult(
|
||||
added,
|
||||
removedBeforeAdding + removedAfterAdding);
|
||||
if (result.Changed)
|
||||
{
|
||||
identity.UpdatedAt = createdAt;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var neighbors = Enumerable.Range(0, vectors.Count)
|
||||
.Select(index => Enumerable.Range(0, vectors.Count)
|
||||
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
|
||||
.ToList())
|
||||
.ToList();
|
||||
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
|
||||
var labels = new int[vectors.Count];
|
||||
var clusterId = 0;
|
||||
|
||||
for (var point = 0; point < vectors.Count; point++)
|
||||
{
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
labels[point] = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
clusterId++;
|
||||
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
|
||||
}
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
private static void ExpandCluster(
|
||||
int seed,
|
||||
int clusterId,
|
||||
IReadOnlyList<List<int>> neighbors,
|
||||
int[] labels,
|
||||
int minimumNeighbors)
|
||||
{
|
||||
labels[seed] = clusterId;
|
||||
var pending = new Queue<int>(neighbors[seed]);
|
||||
var queued = neighbors[seed].ToHashSet();
|
||||
while (pending.TryDequeue(out var point))
|
||||
{
|
||||
if (labels[point] == -1)
|
||||
{
|
||||
labels[point] = clusterId;
|
||||
}
|
||||
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
labels[point] = clusterId;
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var neighbor in neighbors[point])
|
||||
{
|
||||
if (queued.Add(neighbor))
|
||||
{
|
||||
pending.Enqueue(neighbor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int Skip(
|
||||
SpeakerIdentity identity,
|
||||
string reason)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
|
||||
identity.Id,
|
||||
reason);
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedService : BackgroundService
|
||||
{
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly bool speakerIdentificationEnabled;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
|
||||
public ResemblyzerWarmupHostedService(
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerWarmupHostedService> logger)
|
||||
{
|
||||
this.encoder = encoder;
|
||||
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!speakerIdentificationEnabled || !options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Yield();
|
||||
try
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
await encoder.WarmUpAsync(stoppingToken);
|
||||
logger.LogInformation(
|
||||
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
|
||||
VaultPath.Resolve(options.RuntimeFolder));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
|
||||
|
||||
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
||||
|
||||
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
|
||||
|
||||
public List<SpeakerIdentityReference> References { get; set; } = [];
|
||||
|
||||
[NotMapped]
|
||||
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
|
||||
string.IsNullOrWhiteSpace(reference.TranscriptPath);
|
||||
}
|
||||
|
||||
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerVoiceVector
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public string ModelId { get; set; } = "";
|
||||
|
||||
public int Dimensions { get; set; }
|
||||
|
||||
public byte[] VectorBytes { get; set; } = [];
|
||||
|
||||
public string Fingerprint { get; set; } = "";
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerAlias
|
||||
{
|
||||
public int Id { get; set; }
|
||||
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
|
||||
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
||||
|
||||
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
|
||||
|
||||
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.VoiceVectors)
|
||||
.WithOne(vector => vector.SpeakerIdentity)
|
||||
.HasForeignKey(vector => vector.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.References)
|
||||
.WithOne(reference => reference.SpeakerIdentity)
|
||||
.HasForeignKey(reference => reference.SpeakerIdentityId)
|
||||
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
|
||||
|
||||
modelBuilder.Entity<SpeakerIdentityReference>()
|
||||
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
|
||||
.IsUnique();
|
||||
|
||||
@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
||||
|
||||
public SpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeInto(
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker);
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
options.Resemblyzer.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
||||
source.Id,
|
||||
|
||||
@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityMerger
|
||||
{
|
||||
public static void MergeIntoAndPrune(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
|
||||
{
|
||||
outlierPruner.Prune(target);
|
||||
outlierPruner.Prune(source);
|
||||
MergeInto(target, source, maxSnippets, maxVoiceVectors);
|
||||
outlierPruner.Prune(target);
|
||||
}
|
||||
|
||||
public static void MergeInto(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets)
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors = int.MaxValue)
|
||||
{
|
||||
AddAlias(target, source.CanonicalName);
|
||||
foreach (var alias in source.Aliases)
|
||||
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
|
||||
.ToList();
|
||||
target.Snippets.Clear();
|
||||
target.Snippets.AddRange(retainedSnippets);
|
||||
|
||||
var retainedVectors = target.VoiceVectors
|
||||
.Concat(source.VoiceVectors)
|
||||
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
|
||||
.OrderByDescending(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Take(Math.Max(1, maxVoiceVectors))
|
||||
.Select(vector => new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentity = target,
|
||||
ModelId = vector.ModelId,
|
||||
Dimensions = vector.Dimensions,
|
||||
VectorBytes = vector.VectorBytes.ToArray(),
|
||||
Fingerprint = vector.Fingerprint,
|
||||
CreatedAt = vector.CreatedAt
|
||||
})
|
||||
.ToList();
|
||||
target.VoiceVectors.Clear();
|
||||
target.VoiceVectors.AddRange(retainedVectors);
|
||||
target.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityNaming
|
||||
{
|
||||
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[] { identity.CanonicalName }
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool MatchesAnyAcceptedName(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
public static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(MeetingAttendeeNames.NormalizeDisplayName)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool UpdateCandidateNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
var resetToAttendees = intersection.Count == 0;
|
||||
SetCandidates(identity, resetToAttendees ? attendees : intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
}
|
||||
|
||||
return resetToAttendees;
|
||||
}
|
||||
|
||||
public static void SetCandidates(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
|
||||
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
|
||||
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
|
||||
"SpeakerIdentityId" INTEGER NOT NULL,
|
||||
"ModelId" TEXT NOT NULL,
|
||||
"Dimensions" INTEGER NOT NULL,
|
||||
"VectorBytes" BLOB NOT NULL,
|
||||
"Fingerprint" TEXT NOT NULL,
|
||||
"CreatedAt" TEXT NOT NULL,
|
||||
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
|
||||
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
|
||||
);
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
||||
|
||||
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
target.UpdatedAt = now;
|
||||
ResetCandidates(target, [targetName]);
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
AddMeetingReference(target, meetingReference);
|
||||
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(speaker.Trim());
|
||||
}
|
||||
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
speakerMappings[speaker] = speakerName;
|
||||
alreadyIdentifiedNames.Add(speakerName);
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(acceptedName);
|
||||
}
|
||||
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
speakerName,
|
||||
GetAcceptedNames(identity).ToList()));
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = MatchesAttendees(identity, attendees),
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration);
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration);
|
||||
logger.LogInformation(
|
||||
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
||||
operation,
|
||||
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Include(identity => identity.References)
|
||||
.ToListAsync(cancellationToken);
|
||||
return identities
|
||||
.Where(identity => GetAcceptedNames(identity).Contains(name))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.ToListAsync(cancellationToken);
|
||||
return candidates
|
||||
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => identity.CandidateNames.Any(candidate =>
|
||||
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
|
||||
identity.Aliases.Any(alias =>
|
||||
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
||||
}
|
||||
|
||||
private static bool IsSameReference(
|
||||
SpeakerIdentityReference first,
|
||||
SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool MatchesAcceptedNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
private static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
|
||||
}
|
||||
|
||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (intersection.Count == 0)
|
||||
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
if (resetToAttendees)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
||||
identity.Id,
|
||||
FormatNames(attendees));
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
|
||||
identity.Id,
|
||||
FormatNames(currentCandidates),
|
||||
FormatNames(attendees),
|
||||
FormatNames(intersection));
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
||||
identity.Id,
|
||||
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
}
|
||||
}
|
||||
|
||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
|
||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
identity.CanonicalName
|
||||
}
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
|
||||
}
|
||||
|
||||
private static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record SpeakerSampleCollectionPolicy(
|
||||
int MinimumRetainedSamples,
|
||||
bool RequireNonOverlappingSamples)
|
||||
{
|
||||
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
|
||||
|
||||
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
|
||||
{
|
||||
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
|
||||
}
|
||||
|
||||
public int ResolveRetainedSampleLimit(int configuredLimit)
|
||||
{
|
||||
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SpeakerSampleDurationConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
SpeakerIdentificationOptions options,
|
||||
string context)
|
||||
{
|
||||
ValidateOrThrow(
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
context);
|
||||
}
|
||||
|
||||
public static void ValidateOrThrow(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(minimumDuration, maximumDuration);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (minimumDuration < TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
|
||||
}
|
||||
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
|
||||
}
|
||||
|
||||
return minimumDuration > maximumDuration
|
||||
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ResemblyzerSpeakerRecognitionConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(options);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
|
||||
{
|
||||
if (options.RequiredVectorsPerSpeaker < 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
|
||||
}
|
||||
|
||||
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumNeighbors < 1 ||
|
||||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
|
||||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumClusterCohesion))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumIdentitySimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
return !double.IsFinite(options.MinimumSimilarityMargin) ||
|
||||
options.MinimumSimilarityMargin is < 0 or > 2
|
||||
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
|
||||
: null;
|
||||
}
|
||||
|
||||
private static bool IsCosine(double value)
|
||||
{
|
||||
return double.IsFinite(value) && value is >= -1 and <= 1;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
|
||||
: IValidateOptions<MeetingAssistantOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
|
||||
{
|
||||
var error = SpeakerSampleDurationConfiguration.GetError(
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleDuration) ??
|
||||
ResemblyzerSpeakerRecognitionConfiguration.GetError(
|
||||
options.SpeakerIdentification.Resemblyzer);
|
||||
return error is null
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record SpeakerSampleSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
TimeSpan SpeechDuration,
|
||||
TimeSpan MaximumDuration,
|
||||
IReadOnlyList<TranscriptionSegment> Segments)
|
||||
{
|
||||
public static SpeakerSampleSpan? Create(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(maximumDuration),
|
||||
maximumDuration,
|
||||
"Speaker sample maximum duration must be greater than zero.");
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
|
||||
return boundedEnd > segment.Start
|
||||
? new SpeakerSampleSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
boundedEnd,
|
||||
boundedEnd - segment.Start,
|
||||
maximumDuration,
|
||||
[segment with { End = boundedEnd }])
|
||||
: null;
|
||||
}
|
||||
|
||||
public bool TryExtend(
|
||||
TranscriptionSegment segment,
|
||||
out SpeakerSampleSpan extended)
|
||||
{
|
||||
extended = this;
|
||||
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
|
||||
segment.Start >= Start + MaximumDuration)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, Start + MaximumDuration);
|
||||
if (boundedEnd <= segment.Start)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var uncoveredStart = segment.Start > End ? segment.Start : End;
|
||||
var additionalSpeechDuration = boundedEnd > uncoveredStart
|
||||
? boundedEnd - uncoveredStart
|
||||
: TimeSpan.Zero;
|
||||
extended = this with
|
||||
{
|
||||
End = boundedEnd > End ? boundedEnd : End,
|
||||
SpeechDuration = SpeechDuration + additionalSpeechDuration,
|
||||
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', Segments
|
||||
.Select(segment => segment.Text)
|
||||
.Where(text => !string.IsNullOrWhiteSpace(text))));
|
||||
}
|
||||
|
||||
private static TimeSpan Min(TimeSpan left, TimeSpan right)
|
||||
{
|
||||
return left < right ? left : right;
|
||||
}
|
||||
}
|
||||
@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerSampleSpanSelector
|
||||
{
|
||||
public static bool CanExtend(
|
||||
string currentSpeaker,
|
||||
TimeSpan currentEnd,
|
||||
TranscriptionSegment nextSegment,
|
||||
TimeSpan maximumSegmentGap)
|
||||
{
|
||||
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
|
||||
nextSegment.Start - currentEnd <= maximumSegmentGap;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan maximumSegmentGap,
|
||||
TimeSpan minimumDuration)
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
var best = new List<TranscriptionSegment>();
|
||||
var current = new List<TranscriptionSegment>();
|
||||
|
||||
SpeakerSampleSpan? best = null;
|
||||
SpeakerSampleSpan? current = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (current.Count == 0)
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
if (IsSpeaker(segment, speaker))
|
||||
{
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is not null &&
|
||||
(best is null || current.SpeechDuration > best.SpeechDuration))
|
||||
{
|
||||
current.Clear();
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
best = current;
|
||||
}
|
||||
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
return SpanDuration(best) >= minimumDuration
|
||||
? best
|
||||
return best is not null && best.SpeechDuration >= minimumDuration
|
||||
? best.Segments
|
||||
: [];
|
||||
}
|
||||
|
||||
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
int maxSpans)
|
||||
{
|
||||
if (maxSpans <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var completed = new List<SpeakerSampleSpan>();
|
||||
SpeakerSampleSpan? current = null;
|
||||
TimeSpan? lastCompletedEnd = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (lastCompletedEnd is { } end && segment.Start < end)
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is null || current.SpeechDuration < minimumDuration)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
completed.Add(current);
|
||||
lastCompletedEnd = current.End;
|
||||
current = null;
|
||||
}
|
||||
|
||||
return completed
|
||||
.OrderByDescending(span => span.SpeechDuration)
|
||||
.ThenBy(span => span.Start)
|
||||
.Take(maxSpans)
|
||||
.OrderBy(span => span.Start)
|
||||
.Select(span => span.Segments)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return segments.Count == 0
|
||||
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
|
||||
: segments[^1].End - segments[0].Start;
|
||||
}
|
||||
|
||||
private static SpeakerSampleSpan? ExtendOrStart(
|
||||
SpeakerSampleSpan? current,
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
return current is not null && current.TryExtend(segment, out var extended)
|
||||
? extended
|
||||
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
}
|
||||
|
||||
private static bool IsSpeaker(
|
||||
TranscriptionSegment segment,
|
||||
string speaker)
|
||||
{
|
||||
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> LongerSpan(
|
||||
List<TranscriptionSegment> current,
|
||||
List<TranscriptionSegment> best)
|
||||
{
|
||||
return SpanDuration(current) > SpanDuration(best)
|
||||
? current.ToList()
|
||||
: best;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record DecodedSpeakerVoiceVector(
|
||||
SpeakerVoiceVector Stored,
|
||||
float[] Vector);
|
||||
|
||||
internal static class ResemblyzerVectorContract
|
||||
{
|
||||
public const int Dimensions = 256;
|
||||
}
|
||||
|
||||
internal static class SpeakerVoiceVectors
|
||||
{
|
||||
public static float[] Decode(SpeakerVoiceVector stored)
|
||||
{
|
||||
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
|
||||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
|
||||
}
|
||||
|
||||
var vector = new float[stored.Dimensions];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
|
||||
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
|
||||
}
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
public static int AddDistinct(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<float[]> vectors,
|
||||
string modelId,
|
||||
int maximumCount,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var limit = Math.Max(1, maximumCount);
|
||||
var fingerprints = identity.VoiceVectors
|
||||
.Select(vector => vector.Fingerprint)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var added = 0;
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
if (identity.VoiceVectors.Count >= limit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var bytes = Encode(vector);
|
||||
var fingerprint = Fingerprint(modelId, bytes);
|
||||
if (!fingerprints.Add(fingerprint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = modelId,
|
||||
Dimensions = vector.Length,
|
||||
VectorBytes = bytes,
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
});
|
||||
added++;
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<float[]> DecodeCompatible(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
return DecodeCompatibleEntries(identity, modelId, logger)
|
||||
.Select(entry => entry.Vector)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
var vectors = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var stored in identity.VoiceVectors
|
||||
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
|
||||
.OrderBy(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Id))
|
||||
{
|
||||
try
|
||||
{
|
||||
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
|
||||
stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
public static double Cosine(float[] first, float[] second)
|
||||
{
|
||||
double dot = 0;
|
||||
for (var index = 0; index < first.Length; index++)
|
||||
{
|
||||
dot += first[index] * second[index];
|
||||
}
|
||||
|
||||
return dot;
|
||||
}
|
||||
|
||||
public static float[] Normalize(float[]? vector, string description)
|
||||
{
|
||||
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
double magnitudeSquared = 0;
|
||||
foreach (var value in vector)
|
||||
{
|
||||
if (!float.IsFinite(value))
|
||||
{
|
||||
throw new InvalidDataException($"{description} contains a non-finite value.");
|
||||
}
|
||||
|
||||
magnitudeSquared += value * value;
|
||||
}
|
||||
|
||||
var magnitude = Math.Sqrt(magnitudeSquared);
|
||||
if (magnitude <= double.Epsilon)
|
||||
{
|
||||
throw new InvalidDataException($"{description} has zero magnitude.");
|
||||
}
|
||||
|
||||
return vector.Select(value => (float)(value / magnitude)).ToArray();
|
||||
}
|
||||
|
||||
private static byte[] Encode(float[] vector)
|
||||
{
|
||||
if (vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException("Voice vector contains a non-finite value.");
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteSingleLittleEndian(
|
||||
bytes.AsSpan(index * sizeof(float), sizeof(float)),
|
||||
vector[index]);
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static string Fingerprint(string modelId, byte[] vectorBytes)
|
||||
{
|
||||
var modelBytes = Encoding.UTF8.GetBytes(modelId);
|
||||
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
|
||||
modelBytes.CopyTo(input, 0);
|
||||
input[modelBytes.Length] = 0;
|
||||
vectorBytes.CopyTo(input, modelBytes.Length + 1);
|
||||
return Convert.ToHexString(SHA256.HashData(input));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public interface IResemblyzerVoiceEncoder
|
||||
{
|
||||
Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task WarmUpAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private const string EnvironmentSchemaVersion = "venv-v2";
|
||||
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
|
||||
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
|
||||
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
|
||||
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
|
||||
private const string LibrosaRequirement = "librosa>=0.9.1";
|
||||
private const string ScipyRequirement = "scipy>=1.2.1";
|
||||
private readonly SemaphoreSlim commandLock = new(1, 1);
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
|
||||
private string? verifiedEnvironmentPythonPath;
|
||||
|
||||
public VenvResemblyzerVoiceEncoder(
|
||||
ICommandRunner commandRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<VenvResemblyzerVoiceEncoder> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (wavSamples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
if (wavSamples.Any(sample => sample.Length == 0))
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
|
||||
}
|
||||
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(inputFolder);
|
||||
try
|
||||
{
|
||||
for (var index = 0; index < wavSamples.Count; index++)
|
||||
{
|
||||
await File.WriteAllBytesAsync(
|
||||
Path.Combine(inputFolder, $"{index:D4}.wav"),
|
||||
wavSamples[index],
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(inputFolder, recursive: true);
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
catch (UnauthorizedAccessException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
|
||||
string inputFolder,
|
||||
int expectedCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
var pythonPath = await EnsureEnvironmentAsync(token);
|
||||
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
|
||||
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
|
||||
var result = await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[scriptPath, Path.GetFullPath(inputFolder)],
|
||||
"encoding",
|
||||
token);
|
||||
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
await EnsureEnvironmentAsync(token);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateDependencySettings();
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
|
||||
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
|
||||
var readyPath = Path.Combine(environmentFolder, ".ready");
|
||||
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
|
||||
&& File.Exists(pythonPath)
|
||||
&& File.Exists(readyPath))
|
||||
{
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
if (File.Exists(pythonPath) && File.Exists(readyPath))
|
||||
{
|
||||
var verification = await commandRunner.RunAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
cancellationToken);
|
||||
if (verification.ExitCode == 0)
|
||||
{
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
|
||||
verification.ExitCode,
|
||||
verification.StandardError);
|
||||
File.Delete(readyPath);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(environmentFolder);
|
||||
await RunRequiredAsync(
|
||||
options.PythonCommand,
|
||||
["-m", "venv", "--clear", environmentFolder],
|
||||
"virtual environment creation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
["-m", "pip", "install", "--upgrade", "pip"],
|
||||
"pip upgrade",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
"--index-url", options.TorchIndexUrl,
|
||||
TorchRequirement
|
||||
],
|
||||
"CPU PyTorch installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
.. RuntimeDependencyRequirements
|
||||
],
|
||||
"Resemblyzer dependency installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install", "--no-deps",
|
||||
ResemblyzerRequirement
|
||||
],
|
||||
"Resemblyzer installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
"environment verification",
|
||||
cancellationToken);
|
||||
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunRequiredAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
|
||||
ThrowIfFailed(result, operation);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string BuildEncodingScript()
|
||||
{
|
||||
return
|
||||
"import json\n"
|
||||
+ "import sys\n"
|
||||
+ "import numpy as np\n"
|
||||
+ "from pathlib import Path\n"
|
||||
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
|
||||
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
|
||||
+ "vectors = []\n"
|
||||
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
|
||||
+ " wav = preprocess_wav(path)\n"
|
||||
+ " vector = encoder.embed_utterance(wav)\n"
|
||||
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
|
||||
+ $"print('{JsonStart}')\n"
|
||||
+ "print(json.dumps(vectors, allow_nan=False))\n"
|
||||
+ $"print('{JsonEnd}')\n";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
|
||||
{
|
||||
var json = ExtractJson(output);
|
||||
float[][]? vectors;
|
||||
try
|
||||
{
|
||||
vectors = JsonSerializer.Deserialize<float[][]>(json);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
|
||||
}
|
||||
|
||||
if (vectors is null || vectors.Length != expectedCount)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
|
||||
}
|
||||
|
||||
return vectors
|
||||
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string ExtractJson(string output)
|
||||
{
|
||||
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||
if (start < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
|
||||
}
|
||||
|
||||
start += JsonStart.Length;
|
||||
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||
if (end < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
|
||||
}
|
||||
|
||||
return output[start..end].Trim();
|
||||
}
|
||||
|
||||
private string BuildEnvironmentKey()
|
||||
{
|
||||
var settings = string.Join('\n', EnvironmentManifest);
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
|
||||
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
|
||||
}
|
||||
|
||||
private string TorchRequirement => $"torch=={options.TorchVersion}";
|
||||
|
||||
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
|
||||
|
||||
private static string[] WarmupArguments =>
|
||||
[
|
||||
"-c",
|
||||
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
|
||||
];
|
||||
|
||||
private string[] RuntimeDependencyRequirements =>
|
||||
[
|
||||
NumpyBeforePython313Requirement,
|
||||
NumpyFromPython313Requirement,
|
||||
LibrosaRequirement,
|
||||
ScipyRequirement,
|
||||
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
|
||||
];
|
||||
|
||||
private IEnumerable<string> EnvironmentManifest =>
|
||||
[
|
||||
EnvironmentSchemaVersion,
|
||||
options.PythonCommand,
|
||||
options.TorchIndexUrl,
|
||||
TorchRequirement,
|
||||
.. RuntimeDependencyRequirements,
|
||||
ResemblyzerRequirement
|
||||
];
|
||||
|
||||
private static string GetEnvironmentPythonPath(string environmentFolder)
|
||||
{
|
||||
return OperatingSystem.IsWindows()
|
||||
? Path.Combine(environmentFolder, "Scripts", "python.exe")
|
||||
: Path.Combine(environmentFolder, "bin", "python");
|
||||
}
|
||||
|
||||
private void ValidateDependencySettings()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.PythonCommand))
|
||||
{
|
||||
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
|
||||
}
|
||||
|
||||
ValidateVersion(options.PackageVersion, "Resemblyzer package");
|
||||
ValidateVersion(options.TorchVersion, "PyTorch");
|
||||
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
|
||||
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|
||||
|| indexUri.Scheme != Uri.UriSchemeHttps)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateVersion(string version, string dependency)
|
||||
{
|
||||
if (!PackageVersionPattern().IsMatch(version))
|
||||
{
|
||||
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<T> RunWithTimeoutAsync<T>(
|
||||
Func<CancellationToken, Task<T>> operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(options.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
try
|
||||
{
|
||||
return await operation(linkedSource?.Token ?? cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (
|
||||
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ThrowIfFailed(CommandResult result, string operation)
|
||||
{
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
}
|
||||
|
||||
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex PackageVersionPattern();
|
||||
}
|
||||
@@ -7,7 +7,6 @@ using System.Text.Json.Nodes;
|
||||
using Microsoft.Agents.AI.Compaction;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenAI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
namespace MeetingAssistant.Summary;
|
||||
@@ -798,7 +797,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
|
||||
string apiKey,
|
||||
AsyncLocal<string?> requestInitiator)
|
||||
{
|
||||
var options = new OpenAIClientOptions
|
||||
var options = new ResponsesClientOptions
|
||||
{
|
||||
Endpoint = httpClient.BaseAddress
|
||||
?? throw new InvalidOperationException("LiteLLM HTTP client requires a base address."),
|
||||
|
||||
@@ -9,6 +9,8 @@ public enum MeetingTaskbarAction
|
||||
OpenSubmenu,
|
||||
StartRecording,
|
||||
StopRecording,
|
||||
PauseTranscription,
|
||||
UnpauseTranscription,
|
||||
AbortRecording,
|
||||
SwitchProfile,
|
||||
SelectMicrophone,
|
||||
@@ -51,6 +53,15 @@ public static class MeetingTaskbarMenuBuilder
|
||||
}
|
||||
|
||||
var secondaryControls = new List<MeetingTaskbarMenuItem>();
|
||||
if (status.IsRecording)
|
||||
{
|
||||
secondaryControls.Add(new MeetingTaskbarMenuItem(
|
||||
status.IsPaused ? "Unpause transcription" : "Pause transcription",
|
||||
status.IsPaused
|
||||
? MeetingTaskbarAction.UnpauseTranscription
|
||||
: MeetingTaskbarAction.PauseTranscription));
|
||||
}
|
||||
|
||||
if (microphones is { Count: > 0 })
|
||||
{
|
||||
secondaryControls.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
|
||||
|
||||
@@ -249,6 +249,12 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
|
||||
case MeetingTaskbarAction.StopRecording:
|
||||
await coordinator.StopAsync(CancellationToken.None);
|
||||
break;
|
||||
case MeetingTaskbarAction.PauseTranscription:
|
||||
await coordinator.SetTranscriptionPausedAsync(true, CancellationToken.None);
|
||||
break;
|
||||
case MeetingTaskbarAction.UnpauseTranscription:
|
||||
await coordinator.SetTranscriptionPausedAsync(false, CancellationToken.None);
|
||||
break;
|
||||
case MeetingTaskbarAction.AbortRecording:
|
||||
await coordinator.AbortAsync(CancellationToken.None);
|
||||
break;
|
||||
|
||||
@@ -96,14 +96,18 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions()
|
||||
private IEnumerable<PyannoteRuntimeOptions> GetEnabledDiarizationOptions()
|
||||
{
|
||||
return launchProfiles.GetProfiles()
|
||||
.SelectMany(profile => GetEnabledDiarizationOptions(profile.Options))
|
||||
var transcriptionRuntimes = launchProfiles.GetProfiles()
|
||||
.SelectMany(profile => GetEnabledTranscriptionDiarizationOptions(profile.Options));
|
||||
var speakerValidationRuntimes = GetEnabledSpeakerValidationDiarizationOptions(
|
||||
launchProfiles.GetRequiredProfile(null).Options);
|
||||
return transcriptionRuntimes
|
||||
.Concat(speakerValidationRuntimes)
|
||||
.DistinctBy(CreateWarmUpKey);
|
||||
}
|
||||
|
||||
private static IEnumerable<PyannoteDiarizationOptions> GetEnabledDiarizationOptions(
|
||||
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledTranscriptionDiarizationOptions(
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
if (options.Recording.TranscriptionProvider.Equals("whisper-local", StringComparison.OrdinalIgnoreCase) &&
|
||||
@@ -111,15 +115,19 @@ public sealed class PyannoteDiarizationWarmupHostedService : IHostedService
|
||||
{
|
||||
yield return options.WhisperLocal.Diarization;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.SpeakerIdentification.PyannoteValidation.Enabled &&
|
||||
options.SpeakerIdentification.PyannoteValidation.Diarization.Enabled)
|
||||
private static IEnumerable<PyannoteRuntimeOptions> GetEnabledSpeakerValidationDiarizationOptions(
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
if (!options.SpeakerIdentification.Resemblyzer.Enabled &&
|
||||
options.SpeakerIdentification.PyannoteValidation.Enabled)
|
||||
{
|
||||
yield return options.SpeakerIdentification.PyannoteValidation.Diarization;
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateWarmUpKey(PyannoteDiarizationOptions diarization)
|
||||
private static string CreateWarmUpKey(PyannoteRuntimeOptions diarization)
|
||||
{
|
||||
return string.Join(
|
||||
'\u001f',
|
||||
|
||||
@@ -28,7 +28,12 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await FinalizeAsync(
|
||||
if (!options.WhisperLocal.Diarization.Enabled)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await FinalizeEnabledAsync(
|
||||
audioPath,
|
||||
liveSegments,
|
||||
options.WhisperLocal.Diarization,
|
||||
@@ -36,14 +41,14 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<TranscriptionSegment>> FinalizeAsync(
|
||||
internal async Task<IReadOnlyList<TranscriptionSegment>> FinalizeEnabledAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> liveSegments,
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!diarization.Enabled || liveSegments.Count == 0)
|
||||
if (liveSegments.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
@@ -83,14 +88,9 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!diarization.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var token = ResolveToken(diarization);
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
@@ -137,7 +137,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
private async Task<CommandResult> RunDiarizationAsync(
|
||||
string fullAudioPath,
|
||||
string token,
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -171,7 +171,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
|
||||
private async Task EnsureDockerImageAsync(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
string modelsFolder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -198,7 +198,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildDockerfile(PyannoteDiarizationOptions diarization)
|
||||
private static string BuildDockerfile(PyannoteRuntimeOptions diarization)
|
||||
{
|
||||
return
|
||||
$"FROM {diarization.BaseImage}\n"
|
||||
@@ -211,7 +211,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
|
||||
private string[] BuildDockerArguments(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
string fullAudioPath,
|
||||
string modelsFolder,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||
@@ -242,7 +242,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
|
||||
private string[] BuildWarmUpDockerArguments(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
string modelsFolder)
|
||||
{
|
||||
return
|
||||
@@ -269,7 +269,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
}
|
||||
|
||||
private static string BuildPythonCommand(
|
||||
PyannoteDiarizationOptions diarization,
|
||||
PyannoteRuntimeOptions diarization,
|
||||
SpeechRecognitionPipelineOptions pipelineOptions)
|
||||
{
|
||||
var model = diarization.Model;
|
||||
@@ -296,7 +296,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
+ "python /tmp/meeting_assistant_pyannote.py";
|
||||
}
|
||||
|
||||
private static string BuildWarmUpPythonCommand(PyannoteDiarizationOptions diarization)
|
||||
private static string BuildWarmUpPythonCommand(PyannoteRuntimeOptions diarization)
|
||||
{
|
||||
var model = diarization.Model;
|
||||
return
|
||||
@@ -475,7 +475,7 @@ public sealed class PyannoteTranscriptFinalizer
|
||||
return turns;
|
||||
}
|
||||
|
||||
private static string ResolveToken(PyannoteDiarizationOptions options)
|
||||
private static string ResolveToken(PyannoteRuntimeOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Token))
|
||||
{
|
||||
|
||||
@@ -24,6 +24,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
||||
private readonly IMeetingMetadataProvider meetingMetadataProvider;
|
||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||
private readonly ISpeakerIdentityMergeService identityMergeService;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly AsrDiagnosticService asrDiagnosticService;
|
||||
private readonly IConfiguration configuration;
|
||||
|
||||
@@ -38,6 +39,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
||||
IMeetingMetadataProvider meetingMetadataProvider,
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
ISpeakerIdentityMergeService identityMergeService,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
AsrDiagnosticService asrDiagnosticService,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
@@ -51,6 +53,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
||||
this.meetingMetadataProvider = meetingMetadataProvider;
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.identityMergeService = identityMergeService;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.asrDiagnosticService = asrDiagnosticService;
|
||||
this.configuration = configuration;
|
||||
}
|
||||
@@ -78,7 +81,8 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
|
||||
identityMergeService,
|
||||
asrDiagnosticService,
|
||||
configuration,
|
||||
logger: logger);
|
||||
logger: logger,
|
||||
outlierPruner: outlierPruner);
|
||||
var messages = conversation
|
||||
.Select(ToChatMessage)
|
||||
.Append(new ChatMessage(ChatRole.User, userMessage.Trim()))
|
||||
|
||||
@@ -38,10 +38,14 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor
|
||||
Explain the final change briefly after the tools finish.
|
||||
""";
|
||||
|
||||
private readonly HostOperatingSystem hostOperatingSystem;
|
||||
private readonly ILogger<WorkflowRulesEditorInstructionBuilder> logger;
|
||||
|
||||
public WorkflowRulesEditorInstructionBuilder(ILogger<WorkflowRulesEditorInstructionBuilder> logger)
|
||||
public WorkflowRulesEditorInstructionBuilder(
|
||||
HostOperatingSystem hostOperatingSystem,
|
||||
ILogger<WorkflowRulesEditorInstructionBuilder> logger)
|
||||
{
|
||||
this.hostOperatingSystem = hostOperatingSystem;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -53,8 +57,10 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor
|
||||
: editorOptions.InitialPrompt!;
|
||||
var rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath) ?? "<not configured>";
|
||||
var docs = await ReadWorkflowDocsAsync(cancellationToken);
|
||||
var platformGuidance = BuildPlatformGuidance(hostOperatingSystem);
|
||||
|
||||
return configuredPrompt.Trim() + Environment.NewLine + Environment.NewLine +
|
||||
platformGuidance + Environment.NewLine + Environment.NewLine +
|
||||
$"Configured workflow rules file: {rulesPath}" + Environment.NewLine + Environment.NewLine +
|
||||
"Configuration tools can read and replace the local appsettings JSON file. Use read_config_docs for the configuration reference." + Environment.NewLine + Environment.NewLine +
|
||||
"Log tools can read and search the current application-owned log file and four rotated older files under the temp log folder." + Environment.NewLine + Environment.NewLine +
|
||||
@@ -69,6 +75,28 @@ public sealed class WorkflowRulesEditorInstructionBuilder : IWorkflowRulesEditor
|
||||
"```";
|
||||
}
|
||||
|
||||
private static string BuildPlatformGuidance(HostOperatingSystem operatingSystem)
|
||||
{
|
||||
var platformGuidance = operatingSystem.Kind switch
|
||||
{
|
||||
HostOperatingSystemKind.Windows => """
|
||||
Detected host operating system: Windows.
|
||||
Use Windows-native operational guidance: PowerShell and native Windows commands, Windows-style paths, Windows environment-variable conventions, Windows services and processes, and Task Manager. Do not use macOS-only commands or concepts.
|
||||
""",
|
||||
HostOperatingSystemKind.MacOS => """
|
||||
Detected host operating system: macOS.
|
||||
Use macOS-native operational guidance: zsh and native macOS commands, POSIX-style paths, macOS environment-variable conventions, launchd and LaunchAgents, application processes, and Activity Monitor. Do not use Windows-only commands or concepts.
|
||||
""",
|
||||
_ => """
|
||||
Detected host operating system: unsupported.
|
||||
Use portable .NET concepts. Do not assume Windows or macOS commands, paths, services, environment conventions, or desktop features.
|
||||
"""
|
||||
};
|
||||
|
||||
return platformGuidance + Environment.NewLine +
|
||||
"Prefer the provided Meeting Assistant tools over shell commands when a matching tool exists. Operating-system detection selects appropriate guidance; it does not make unavailable integrations available.";
|
||||
}
|
||||
|
||||
private async Task<string> ReadWorkflowDocsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var path in RuntimeContentLocator.CandidateDocumentationPaths("meeting-workflow-engine.md"))
|
||||
|
||||
@@ -8,6 +8,7 @@ public interface IWorkflowRulesEditorSamplePlaybackQueue
|
||||
Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
#if WINDOWS
|
||||
public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue, IDisposable
|
||||
{
|
||||
private readonly Channel<QueuedSample> channel = Channel.CreateUnbounded<QueuedSample>();
|
||||
@@ -98,3 +99,13 @@ public sealed class WorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEdito
|
||||
|
||||
private sealed record QueuedSample(int SampleId, byte[] WavBytes);
|
||||
}
|
||||
#else
|
||||
public sealed class UnavailableWorkflowRulesEditorSamplePlaybackQueue : IWorkflowRulesEditorSamplePlaybackQueue
|
||||
{
|
||||
public Task<string> QueueAsync(int sampleId, byte[] wavBytes, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return Task.FromResult("Speaker sample playback requires the Windows build of Meeting Assistant.");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -10,6 +10,7 @@ using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using YamlDotNet.Core;
|
||||
using YamlDotNet.Serialization;
|
||||
|
||||
@@ -42,6 +43,7 @@ public sealed class WorkflowRulesEditorTools
|
||||
private readonly IMeetingMetadataProvider? meetingMetadataProvider;
|
||||
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
||||
private readonly ISpeakerIdentityMergeService? identityMergeService;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly AsrDiagnosticService? asrDiagnosticService;
|
||||
private readonly IConfiguration? configuration;
|
||||
private readonly ILogger? logger;
|
||||
@@ -61,7 +63,8 @@ public sealed class WorkflowRulesEditorTools
|
||||
string? logDirectory = null,
|
||||
string? specRootPath = null,
|
||||
string? projectAgentsTemplatePath = null,
|
||||
ILogger? logger = null)
|
||||
ILogger? logger = null,
|
||||
ResemblyzerVoiceVectorOutlierPruner? outlierPruner = null)
|
||||
{
|
||||
this.options = options;
|
||||
rulesPath = WorkflowRulesPathResolver.Resolve(options.Automation.RulesPath);
|
||||
@@ -82,6 +85,9 @@ public sealed class WorkflowRulesEditorTools
|
||||
this.meetingMetadataProvider = meetingMetadataProvider;
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.identityMergeService = identityMergeService;
|
||||
this.outlierPruner = outlierPruner ?? new ResemblyzerVoiceVectorOutlierPruner(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||
this.asrDiagnosticService = asrDiagnosticService;
|
||||
this.configuration = configuration;
|
||||
this.logger = logger;
|
||||
@@ -743,8 +749,8 @@ public sealed class WorkflowRulesEditorTools
|
||||
string[]? candidateNames = null)
|
||||
{
|
||||
return Task.FromResult(
|
||||
"Refused: speaker identities require at least one audio sample. " +
|
||||
"Use a speaker override from an existing transcript speaker sample, or update/merge an existing sampled identity.");
|
||||
"Refused: speaker identities require audio evidence (a WAV sample or voice vector). " +
|
||||
"Use a speaker override from an existing transcript speaker, or update/merge an existing identity.");
|
||||
}
|
||||
|
||||
public async Task<string> UpdateIdentity(
|
||||
@@ -826,10 +832,12 @@ public sealed class WorkflowRulesEditorTools
|
||||
return $"Could not find target {targetIdentityId} or source {sourceIdentityId}.";
|
||||
}
|
||||
|
||||
SpeakerIdentityMerger.MergeInto(
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
speakerOptions.MaxSnippetsPerSpeaker);
|
||||
speakerOptions.MaxSnippetsPerSpeaker,
|
||||
speakerOptions.Resemblyzer.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
context.SpeakerIdentities.Remove(source);
|
||||
await context.SaveChangesAsync();
|
||||
return ToJson(ToIdentityDetail(target));
|
||||
@@ -1567,9 +1575,11 @@ public sealed class WorkflowRulesEditorTools
|
||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References);
|
||||
}
|
||||
|
||||
@@ -1591,6 +1601,7 @@ public sealed class WorkflowRulesEditorTools
|
||||
identity.Aliases.Select(alias => alias.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
||||
identity.CandidateNames.Select(candidate => candidate.Name).Order(StringComparer.OrdinalIgnoreCase).ToArray(),
|
||||
identity.Snippets.Count,
|
||||
identity.VoiceVectors.Count,
|
||||
identity.References.Count,
|
||||
identity.UpdatedAt);
|
||||
}
|
||||
@@ -1650,6 +1661,7 @@ public sealed class WorkflowRulesEditorTools
|
||||
IReadOnlyList<string> Aliases,
|
||||
IReadOnlyList<string> CandidateNames,
|
||||
int SampleCount,
|
||||
int VoiceVectorCount,
|
||||
int ReferenceCount,
|
||||
DateTimeOffset UpdatedAt);
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
"00:10:00"
|
||||
],
|
||||
"AutoStopAfter": "00:30:00",
|
||||
"MaximumPauseDuration": "04:00:00",
|
||||
"InferredEndPadding": "00:01:00",
|
||||
"CheckInterval": "00:00:15"
|
||||
}
|
||||
@@ -119,18 +120,17 @@
|
||||
"MaxMatchCandidates": 100,
|
||||
"MatchIdentityActiveAge": "365.00:00:00",
|
||||
"MaxSnippetsPerSpeaker": 3,
|
||||
"MinimumSampleSpeechDuration": "00:00:30",
|
||||
"MaximumSampleSegmentGap": "00:00:01",
|
||||
"MinimumSampleSpeechDuration": "00:00:10",
|
||||
"MaximumSampleDuration": "00:01:00",
|
||||
"SilenceBetweenSnippetsSeconds": 1,
|
||||
"LiveSampleBufferDuration": "00:10:00",
|
||||
"MergeRecentIdentityAge": "14.00:00:00",
|
||||
"MatchTimeout": "00:03:00",
|
||||
"PyannoteValidation": {
|
||||
"Enabled": true,
|
||||
"Enabled": false,
|
||||
"MinimumSingleSpeakerCoverage": 0.9,
|
||||
"MinimumMatchingKnownSnippetRatio": 0.5,
|
||||
"Diarization": {
|
||||
"Enabled": false,
|
||||
"DockerCommand": "docker",
|
||||
"BaseImage": "python:3.11-slim",
|
||||
"Image": "meeting-assistant-pyannote:local",
|
||||
@@ -142,6 +142,26 @@
|
||||
"TokenEnv": "HF_TOKEN",
|
||||
"CommandTimeout": "01:00:00"
|
||||
}
|
||||
},
|
||||
"Resemblyzer": {
|
||||
"Enabled": false,
|
||||
"RequiredVectorsPerSpeaker": 5,
|
||||
"MaxVectorsPerIdentity": 1000,
|
||||
"OutlierPruningMinimumVectors": 20,
|
||||
"OutlierPruningNeighborSimilarity": 0.75,
|
||||
"OutlierPruningMinimumNeighbors": 3,
|
||||
"OutlierPruningMinimumClusterRatio": 0.60,
|
||||
"MinimumClusterCohesion": 0.75,
|
||||
"MinimumIdentitySimilarity": 0.75,
|
||||
"MinimumSimilarityMargin": 0.05,
|
||||
"ModelId": "resemblyzer-0.1.4-pretrained",
|
||||
"PackageVersion": "0.1.4",
|
||||
"PythonCommand": "python",
|
||||
"TorchVersion": "2.14.0+cpu",
|
||||
"TorchIndexUrl": "https://download.pytorch.org/whl/cpu",
|
||||
"WebRtcVadVersion": "2.0.14",
|
||||
"RuntimeFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Resemblyzer",
|
||||
"CommandTimeout": "00:15:00"
|
||||
}
|
||||
},
|
||||
"Automation": {
|
||||
|
||||
@@ -1,61 +1,69 @@
|
||||
# Meeting Assistant
|
||||
|
||||
Meeting Assistant is Manuel's local .NET meeting capture and knowledge service. It runs on the Windows workstation, records microphone plus system audio, writes Obsidian meeting artifacts, transcribes with speaker attribution, enriches meetings from local context, and drives summary/project agents without depending on Teams, Zoom, or another meeting-platform API for the primary flow.
|
||||
Meeting Assistant is Manuel's local .NET meeting capture and knowledge service. It runs on Windows and macOS, records microphone plus system audio, maintains meeting artifacts in an Obsidian vault, and runs transcription, speaker attribution, screenshot OCR, and agentic summarization without depending on a meeting-platform API for the primary flow.
|
||||
|
||||
## What This Repo Owns
|
||||
## Scope And Runtime Boundary
|
||||
|
||||
This repository owns the application source, tests, OpenSpec requirements, configuration examples, local runtime documentation, and CI build/test workflow for Meeting Assistant. It does not own a homelab deployment stack, Traefik routing, Docker Compose deployment, or production image tag selection.
|
||||
|
||||
The app is intentionally local-first:
|
||||
This repository owns the application source, tests, OpenSpec requirements, checked-in configuration, and CI validation. It does not own workstation startup automation, a homelab deployment stack, Traefik routing, Docker Compose, or a deployed image tag.
|
||||
|
||||
- Windows builds provide the tray icon, global hotkeys, NAudio capture, Outlook Classic COM enrichment, active-window screenshots, and notifications.
|
||||
- Non-Windows builds keep the server/testable service surface but compile out Windows-only integrations.
|
||||
- Portable builds on macOS provide a native menu-bar icon and global hotkeys, and capture the default microphone through AVFoundation plus computer output through ScreenCaptureKit.
|
||||
- Portable builds on other hosts keep the server/testable service surface but do not provide meeting audio capture.
|
||||
- The normal runtime endpoint is local HTTP on port `5090`, with `/health` and `/recording/status` as the safe first checks.
|
||||
- Behavior changes are OpenSpec-driven; current accepted requirements live under `openspec/specs`.
|
||||
Accepted requirements live under `openspec/specs`. Relevant active changes under `openspec/changes` can describe implemented behavior that has not yet been folded into the accepted specs.
|
||||
|
||||
## Quick Start
|
||||
|
||||
For a development build:
|
||||
The project requires the .NET 10 SDK. Build and test it with:
|
||||
|
||||
```powershell
|
||||
dotnet restore MeetingAssistant.slnx
|
||||
dotnet build MeetingAssistant.slnx
|
||||
dotnet test MeetingAssistant.slnx
|
||||
dotnet run --project MeetingAssistant
|
||||
```
|
||||
|
||||
For the durable local background instance, use the snippets restart helper:
|
||||
Do not start a second copy over the live workstation instance. Check the local control surface first:
|
||||
|
||||
```powershell
|
||||
Invoke-RestMethod http://127.0.0.1:5090/health
|
||||
Invoke-RestMethod http://127.0.0.1:5090/recording/status
|
||||
```
|
||||
|
||||
On macOS, build the portable target on the Mac that will run it:
|
||||
|
||||
```zsh
|
||||
dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0
|
||||
dotnet run --project MeetingAssistant/MeetingAssistant.csproj -f net10.0
|
||||
```
|
||||
|
||||
The macOS build compiles bundled Swift helpers into `Native/macos-meeting-audio-capture`, `Native/macos-desktop-controls`, and `Native/macos-meeting-integrations`. On first use, allow **Microphone**, **Screen & System Audio Recording**, and **Calendar Full Access** under System Settings > Privacy & Security. The menu-bar icon and global hotkeys use the same configured bindings as Windows and call the local service surface.
|
||||
|
||||
For a foreground Windows development run when port `5090` is free:
|
||||
|
||||
```powershell
|
||||
dotnet run --project MeetingAssistant --framework net10.0-windows10.0.19041.0
|
||||
```
|
||||
|
||||
The durable workstation instance is published and replaced through automation owned by `Manuel/snippets`:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File C:\Manuel\snippets\restart-meeting-assistant.ps1
|
||||
```
|
||||
|
||||
The helper publishes `MeetingAssistant\MeetingAssistant.csproj` for `net10.0-windows10.0.19041.0` into timestamped folders below `C:\Manuel\meeting-assistant\tmp\meeting-assistant-runtime`. It stops the old port-`5090` instance only after publishing succeeds, starts the published `MeetingAssistant.exe`, waits for `/health`, and writes process logs plus the last PID to:
|
||||
The helper publishes before stopping the old process, starts the timestamped Windows publish on loopback port `5090`, waits for `/health`, and records stdout, stderr, and the PID under `%LOCALAPPDATA%\MeetingAssistant\Logs`. Always confirm `/recording/status` is idle before invoking it.
|
||||
|
||||
```text
|
||||
%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.out.log
|
||||
%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.err.log
|
||||
%LOCALAPPDATA%\MeetingAssistant\Logs\meeting-assistant.pid
|
||||
```
|
||||
## Recording And Control
|
||||
|
||||
Before restarting, always check the running process:
|
||||
|
||||
```powershell
|
||||
Invoke-RestMethod http://127.0.0.1:5090/recording/status
|
||||
```
|
||||
|
||||
Do not restart while a meeting is recording, transcribing, finalizing speaker recognition, waiting on OCR, or summarizing unless the user explicitly approves the interruption.
|
||||
|
||||
## Runtime Behavior
|
||||
|
||||
Recording can be controlled through global hotkeys, the Windows tray icon, or local HTTP endpoints. The default hotkeys are:
|
||||
Recording can be controlled through global hotkeys, the Windows tray icon, the macOS menu-bar icon, or local HTTP endpoints. The default hotkeys are:
|
||||
|
||||
- `Ctrl+Alt+M`: toggle the default recording profile.
|
||||
- `Ctrl+Alt+L`: toggle or switch to the configured `english` launch profile.
|
||||
- `Ctrl+Alt+Z`: abort the active recording and delete that run's artifacts.
|
||||
- `Ctrl+Alt+S`: capture the active window into the current meeting context.
|
||||
- `Ctrl+Alt+L`: start or switch to the configured `english` profile.
|
||||
- `Ctrl+Alt+Z`: abort the active run and delete its artifacts.
|
||||
- `Ctrl+Alt+S`: capture the active window into the meeting context.
|
||||
|
||||
The main local endpoints are:
|
||||
The Windows tray presents `Finish meeting` as the primary action during capture; microphone selection, cancel/discard, profile switching, and pause/unpause transcription remain separate controls. A paused meeting stays active and can still be finished or canceled normally. Windows `Exit` requires confirmation while any meeting is recording or processing. The macOS menu offers profile start actions while idle and `Stop meeting recording and transcribe` plus cancel/discard during capture; its `Exit` action currently exits directly.
|
||||
|
||||
The loopback HTTP surface has no application authentication, so port `5090` must remain a trusted loopback-only control surface. The main endpoints are:
|
||||
|
||||
- `GET /health`
|
||||
- `GET /recording/status`
|
||||
@@ -65,50 +73,61 @@ The main local endpoints are:
|
||||
- `POST /diagnostics/workflow/reload`
|
||||
- `POST /diagnostics/settings-and-logs/show`
|
||||
- `POST /diagnostics/workflow/rules-editor/show`
|
||||
- `GET /diagnostics/platform-capabilities`
|
||||
- `POST /meetings/current/summary/run`
|
||||
- `POST` or `GET /meetings/summary/retry`
|
||||
- `POST` or `GET /meetings/screenshot-ocr/retry`
|
||||
|
||||
Stopping a recording stops new audio capture but lets that run continue transcription drain, speaker processing, meeting-note image OCR, screenshot OCR waits, summary generation, and final artifact updates. A later recording can start while an older stopped run is still finalizing; each run keeps isolated artifact paths and options.
|
||||
Some generated retry links use `GET` while starting work.
|
||||
|
||||
Stopping capture lets buffered transcription, speaker work, meeting-note image OCR, screenshot OCR, and summary generation finish. Another meeting can start while an older stopped run finalizes; each run retains isolated options and artifact paths.
|
||||
|
||||
Pausing transcription keeps the active recognition pipeline and meeting session alive, including Azure conversation transcription and the run's speaker context. Captured audio is replaced with equal-length silence before it reaches the temporary WAV or transcription backend, so real audio from the paused interval is discarded while provider continuity and meeting-relative timing are preserved. Normal transcript-inactivity notifications and auto-stop are suppressed during pause, while a separate four-hour maximum continuous pause prevents a forgotten paused meeting from running indefinitely. The recording status response exposes pause state as `isPaused`.
|
||||
|
||||
During an active run, microphone creation failures and disconnects are retried every second with fresh endpoint selection. The meeting and system-loopback capture stay active, with microphone silence mixed in until capture resumes. This recovery does not cover a failed system-loopback source.
|
||||
|
||||
Outlook enrichment selects an unambiguous current or imminent appointment. A scheduled prompt shown during an active recording can apply that exact appointment's title, eligible attendees, agenda, and scheduled end without interrupting capture; explicit prompt metadata wins over a slower background lookup.
|
||||
|
||||
## Data And Side Effects
|
||||
|
||||
Meeting Assistant writes durable meeting knowledge to the configured Obsidian vault:
|
||||
Meeting Assistant writes meeting notes, transcripts, assistant context, summaries, and project knowledge into the configured Obsidian vault. Assistant context is persistent meeting-specific memory: the summarizer records problems and assumptions, and the interactive agent can read it and append later repairs and conclusions.
|
||||
|
||||
- meeting notes
|
||||
- transcripts
|
||||
- assistant context notes
|
||||
- summary notes
|
||||
- project knowledge files that agents may read or update
|
||||
- optional dictation words used as speech-recognition phrase hints
|
||||
Agents are intentionally stateful. Depending on the invoked tools, they can change workflow rules and appsettings, create or update project and meeting files, change frontmatter, merge or delete speaker identities and samples, run diagnostics, and trigger transcription or summary work. Screenshot OCR can add recognized attendee names to the meeting note after workflow transformation; OCR of images already embedded in the meeting note does not add attendees or modify that note.
|
||||
|
||||
It also writes local runtime state outside the vault:
|
||||
Local runtime state outside the vault includes:
|
||||
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\Recordings`: temporary mixed WAV files during active/finalizing runs; stale files are deleted at startup and completed runs delete their temporary audio.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\SpeakerIdentity\speaker-identities.db`: local SQLite speaker identity database, aliases, meeting references, and bounded voice snippets.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\FunASR\models`: persistent FunASR model and hotword cache when the managed FunASR backend is enabled.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\Pyannote\models`: persistent pyannote/Hugging Face/torch cache when pyannote diarization or validation is enabled.
|
||||
- `%TEMP%\MeetingAssistant\Logs\meeting-assistant.log`: application-owned rotating logs; the snippets starter separately captures stdout/stderr under `%LOCALAPPDATA%`.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\Recordings`: mixed WAV files are normally deleted after completion, and unqueued stale files are deleted at startup. If an Azure stop cannot drain within `Recording:StopProcessingTimeout`, the WAV plus a JSON item under `offline-transcription-backlog` are retained and retried every minute. Both are removed only after successful replay, transcript finalization, and summary processing.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\SpeakerIdentity\speaker-identities.db`: SQLite identities, aliases, meeting references, bounded WAV snippets for the existing matcher, and separate versioned voice vectors for the optional Resemblyzer matcher.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\Resemblyzer`: content-versioned managed Python environments, the local encoder script, and temporary encoder inputs. Per-meeting WAV inputs are deleted after each encoding command.
|
||||
- `%LOCALAPPDATA%\MeetingAssistant\FunASR\models` and `%LOCALAPPDATA%\MeetingAssistant\Pyannote\models`: persistent model, hotword, Hugging Face, and torch caches for optional local backends.
|
||||
- `%TEMP%\MeetingAssistant\Logs\meeting-assistant.log`: application log with four rotated predecessors. Paths, transcript text, agent diagnostics, and provider errors can make these logs sensitive.
|
||||
|
||||
The repo intentionally ignores local rule files, local appsettings overrides, models, recordings, runtime caches, build output, and temporary publish folders.
|
||||
Abort is destructive: it removes the active run's note, transcript, context, summary, and linked screenshot attachments and skips summarization. A normal stop below `Recording:MinimumCompletedMeetingDuration`, or an otherwise content-empty stop, can also remove generated artifacts.
|
||||
|
||||
## External Data Boundaries
|
||||
|
||||
“Local” describes control and durable storage, not every processing step:
|
||||
|
||||
- The default `azure-speech` provider sends mixed meeting audio and dictation phrase hints to Azure AI Speech. Azure-backed speaker matching also sends selected voice audio.
|
||||
- Summary, screenshot OCR, and interactive-agent requests go to the configured OpenAI-compatible Responses endpoint. They can include meeting/transcript/project text, screenshots, configuration, logs, and speaker samples when corresponding tools are used. The checked-in endpoint is a loopback proxy; its ultimate provider, data path, and retention policy are outside this repository.
|
||||
- Outlook Classic access on Windows is local COM. EventKit access on macOS reads calendars synchronized into the Calendar app. Neither provides the primary capture path.
|
||||
- A managed FunASR run pulls its configured image, removes any same-named container, starts a disposable container privileged by default, publishes the configured host port, and mounts the model/hotword cache. Pyannote may build a local image and start disposable containers with audio inputs mounted read-only; those paths require Docker Desktop or a compatible Docker CLI. The opt-in Resemblyzer speaker matcher instead provisions an isolated local Python venv with CPU-only dependencies. First use may download images, Python packages, or models from external registries.
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration is normal .NET configuration under `MeetingAssistant`; the checked-in `MeetingAssistant/appsettings.json` is the canonical example. Keep secret values out of source and use the configured environment-variable names instead.
|
||||
`MeetingAssistant/appsettings.json` is the canonical configuration example. Keep secret values out of tracked JSON and local rule/config files out of source control.
|
||||
|
||||
Important settings:
|
||||
The settings with the largest operational effect are:
|
||||
|
||||
- `Vault`: controls the Obsidian vault root and the relative folders for notes, transcripts, summaries, assistant context, project knowledge, and dictation words.
|
||||
- `Recording:TranscriptionProvider`: selects `azure-speech`, `funasr`, or `whisper-local`.
|
||||
- `Recording:MicrophoneDeviceId`: optionally pins the Windows microphone endpoint; blank follows the Windows default, and the tray icon can override it for later starts.
|
||||
- `Recording:TemporaryRecordingsFolder`: controls temporary mixed WAV storage.
|
||||
- `Recording:InactivitySafeguard`: prompts and then auto-stops forgotten silent recordings without aborting artifacts.
|
||||
- `LaunchProfiles`: overlays named profile settings onto the default profile; profile hotkeys must be distinct.
|
||||
- `Automation:RulesPath`: points to the local YAML workflow-rules file. The default `meeting-rules.local.yaml` is ignored by git.
|
||||
- `CalendarRecordingPrompts`: enables Outlook Classic Teams-start prompts on Windows.
|
||||
- `Screenshots`: controls the capture hotkey, attachment folder, and optional OCR/vision model.
|
||||
- `Agent`: configures the OpenAI-compatible summary/project agent endpoint, model, retries, output limits, and compaction.
|
||||
- `WorkflowRulesEditor`: optionally overrides the agent settings for the tray-launched assistant.
|
||||
- `Vault`: selects the durable vault and artifact/project locations.
|
||||
- `Recording:TranscriptionProvider`: selects `azure-speech`, `funasr`, or `whisper-local`; the latter requires a local Whisper model file.
|
||||
- `Recording:MicrophoneDeviceId`, mix gains, stop timeout, minimum duration, and temporary folder: control capture selection, audio, cleanup, and Azure backlog behavior. Microphone-device selection is Windows-only; macOS follows the system default input device.
|
||||
- `Recording:InactivitySafeguard`: prompts and can auto-finish a run after no new transcript text; it is not an audio-silence detector.
|
||||
- `LaunchProfiles`: overlay named recording/ASR/agent settings and require distinct hotkeys.
|
||||
- `SpeakerIdentification:Resemblyzer:Enabled`: selects the local, managed-Python-venv vector matcher for the whole application; when disabled, the existing WAV/Azure path stays active. Five vectors unlock matching by default without capping retained evidence, and mature profiles use configurable fail-safe density clustering to remove likely mixed-speaker outliers.
|
||||
- `Automation:RulesPath`: points to the local YAML workflow-rules file, normally ignored `meeting-rules.local.yaml`.
|
||||
- `CalendarRecordingPrompts` and `Screenshots`: control Outlook prompts on Windows, EventKit prompts on macOS, capture, attachments, and configured OCR.
|
||||
- `Agent` and `WorkflowRulesEditor`: select the Responses endpoint/model, streaming or non-streaming transport, reasoning, retry, output, and compaction behavior; the available tools are defined by the application.
|
||||
|
||||
Required or commonly used secrets:
|
||||
|
||||
@@ -122,6 +141,8 @@ See `docs/meeting-assistant-configuration.md` for the full configuration referen
|
||||
|
||||
- **Obsidian vault**: primary durable store for notes, transcripts, summaries, assistant context, project files, and generated links.
|
||||
- **Outlook Classic on Windows**: optional COM metadata lookup and scheduled Teams-meeting start prompts.
|
||||
- **EventKit on macOS**: metadata lookup and native recording prompts for Teams events in calendars available to macOS, including Outlook-synced calendars.
|
||||
- **Native active-window capture**: Windows foreground-window capture or macOS frontmost-application window capture, feeding the same screenshot/OCR pipeline.
|
||||
- **Azure AI Speech**: default checked-in ASR path, live diarized conversation transcription, and speaker identity matching.
|
||||
- **FunASR**: optional WebSocket streaming ASR. When managed backend startup is enabled, the app pulls and runs the configured Docker image as `meeting-assistant-funasr` on port `10095`.
|
||||
- **Whisper.NET plus pyannote**: optional local Whisper fallback and Docker-backed final diarization.
|
||||
@@ -132,39 +153,29 @@ See `docs/meeting-assistant-configuration.md` for the full configuration referen
|
||||
|
||||
Meeting-specific automation lives in a local YAML file, not in committed personal rules. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, transcript-line writes, and added attendees. They can add/remove attendees, set supported properties, add context, add projects, rewrite a transcript line, and transform an attendee name before it is stored.
|
||||
|
||||
The tray menu exposes `Open agent`, which opens the `Meeting Summary Agent` window. It can edit workflow rules with validation, inspect logs and health/status, manage speaker identities and samples, run ASR diagnostics, and read/write scoped meeting/project artifacts through explicit tools.
|
||||
The Windows tray and macOS menu-bar menus expose `Open agent`, which opens the `Meeting Summary Agent` window. It can edit workflow rules with validation, inspect logs and health/status, manage speaker identities and samples, run ASR diagnostics, and read/write scoped meeting/project artifacts through explicit tools. Windows renders the chat with WPF; macOS renders the same agent pipeline in a native AppKit window backed by WebKit.
|
||||
|
||||
At startup, Meeting Assistant detects whether the host is Windows or macOS and includes that immutable runtime context in the interactive settings/logs assistant instructions. On Windows the assistant uses Windows commands and concepts such as PowerShell, Windows paths, services, and Task Manager. On macOS it uses zsh, POSIX paths, launchd/LaunchAgents, and Activity Monitor. This guidance is also appended when a custom interactive-agent prompt is configured. macOS audio capture, menu-bar controls, global hotkeys, EventKit calendar enrichment/prompts, active-window screenshots, screenshot OCR, speaker identification, FunASR, local diarization, and the AppKit/WebKit workflow editor are available in the portable build. Outlook Classic COM and Windows toast notifications remain Windows-specific implementations.
|
||||
|
||||
Detailed workflow syntax and extension guidance live in `docs/meeting-workflow-engine.md`.
|
||||
|
||||
## Development And CI
|
||||
|
||||
The repo builds against .NET 10 and targets both `net10.0` and `net10.0-windows10.0.19041.0` for the app. Tests target `net10.0`.
|
||||
Behavior changes are OpenSpec-driven and test-first: update the relevant requirement/scenario, add a failing public behavior test, implement the smallest passing change, run focused tests and then the justified broader suite, and validate the active change with `openspec validate <change-id> --strict`. Documentation-only maintenance does not need a new OpenSpec change.
|
||||
|
||||
The Gitea workflow `.gitea/workflows/pr-push-build-and-test.yaml` runs on pull requests, pushes, and manual dispatch. It restores/builds the Windows target on an Ubuntu runner, installs Wine, downloads a matching Windows .NET SDK, and runs the test project through the Windows dotnet host under Wine.
|
||||
The Gitea workflow runs for pull requests, pushes, and manual dispatch. It builds the Windows target on an Ubuntu runner, installs Wine plus a matching Windows .NET SDK, and runs the test project through the Windows host under Wine. It validates source; it does not publish or deploy the workstation application.
|
||||
|
||||
Before behavior changes:
|
||||
## Operations And Limitations
|
||||
|
||||
1. Read `AGENTS.md`, this README, `openspec/config.yaml`, and relevant specs under `openspec/specs`.
|
||||
2. Add or update the OpenSpec requirement/scenario for the behavior.
|
||||
3. Add a failing behavior test through the public surface.
|
||||
4. Implement the smallest passing change.
|
||||
5. Run the narrowest useful tests, then broader tests when the blast radius justifies it.
|
||||
6. Run `openspec validate <change-id> --strict` for active spec changes.
|
||||
|
||||
Documentation-only README maintenance does not need a new OpenSpec change.
|
||||
|
||||
## Operational Notes
|
||||
|
||||
- Treat the app as live user work. Check `/recording/status` before restarting, killing processes, deleting runtime files, or running scripts that might take over port `5090`.
|
||||
- Abort is destructive for the active run: it removes that meeting's note, transcript, assistant context, summary if present, and linked screenshot attachments, and it skips summary generation.
|
||||
- Too-short or empty normal stops can also delete generated artifacts instead of producing summaries.
|
||||
- Summary and screenshot OCR retry links use `Api:PublicBaseUrl`, defaulting to `http://localhost:5090`.
|
||||
- The workflow reload endpoint reloads configuration for future workflow reads, but a meeting run that already captured options may continue with those captured options.
|
||||
- Public hostnames and homelab ingress are intentionally out of scope for this repo.
|
||||
- Treat recording, transcription drain, speaker finalization, OCR, and summarization as live user work. Never restart, kill, or clean runtime files until `/recording/status` is idle unless interruption is explicitly intended.
|
||||
- Summary and screenshot retry links use `Api:PublicBaseUrl`, which defaults to `http://localhost:5090`.
|
||||
- Workstation startup is external to this repository. On `NA-EXC765X84` as of 2026-08-11, the installed Windows Startup shortcut invokes the restart helper at sign-in, so it republishes and replaces the process; it differs from the start-only shortcut tracked in `Manuel/snippets`, and the reason for that local deviation is not documented.
|
||||
- No public hostname, homelab ingress, always-on deployment, or remote-service availability contract is owned here.
|
||||
|
||||
## More Documentation
|
||||
|
||||
- `docs/meeting-assistant-configuration.md`: full configuration reference.
|
||||
- `docs/meeting-workflow-engine.md`: workflow rules engine reference.
|
||||
- `docs/meeting-assistant-configuration.md`: complete configuration and backend behavior.
|
||||
- `docs/meeting-workflow-engine.md`: workflow triggers, conditions, templates, steps, and editor behavior.
|
||||
- `openspec/specs`: accepted behavioral requirements.
|
||||
- `openspec/changes/archive`: archived change proposals and designs behind the accepted specs.
|
||||
- `openspec/changes`: active change specs and designs, including implemented work not yet archived.
|
||||
- `openspec/changes/archive`: historical proposals and decisions behind accepted behavior.
|
||||
|
||||
Binary file not shown.
@@ -38,6 +38,7 @@ This example is abbreviated so the most common shape is readable. The checked-in
|
||||
"FirstPromptAfter": "00:02:00",
|
||||
"ReminderPromptAfter": [ "00:05:00", "00:10:00" ],
|
||||
"AutoStopAfter": "00:30:00",
|
||||
"MaximumPauseDuration": "04:00:00",
|
||||
"InferredEndPadding": "00:01:00",
|
||||
"CheckInterval": "00:00:15"
|
||||
}
|
||||
@@ -131,20 +132,25 @@ The default profile is always named `default`. Non-default profile hotkeys are r
|
||||
|
||||
During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription.
|
||||
|
||||
On macOS, the portable target captures the default microphone through AVFoundation and computer output through ScreenCaptureKit. The build compiles the Swift audio and desktop-integration helpers into the application output and publish `Native` folder; build or publish on macOS with Xcode Command Line Tools installed. The first capture requires both **Microphone** and **Screen & System Audio Recording** permissions under System Settings > Privacy & Security. Restart the application after granting a newly requested permission. `Recording:MicrophoneDeviceId` and runtime microphone selection remain Windows-only; macOS follows the system default input device.
|
||||
|
||||
The tray's fine-grained controls expose `Pause transcription` while a meeting is active and `Unpause transcription` while it is paused. Pause does not stop audio devices, finish the meeting, replace the speech-recognition pipeline, or reset run-local speaker mappings and collected samples. Instead, each captured mixed chunk is replaced with equal-length PCM silence before it reaches the temporary WAV, live speaker buffer, or configured transcription provider. This discards real audio from the paused interval while preserving the provider session and meeting-relative timing. In particular, Azure keeps the same active `ConversationTranscriber` and push stream; pause therefore does not suspend Azure connection time or billing. `Finish meeting` and cancel/discard remain available while paused, and `/recording/status` reports the state in `isPaused`.
|
||||
|
||||
On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits.
|
||||
|
||||
`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV.
|
||||
|
||||
`Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings.
|
||||
`Recording:MaxMetadataAttendeeImportCount` limits how many attendees calendar metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings. Windows reads Outlook Classic through COM. macOS reads EventKit calendars, including Outlook accounts synchronized into macOS Calendar, and requires **Calendar Full Access**.
|
||||
|
||||
`Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text arrives. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with action buttons, requests reminder-style toast behavior, keeps each stop reminder actionable for 1 minute, and automatically stops normally after 30 minutes. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When the safeguard stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding.
|
||||
`Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text is written. Writing new transcript text also dismisses every outstanding inactivity notification and invalidates its actions. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with Yes, No, and `Pause transcription` actions, requests reminder-style toast behavior, keeps each stop reminder actionable for 1 minute, and automatically stops normally after 30 minutes. While transcription is intentionally paused, those prompts and the ordinary transcript-inactivity stop are fully suspended. A separate `MaximumPauseDuration`, defaulting to 4 hours, normally stops a meeting that remains continuously paused without showing an inactivity notification; it remains active when the ordinary inactivity safeguard is disabled, while a non-positive value disables the paused-session cutoff. Unpausing clears the continuous-pause timer and restarts transcript-inactivity timing from that moment. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When transcript inactivity stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding.
|
||||
|
||||
| Setting | Purpose |
|
||||
| --- | --- |
|
||||
| `Enabled` | Enables the inactivity safeguard for active recordings. |
|
||||
| `Enabled` | Enables transcript-inactivity prompts and `AutoStopAfter`; `MaximumPauseDuration` remains independent. |
|
||||
| `FirstPromptAfter` | First transcript-inactivity duration before showing the stop prompt. |
|
||||
| `ReminderPromptAfter` | Additional transcript-inactivity durations before showing another stop prompt. |
|
||||
| `AutoStopAfter` | Transcript-inactivity duration after which Meeting Assistant stops the recording normally without prompting again. |
|
||||
| `MaximumPauseDuration` | Maximum continuous transcription pause before Meeting Assistant stops the meeting normally without an inactivity notification; defaults to 4 hours, and a non-positive value disables it. |
|
||||
| `InferredEndPadding` | Padding added to the last transcript timestamp, or meeting start when no transcript arrived, for safeguard-triggered end times. |
|
||||
| `CheckInterval` | Polling interval for checking the active recording inactivity state. |
|
||||
|
||||
@@ -219,11 +225,11 @@ When `WhisperLocal:Diarization:Enabled` is true, the final post-processing pass
|
||||
|
||||
Active pyannote runtimes are warmed up on application start so image setup and model download do not wait for the first diarization request.
|
||||
|
||||
Pyannote diarization settings are shared by local Whisper finalization and speaker-identification validation:
|
||||
Pyannote runtime settings are shared by local Whisper finalization and speaker-identification validation. `WhisperLocal:Diarization:Enabled` controls the optional Whisper finalization pass; speaker-identification validation instead uses its single outer `SpeakerIdentification:PyannoteValidation:Enabled` switch.
|
||||
|
||||
| Setting | Purpose |
|
||||
| --- | --- |
|
||||
| `Enabled` | Enables the pyannote-backed pass. |
|
||||
| `Enabled` | Available under `WhisperLocal:Diarization` to enable the pyannote-backed Whisper finalization pass. It is not part of the nested speaker-validation runtime settings. |
|
||||
| `DockerCommand` | Docker executable name or path. |
|
||||
| `BaseImage` | Python base image used when building the local pyannote image. |
|
||||
| `Image` | Local pyannote Docker image tag. |
|
||||
@@ -264,7 +270,9 @@ Azure returns generic speaker IDs such as `Guest-1` and `Guest-2`, which Meeting
|
||||
|
||||
## Speaker Identification
|
||||
|
||||
Speaker identity matching keeps candidate samples only after a diarized speaker has at least `SpeakerIdentification:MinimumSampleSpeechDuration` of continuous speech, defaulting to 30 seconds. Adjacent same-speaker segments may be combined when the gap is no larger than `MaximumSampleSegmentGap`, but a different speaker resets the pending span.
|
||||
Speaker identity matching keeps candidate samples only after a diarized speaker has at least `SpeakerIdentification:MinimumSampleSpeechDuration` of same-speaker audio, defaulting to 10 seconds. Consecutive lines with the same diarized speaker are combined even when any configured STT backend splits them around pauses. This applies to both live and final-only diarization. A different speaker ends the pending span, and `MaximumSampleDuration` caps every recognition WAV at 60 seconds by default.
|
||||
|
||||
Configuration is rejected when the minimum duration is negative, the maximum is not positive, or the minimum exceeds the maximum. The same validation applies to launch-profile overrides.
|
||||
|
||||
| Setting | Purpose |
|
||||
| --- | --- |
|
||||
@@ -272,12 +280,12 @@ Speaker identity matching keeps candidate samples only after a diarized speaker
|
||||
| `DatabasePath` | SQLite database path for identities, aliases, references, and samples. |
|
||||
| `InitialDelay` | Delay after recording starts before the first live identity pass. |
|
||||
| `Interval` | Interval between live identity passes. |
|
||||
| `MatchBatchSize` | Number of identities processed per model matching batch. |
|
||||
| `MatchBatchSize` | Number of identities processed per Azure model matching batch. Resemblyzer scores the full capped candidate set together so its ambiguity margin includes the global runner-up. |
|
||||
| `MaxMatchCandidates` | Maximum known identities considered during one match pass. |
|
||||
| `MatchIdentityActiveAge` | Age window for identities considered active enough for automatic matching. |
|
||||
| `MaxSnippetsPerSpeaker` | Maximum stored voice snippets retained per speaker identity. |
|
||||
| `MinimumSampleSpeechDuration` | Minimum uninterrupted same-speaker speech span needed before storing a sample. |
|
||||
| `MaximumSampleSegmentGap` | Maximum gap allowed when combining adjacent same-speaker segments into one sample. |
|
||||
| `MaxSnippetsPerSpeaker` | Maximum stored WAV snippets retained per speaker identity by the existing backend. |
|
||||
| `MinimumSampleSpeechDuration` | Minimum total diarized speaker audio needed in a same-speaker sample; provider-created pauses do not count toward it. |
|
||||
| `MaximumSampleDuration` | Maximum duration of an extracted speaker-recognition WAV. Defaults to 60 seconds. |
|
||||
| `SilenceBetweenSnippetsSeconds` | Silence padding inserted between snippets during Azure identity matching. |
|
||||
| `LiveSampleBufferDuration` | How long live transcript/audio material is retained for extracting samples. |
|
||||
| `MergeRecentIdentityAge` | Age window used by diagnostics that merge recent duplicate identities. |
|
||||
@@ -285,14 +293,46 @@ Speaker identity matching keeps candidate samples only after a diarized speaker
|
||||
|
||||
`SpeakerIdentification:AzureSpeech` is an advanced nested override for speaker identity matching. It uses the same shape as `AzureSpeech` and lets identity matching use different Azure language, endpoint, or key settings than live transcription when needed. If it is left unset, the normal Azure Speech settings remain the practical default.
|
||||
|
||||
`SpeakerIdentification:PyannoteValidation` is an optional secondary confidence layer. When enabled, pyannote rejects multi-speaker samples and must confirm an Azure-confirmed identity match before Meeting Assistant accepts it. It uses the same Docker-based pyannote runtime shape as local Whisper finalization and defaults the validation command timeout to 1 hour because local model setup can take substantial time.
|
||||
`SpeakerIdentification:PyannoteValidation` is an optional application-level secondary confidence layer and defaults to disabled. Its outer `Enabled` setting is the only validation toggle; the nested `Diarization` block contains runtime settings but no second enable switch. Launch profiles do not override speaker validation or its runtime settings. When enabled, pyannote rejects multi-speaker samples and must confirm an Azure-confirmed identity match before Meeting Assistant accepts it. It uses the same Docker-based pyannote runtime shape as local Whisper finalization and defaults the validation command timeout to 1 hour because local model setup can take substantial time.
|
||||
|
||||
| Setting | Purpose |
|
||||
| --- | --- |
|
||||
| `Enabled` | Sole switch for speaker-identity pyannote validation and its startup warm-up. |
|
||||
| `MinimumSingleSpeakerCoverage` | Required fraction of the tested sample that pyannote must attribute to a single speaker. |
|
||||
| `MinimumMatchingKnownSnippetRatio` | Required pyannote agreement ratio between the new sample and known snippets for an accepted identity. |
|
||||
| `Diarization` | Nested pyannote settings used for this validation pass. |
|
||||
|
||||
`SpeakerIdentification:Resemblyzer` is a separate, application-level speaker-recognition backend and defaults to disabled. Setting its single `Enabled` flag to `true` selects the Resemblyzer identification and merge services for the process lifetime; Azure Speech and pyannote are then not used for identity matching or identity-match validation. Launch profiles do not override this choice. The normal `SpeakerIdentification:Enabled` setting remains the master switch for speaker identification as a whole.
|
||||
|
||||
The selected service reuses temporary WAV samples from diarized same-speaker runs, but starts a fresh span after every accepted sample so one speaker's vector inputs do not overlap. It waits for five samples by default before automatic matching; five is an eligibility threshold, not a storage cap. Additional samples for an unresolved speaker trigger another attempt at the configured interval, even when attendees and speaker labels have not changed. All distinct qualifying vectors from the run are retained up to the per-identity limit, including later evidence collected after a live match has already renamed the transcript. Providers that only diarize during finalization can extract missing non-overlapping samples from the completed mixed WAV. Resemblyzer runs locally in an application-managed Python virtual environment with CPU-only PyTorch, produces 256-value embeddings, and stores only versioned float32 vectors in the identity database; it does not persist the temporary WAV inputs as identity evidence. Explicit speaker assignments from the summary agent may save fewer than five available vectors.
|
||||
|
||||
For a query, the matcher first requires the mean pairwise cosine similarity of its vectors to meet `MinimumClusterCohesion`. It represents each compatible known identity by a normalized centroid, scores it with the median query-to-centroid cosine similarity, and accepts only if the best score meets `MinimumIdentitySimilarity` and beats the runner-up by `MinimumSimilarityMargin`. The runner-up margin is waived when only one identity is scoreable. Logs include measured scores and rejection reasons so these initial thresholds can be calibrated.
|
||||
|
||||
Once an identity has at least `OutlierPruningMinimumVectors` compatible vectors, a separate DBSCAN-style cosine-density pass protects the profile from mixed-speaker diarization errors. A vector is a neighbor when its cosine similarity meets `OutlierPruningNeighborSimilarity`, and a dense point requires `OutlierPruningMinimumNeighbors` neighbors including itself. Vectors outside the uniquely largest dense cluster are removed only when that cluster contains at least `OutlierPruningMinimumClusterRatio` of all compatible vectors. If there is no dense cluster, the largest clusters tie, or the ratio is too low, pruning fails safe and retains every vector. Invalid vectors and vectors produced by other model versions are preserved.
|
||||
|
||||
| Resemblyzer setting | Purpose |
|
||||
| --- | --- |
|
||||
| `Enabled` | Selects the complete local Resemblyzer identity backend. Defaults to `false`. |
|
||||
| `RequiredVectorsPerSpeaker` | Independent vectors required for automatic matching and per-cluster merge validation. Defaults to `5`; this does not cap collection or persistence. |
|
||||
| `MaxVectorsPerIdentity` | Maximum distinct vectors retained per identity. Defaults to `1000`; merges prefer the newest vectors. |
|
||||
| `OutlierPruningMinimumVectors` | Compatible vectors required before density-cluster pruning runs. Defaults to `20`. |
|
||||
| `OutlierPruningNeighborSimilarity` | Minimum cosine similarity for two vectors to count as density neighbors. Defaults to `0.75`. |
|
||||
| `OutlierPruningMinimumNeighbors` | Neighbors required for a dense point, including the point itself. Defaults to `3`. |
|
||||
| `OutlierPruningMinimumClusterRatio` | Minimum fraction of compatible vectors that the uniquely largest cluster must contain before other vectors are removed. Defaults to `0.60`. |
|
||||
| `MinimumClusterCohesion` | Minimum mean pairwise cosine similarity within a query cluster. |
|
||||
| `MinimumIdentitySimilarity` | Minimum median similarity from query vectors to a known identity centroid. |
|
||||
| `MinimumSimilarityMargin` | Required difference between the best and runner-up identity scores. |
|
||||
| `ModelId` | Version identifier persisted with vectors and required for compatible comparisons. |
|
||||
| `PackageVersion` | Resemblyzer package version installed in the managed virtual environment. |
|
||||
| `PythonCommand` | Python executable used to create the managed virtual environment. |
|
||||
| `TorchVersion` | CPU-only PyTorch wheel version installed in the managed environment. |
|
||||
| `TorchIndexUrl` | HTTPS package index used for the CPU-only PyTorch wheel. |
|
||||
| `WebRtcVadVersion` | Version of the prebuilt Windows-compatible `webrtcvad-wheels` package. |
|
||||
| `RuntimeFolder` | Local folder for versioned virtual environments, the encoder script, and short-lived WAV input batches. |
|
||||
| `CommandTimeout` | Bound for first-time environment provisioning, warm-up, and encoder commands. |
|
||||
|
||||
The first enabled startup creates a content-versioned virtual environment under `RuntimeFolder`. Dependency-version changes select a new environment automatically. The workstation's global Python packages are not modified, and Docker is not required for Resemblyzer speaker recognition.
|
||||
|
||||
## Automation
|
||||
|
||||
`Automation:RulesPath` points to an optional local YAML rules file. The default `meeting-rules.local.yaml` is ignored by git. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, or transcript line writes; conditions are evaluated with NCalc-style expressions and step values can use Razor syntax against `Model.Meeting`, `Model.Event`, `Model.Speaker`, and `Model.Transcript`.
|
||||
@@ -303,20 +343,22 @@ See `docs/meeting-workflow-engine.md` for the detailed YAML format, supported va
|
||||
|
||||
## Calendar Recording Prompts
|
||||
|
||||
`CalendarRecordingPrompts` controls optional Outlook Classic calendar prompts for starting recordings. It is enabled by default on Windows.
|
||||
`CalendarRecordingPrompts` controls optional calendar prompts for starting recordings. It is enabled by default on Windows and macOS.
|
||||
|
||||
When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Classic calendar appointments through COM, filters all non-canceled Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. It does not query Outlook for every prompt. The notification asks `Record meeting?` with `Yes` and `No` actions when a cached Teams meeting reaches its start window, requests reminder-style toast behavior, and remains actionable for 5 minutes. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply.
|
||||
Meeting Assistant periodically syncs today's calendar appointments, filters all non-canceled Teams meeting markers for the day into an in-memory cache, and schedules prompt checks from that cache. Windows reads Outlook Classic through COM and displays a Windows app notification. macOS prefers EventKit and displays a native AppKit `Record`/`Skip` alert; the first read requests Calendar Full Access. When EventKit access is denied, it falls back to Calendar application automation so an already-authorized local Calendar setup remains usable. It does not query the calendar for every prompt. Accepting starts a recording. If another recording is active, Meeting Assistant stops that recording through the normal stop path first, then starts the new recording, so the usual empty/too-short cleanup and summary handoff rules still apply.
|
||||
|
||||
| Setting | Purpose |
|
||||
| --- | --- |
|
||||
| `Enabled` | Enables scheduled Outlook Teams recording-start prompts. |
|
||||
| `SyncInterval` | How often Meeting Assistant refreshes today's Outlook calendar cache. Defaults to 30 minutes when unset or invalid. |
|
||||
| `Enabled` | Enables scheduled Teams recording-start prompts from the platform calendar. |
|
||||
| `SyncInterval` | How often Meeting Assistant refreshes today's calendar cache. Defaults to 30 minutes when unset or invalid. |
|
||||
| `PromptWindow` | How long after a Teams appointment's scheduled start it remains eligible for the one-time prompt. Defaults to 5 minutes when unset or invalid. |
|
||||
|
||||
## Screenshots
|
||||
|
||||
`Screenshots:Hotkey` configures a global hotkey that captures the currently active window during an active meeting. Screenshots are written under `Screenshots:AttachmentsFolder`, which defaults to an `Attachments` folder beside the assistant context note, and each capture appends a timestamped markdown image link to the assistant context.
|
||||
|
||||
Windows captures the foreground window through the existing Win32 provider. macOS captures the foremost on-screen window belonging to the frontmost application and requires **Screen & System Audio Recording** permission. Both paths feed the same attachment, OCR, crop, attendee-enrichment, retry, and summary-wait pipeline.
|
||||
|
||||
`Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Screenshot OCR also inherits `Agent:UseStreaming`, using the Responses SSE transport when it is `true` and the non-streaming Responses transport when it is `false`. Automatic summarization scans the meeting note for user-added Obsidian image embeds such as `![[whiteboard.png]]` and Markdown image embeds such as ``, adds resolvable local images to the assistant context without copying them or changing the meeting note, runs OCR without crop or attendee updates, and waits for all pending OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. Failed or timed-out screenshot OCR writes a retry link that targets `/meetings/screenshot-ocr/retry` with the exact saved screenshot and OCR block id.
|
||||
|
||||
| Setting | Purpose |
|
||||
@@ -380,7 +422,8 @@ The top-level `Logging` section is standard ASP.NET Core logging configuration a
|
||||
Independently from stdout and stderr redirection, Meeting Assistant writes an application-owned log under:
|
||||
|
||||
```text
|
||||
%TEMP%\MeetingAssistant\Logs\meeting-assistant.log
|
||||
Windows: %TEMP%\MeetingAssistant\Logs\meeting-assistant.log
|
||||
macOS: $TMPDIR/MeetingAssistant/Logs/meeting-assistant.log
|
||||
```
|
||||
|
||||
On startup it rotates the previous current file to `meeting-assistant.log.1` and keeps up to `.4`. The tray-launched assistant reads and searches these current and rotated files. If another app instance or test host still has the file open, rotation is skipped and logging appends to the current file so startup is not blocked.
|
||||
@@ -388,3 +431,5 @@ On startup it rotates the previous current file to `meeting-assistant.log.1` and
|
||||
## API
|
||||
|
||||
`Api:PublicBaseUrl` is used to build clickable local links, such as summary retry links in generated failure notes.
|
||||
|
||||
`GET /diagnostics/platform-capabilities` reports the detected host, concrete metadata/calendar/prompt/screenshot/workflow-editor provider names, explicit workflow-editor availability, and effective enabled switches for calendar prompts, speaker identification, screenshot OCR, the FunASR backend, FunASR diarization, and Whisper-local diarization. It is read-only and is the quickest way to confirm that a platform is not using no-op providers.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Meeting Workflow Engine
|
||||
|
||||
The interactive rules and identities editor can queue speaker samples for playback in the Windows build. The neutral build reports that playback requires the Windows build; it does not enqueue audio or initialize Windows audio devices.
|
||||
|
||||
Meeting Assistant has a small local workflow engine for meeting-specific automation. It is intended for rules that are too personal or environment-specific to hard-code, such as adding default attendees, cleaning meeting titles, binding projects, or adding context notes when known speakers are detected.
|
||||
|
||||
The workflow engine is deliberately narrow. It runs from a local YAML file, evaluates rules against the current meeting note read from disk, and applies a small set of meeting-safe mutations.
|
||||
@@ -24,6 +26,8 @@ If the configured path is empty, missing, or points to a blank file, the workflo
|
||||
|
||||
The tray menu includes `Open agent`, which opens the `Meeting Summary Agent` chat window for this configured rules file, the local speaker identity database, appsettings configuration, and application logs. The assistant uses the summarizer agent configuration by default and can be overridden through `MeetingAssistant:WorkflowRulesEditor`.
|
||||
|
||||
Meeting Assistant detects the host operating system once during application startup and appends that context to the interactive agent prompt. Windows hosts receive PowerShell, Windows-path, Windows-service, and Task Manager guidance. macOS hosts receive zsh, POSIX-path, launchd/LaunchAgent, and Activity Monitor guidance. Other hosts receive neutral portable guidance. The platform context remains present when `WorkflowRulesEditor:InitialPrompt` is configured, and it does not imply that unavailable platform integrations have been implemented.
|
||||
|
||||
```json
|
||||
{
|
||||
"MeetingAssistant": {
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"sdk": {
|
||||
"version": "10.0.100",
|
||||
"rollForward": "latestFeature"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
## Context
|
||||
|
||||
The application already exposes local HTTP endpoints for recording lifecycle actions. Windows desktop integrations invoke the coordinator in-process, while macOS currently has no desktop process capable of participating in AppKit or Carbon event loops.
|
||||
|
||||
## Decisions
|
||||
|
||||
### Native helper owns macOS desktop APIs
|
||||
|
||||
A bundled Swift helper owns the AppKit status item and Carbon global-hotkey registrations. The .NET host starts the helper only on macOS and terminates it during shutdown.
|
||||
|
||||
### Existing HTTP endpoints remain the control seam
|
||||
|
||||
The helper invokes loopback endpoints for recording and diagnostic actions. It does not duplicate recording state or lifecycle policy. It polls the status endpoint to keep the menu-bar presentation current.
|
||||
|
||||
### Configuration is passed as an explicit manifest
|
||||
|
||||
The .NET host derives a manifest from configured launch profiles. Shortcut strings are passed unchanged so macOS uses the same bindings displayed and registered on Windows.
|
||||
|
||||
## Verification
|
||||
|
||||
- Behavior tests prove the manifest preserves configured shortcuts and maps actions to the existing endpoints.
|
||||
- The Swift helper supports a describe/self-test mode for deterministic packaging validation.
|
||||
- Runtime verification launches the application, starts a new recording, and captures the visible macOS menu-bar recording state.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Implementation Evidence
|
||||
|
||||
## Automated
|
||||
|
||||
- `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --filter FullyQualifiedName~MacOsDesktopControlManifestTests --no-restore`
|
||||
- Passed: 1, Failed: 0.
|
||||
- `dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 --no-restore`
|
||||
- Passed and produced `Native/macos-desktop-controls` as an arm64 Mach-O executable.
|
||||
- `openspec validate add-macos-desktop-controls --strict`
|
||||
- Passed.
|
||||
- Full portable test project:
|
||||
- Passed: 427.
|
||||
- Failed: 9 existing macOS/environment-dependent tests: Windows path expectations, unavailable GDI+, and user-environment key resolution.
|
||||
- Whole solution:
|
||||
- Cannot build the Windows target on macOS without enabling cross-Windows targeting (`NETSDK1100`).
|
||||
|
||||
## Runtime
|
||||
|
||||
- Installed the rebuilt application assembly and both native helpers into the existing local LaunchAgent application directory.
|
||||
- LaunchAgent restarted successfully.
|
||||
- `/health` returned `ok`.
|
||||
- `macos-desktop-controls` ran as a child companion process.
|
||||
- The helper manifest contained six configured hotkeys, including the unchanged defaults `Ctrl+Alt+M`, `Ctrl+Alt+L`, `Ctrl+Alt+Z`, and `Ctrl+Alt+S`.
|
||||
- A fresh default-profile recording request created the `20260724-2015` meeting note and linked artifacts.
|
||||
- Screenshot evidence: `tmp/macos-desktop-recording-verification.png`.
|
||||
|
||||
## Live desktop-control proof (2026-07-24)
|
||||
|
||||
- The installed LaunchAgent ran:
|
||||
- `.NET host` PID `23023`
|
||||
- native `macos-desktop-controls` PID `23097`
|
||||
- The native helper manifest exposed:
|
||||
- `Ctrl+Alt+M` -> `/profiles/default/recording/toggle`
|
||||
- `Ctrl+Alt+L` -> `/profiles/english/recording/toggle`
|
||||
- `Ctrl+Alt+Z` -> `/profiles/default/recording/abort`
|
||||
- `Ctrl+Alt+S` -> `/profiles/default/meetings/screenshot/capture`
|
||||
- Accessibility inspection of the real AppKit menu returned:
|
||||
- `Meeting Assistant — idle`
|
||||
- `Open agent`
|
||||
- `Start meeting recording (default)`
|
||||
- `Start meeting recording (english)`
|
||||
- `Exit`
|
||||
- Visible menu screenshot:
|
||||
- `tmp/proof-macos-menu-idle.png`
|
||||
- The menu visibly displays the macOS equivalents `⌃⌥M` and `⌃⌥L`.
|
||||
- Clicking `Start meeting recording (default)` through the real menu produced the sampled status transition `false,true,false`, with `launchProfile: default`, and created:
|
||||
- `Meetings/Notes/20260724-2022-note.md`
|
||||
- `Meetings/Transcripts/20260724-2022-transcript.md`
|
||||
- `Meetings/Assistant Context/20260724-2022-context.md`
|
||||
- Posting the HID-level `Ctrl+Alt+M` combination produced the sampled status transition `false,true,false`, with `launchProfile: default`.
|
||||
- Posting the HID-level `Ctrl+Alt+L` combination produced an active sampled state with `launchProfile: english`.
|
||||
- Raw sampled status evidence:
|
||||
- `tmp/proof-menu-recording-status.jsonl`
|
||||
- `tmp/proof-hotkey-recording-status.jsonl`
|
||||
- `tmp/proof-english-hotkey-status.jsonl`
|
||||
|
||||
## Runtime limitation
|
||||
|
||||
The new recording did not remain active because the local audio capture process stopped shortly after artifact creation. The desktop-control helper, menu-bar item, endpoint routing, and meeting creation were verified; sustained audio capture was not.
|
||||
|
||||
## Open agent regression and runtime proof
|
||||
|
||||
- Original deterministic regression:
|
||||
- `MacOsApplicationRegistersARealInteractiveAgentWindow`
|
||||
- Red result: portable registration resolved exactly `NoopWorkflowRulesEditorWindowService`.
|
||||
- Fix:
|
||||
- macOS now registers `MacOsWorkflowRulesEditorWindowService`.
|
||||
- `Open agent` in the native helper creates an AppKit `NSWindow` titled `Meeting Summary Agent`.
|
||||
- The window hosts a WebKit chat page backed by the existing `WorkflowRulesEditorChatViewModel` and agent pipeline.
|
||||
- Green regression:
|
||||
- Passed: 1, Failed: 0.
|
||||
- Live installed runtime:
|
||||
- Clicking the actual menu item created an on-screen CoreGraphics window:
|
||||
- owner: `macos-desktop-controls`
|
||||
- title: `Meeting Summary Agent`
|
||||
- layer: `0`
|
||||
- bounds: `756x714`
|
||||
- `GET /diagnostics/settings-and-logs` returned the interactive agent HTML page.
|
||||
|
||||
## Open agent prompt execution proof (2026-07-25)
|
||||
|
||||
- Original live failure:
|
||||
- `Meeting Summary Agent failed: No workflow rules editor API key configured.`
|
||||
- Root cause:
|
||||
- The LaunchAgent did not inherit the interactive shell's model API key.
|
||||
- Its configured `127.0.0.1:4021` model endpoint had no running service.
|
||||
- Runtime configuration:
|
||||
- The protected LaunchAgent environment file now supplies the local LiteLLM gateway key.
|
||||
- `MeetingAssistant__WorkflowRulesEditor__Endpoint` targets the running local gateway on port `4000`.
|
||||
- The local gateway exposes the Responses-compatible `chatgpt-gpt-5.5` model alias.
|
||||
- Event-stream compatibility regression:
|
||||
- `ClientReconstructsResponseFromServerSentEvents` failed before the fix with `JsonReaderException`.
|
||||
- The client now reconstructs completed response items from LiteLLM server-sent events.
|
||||
- All 11 `LiteLlmResponsesChatClientTests` pass.
|
||||
- Live installed proof:
|
||||
- `POST /diagnostics/settings-and-logs/chat` with `Reply with exactly: key-ok`
|
||||
returned `{"role":"agent","content":"key-ok"}`.
|
||||
- `/health` returned successfully after deployment.
|
||||
@@ -0,0 +1,24 @@
|
||||
## Why
|
||||
|
||||
Meeting Assistant can record audio on macOS, but the portable build does not register the configured global shortcuts or provide the menu-bar controls used for the normal Windows desktop workflow. macOS users must call HTTP endpoints manually.
|
||||
|
||||
## What Changes
|
||||
|
||||
- Add a native macOS menu-bar companion that starts and stops with Meeting Assistant.
|
||||
- Register the same configured recording, abort, and screenshot shortcuts used on Windows.
|
||||
- Expose recording/profile controls, `Open agent`, and exit from the macOS menu bar.
|
||||
- Keep recording lifecycle behavior behind the existing local HTTP/service surface.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### Modified Capabilities
|
||||
|
||||
- `meeting-recording`: configured global recording controls work on macOS.
|
||||
- `meeting-session`: the desktop control menu is available from the macOS menu bar.
|
||||
|
||||
## Impact
|
||||
|
||||
- `MeetingAssistant/Native/MacOsDesktopControls`
|
||||
- portable macOS build/publish output
|
||||
- startup service registration
|
||||
- desktop-control behavior tests and macOS runtime documentation
|
||||
@@ -0,0 +1,20 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Recording mode is controlled by a configurable hotkey
|
||||
Meeting Assistant SHALL use normal .NET configuration to define global hotkeys that work on Windows and macOS and toggle recording/transcription mode.
|
||||
|
||||
Meeting Assistant SHALL preserve the configured shortcut strings across Windows and macOS, including the defaults `Ctrl+Alt+M`, `Ctrl+Alt+L`, `Ctrl+Alt+Z`, and `Ctrl+Alt+S`.
|
||||
|
||||
Meeting Assistant SHALL expose a configurable abort/discard hotkey for an active recording on Windows and macOS.
|
||||
|
||||
#### Scenario: macOS uses the configured Windows-equivalent shortcuts
|
||||
- **GIVEN** the default and English profiles use `Ctrl+Alt+M` and `Ctrl+Alt+L`
|
||||
- **AND** abort and screenshot use `Ctrl+Alt+Z` and `Ctrl+Alt+S`
|
||||
- **WHEN** Meeting Assistant starts on macOS
|
||||
- **THEN** it registers those same global shortcut combinations
|
||||
- **AND** routes them to toggle, abort, and screenshot actions through the local service surface
|
||||
|
||||
#### Scenario: macOS recording hotkey starts recording
|
||||
- **GIVEN** Meeting Assistant is idle on macOS
|
||||
- **WHEN** the user presses the configured recording hotkey
|
||||
- **THEN** Meeting Assistant starts recording/transcription mode
|
||||
@@ -0,0 +1,28 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Workflow rules and speaker identities can be edited through a tray-launched assistant
|
||||
Meeting Assistant SHALL expose desktop controls from the Windows tray icon and the macOS menu bar.
|
||||
|
||||
The macOS menu-bar control SHALL start with Meeting Assistant, show current recording state, expose each configured launch profile with its configured hotkey, expose stop and abort controls while recording, expose `Open agent`, and expose exit.
|
||||
|
||||
Selecting `Open agent` on macOS SHALL open a native window titled `Meeting Summary Agent` containing the interactive settings-and-logs chat surface.
|
||||
|
||||
#### Scenario: macOS menu bar shows idle recording controls
|
||||
- **GIVEN** Meeting Assistant is idle on macOS
|
||||
- **WHEN** the user opens the menu-bar control
|
||||
- **THEN** it shows `Open agent`
|
||||
- **AND** shows a start action for every configured launch profile
|
||||
- **AND** displays each profile's configured toggle hotkey
|
||||
- **AND** shows an exit action
|
||||
|
||||
#### Scenario: macOS menu bar shows active recording controls
|
||||
- **GIVEN** a meeting is recording on macOS
|
||||
- **WHEN** the user opens the menu-bar control
|
||||
- **THEN** it shows the active recording state
|
||||
- **AND** exposes stop and abort actions
|
||||
|
||||
#### Scenario: macOS menu opens the interactive agent
|
||||
- **GIVEN** Meeting Assistant is running on macOS
|
||||
- **WHEN** the user selects `Open agent` from the menu-bar control
|
||||
- **THEN** a `Meeting Summary Agent` window opens
|
||||
- **AND** the user can submit a chat message and receive the interactive agent response
|
||||
@@ -0,0 +1,18 @@
|
||||
## 1. Contract and behavior
|
||||
|
||||
- [x] 1.1 Define macOS desktop-control requirements and design.
|
||||
- [x] 1.2 Add a failing behavior test for configured shortcut and endpoint preservation.
|
||||
- [x] 1.3 Implement the desktop-control manifest and make the behavior test pass.
|
||||
|
||||
## 2. Native macOS integration
|
||||
|
||||
- [x] 2.1 Add and package the native macOS menu-bar/global-hotkey helper.
|
||||
- [x] 2.2 Start and stop the helper with the .NET application on macOS.
|
||||
- [x] 2.3 Add deterministic helper packaging/self-test coverage.
|
||||
|
||||
## 3. Verification and closeout
|
||||
|
||||
- [x] 3.1 Run narrow and full tests.
|
||||
- [x] 3.2 Run strict OpenSpec validation.
|
||||
- [x] 3.3 Launch Meeting Assistant, start a new recording, and save screenshot evidence.
|
||||
- [x] 3.4 Perform required DRY, SOLID, and KISS refactoring reviews and re-run verification.
|
||||
@@ -0,0 +1,37 @@
|
||||
## Context
|
||||
|
||||
The macOS build already bundles Swift helpers for audio and menu-bar controls. Calendar and screenshot integrations are currently replaced with no-op providers at compile time, while the ASR, diarization, speaker, OCR, and workflow-agent implementations themselves are portable.
|
||||
|
||||
## Decisions
|
||||
|
||||
### EventKit is the preferred macOS calendar source
|
||||
|
||||
A bundled Swift helper reads events through EventKit. This covers calendars exposed to macOS, including Outlook accounts synchronized into Calendar, without linking Windows COM into the portable target. If Calendar Full Access is denied, the managed client falls back to the already-authorized Calendar application's automation dictionary. The helper uses a stable permission-denied exit code rather than coupling the fallback to human-readable error text. The managed providers retain the existing Teams-marker and current-meeting selection rules.
|
||||
|
||||
### Native AppKit owns the recording prompt
|
||||
|
||||
The same helper displays a record/skip AppKit alert and returns the selected response to the existing `IMeetingStartPromptService` callback. Recording policy remains in the existing scheduler/controller.
|
||||
|
||||
### CoreGraphics captures the active app window
|
||||
|
||||
The helper resolves the frontmost application and captures its foremost on-screen window as PNG. The managed screenshot service continues to own attachment naming, note updates, OCR, cropping, attendee enrichment, retries, and waits.
|
||||
|
||||
### Portable managed features stay managed
|
||||
|
||||
Screenshot OCR, speaker identity, Docker-backed FunASR, FunASR diarization, Whisper-local pyannote diarization, and the AppKit/WebKit workflow editor use the same managed implementations on macOS and Windows. Platform branching is limited to the native input providers.
|
||||
|
||||
### Diagnostics report effective behavior
|
||||
|
||||
A read-only endpoint returns the host platform, concrete provider names, and effective enabled switches. This supplies deterministic proof without starting a recording or mutating meeting artifacts.
|
||||
|
||||
## Windows compatibility
|
||||
|
||||
The Windows target keeps its existing conditional registrations and does not compile or invoke the macOS helper. Verification must build the Windows target in addition to the portable target and tests.
|
||||
|
||||
## Verification
|
||||
|
||||
- Regression tests prove macOS resolves real providers and reports enabled capabilities.
|
||||
- Provider behavior tests prove EventKit JSON maps to calendar metadata, native prompt responses reach the callback, and PNG screenshot output reaches the screenshot interface.
|
||||
- The Swift helper provides deterministic self-test output.
|
||||
- A live macOS instance proves health, provider diagnostics, native helper packaging, calendar access, and screenshot capture.
|
||||
- A Windows-target build proves the existing Windows client still compiles with its native providers.
|
||||
@@ -0,0 +1,47 @@
|
||||
## Verification evidence
|
||||
|
||||
Verified on macOS 26 on 2026-07-25.
|
||||
|
||||
### Automated checks
|
||||
|
||||
- Focused macOS/calendar/screenshot/OCR tests: 35 passed, 0 failed.
|
||||
- Full portable test project: 441 passed, 6 failed, 447 total. The six failures are existing macOS-incompatible tests for Windows path assumptions, GDI+ taskbar rendering, and user-environment fallback; none cover this change.
|
||||
- Portable `net10.0` build: succeeded.
|
||||
- Windows `net10.0-windows10.0.19041.0` target build with Windows targeting enabled: succeeded with 0 warnings and 0 errors.
|
||||
- `openspec validate add-macos-meeting-integrations --strict`: valid.
|
||||
- Sequential DRY, SOLID, and KISS reviews were completed and their findings implemented.
|
||||
- Final independent standards and spec reviews reported no findings.
|
||||
|
||||
### Live macOS installation
|
||||
|
||||
The recording status was idle before deployment. The repository updater published the portable build, installed it into `~/Library/Application Support/MeetingAssistant/app`, restarted the LaunchAgent, and passed its health check.
|
||||
|
||||
`GET /diagnostics/platform-capabilities` returned:
|
||||
|
||||
```json
|
||||
{
|
||||
"host": "macOS",
|
||||
"meetingMetadataProvider": "MacOsMeetingMetadataProvider",
|
||||
"calendarMeetingProvider": "MacOsCalendarMeetingProvider",
|
||||
"meetingStartPromptProvider": "MacOsMeetingStartPromptService",
|
||||
"screenshotProvider": "MacOsDesktopIntegrationClient",
|
||||
"workflowEditorProvider": "MacOsWorkflowRulesEditorWindowService",
|
||||
"workflowEditorAvailable": true,
|
||||
"calendarRecordingPromptsEnabled": true,
|
||||
"speakerIdentificationEnabled": true,
|
||||
"screenshotOcrEnabled": true,
|
||||
"funAsrBackendEnabled": true,
|
||||
"funAsrDiarizationEnabled": true,
|
||||
"whisperLocalDiarizationEnabled": true
|
||||
}
|
||||
```
|
||||
|
||||
The installed arm64 Mach-O helper returned:
|
||||
|
||||
```json
|
||||
{"features":["calendar","prompt","screenshot","crop"],"status":"ok"}
|
||||
```
|
||||
|
||||
Direct native screenshot capture selected the foreground ChatGPT window and produced a valid 3258 by 1920 PNG. The asymmetric native crop regression test also proves OCR top-left pixel coordinates are preserved.
|
||||
|
||||
The live calendar diagnostic returned `provider=MacOsMeetingMetadataProvider`, `timedOut=false`, and a normal no-matching-event result. EventKit reported missing Full Calendar Access with stable exit code 77, after which the Calendar automation fallback completed; the scheduler logged `Synced 0 Teams calendar meetings`.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user