forked from Manuel/meeting-assistant
Compare commits
85
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
40281b57a5 | ||
|
|
c92e62bdf5 | ||
|
|
b40234d3b5 | ||
|
|
0a30a1ca5a | ||
|
|
1b19b08f2e | ||
|
|
6b1896660f | ||
|
|
05e23857dd | ||
|
|
9a91a81992 | ||
|
|
83726a2233 | ||
|
|
4840b8e3be | ||
|
|
445bff99ce | ||
|
|
1164c26846 | ||
|
|
bd35ebc4c8 | ||
|
|
fc0edd812d | ||
|
|
7b2bcd3631 | ||
|
|
bf6e1e7560 | ||
|
|
723298e720 | ||
|
|
e011e36ef7 | ||
|
|
e17fb8b225 | ||
|
|
c01213278e | ||
|
|
7814f1c7c4 | ||
|
|
88c3b21b2a | ||
|
|
b249e5d977 | ||
|
|
3e4bf88931 | ||
|
|
5e7c9f64d0 | ||
|
|
fbd5a0852f | ||
|
|
568bedb77a | ||
|
|
f21c56b1c7 | ||
|
|
c56de2b7ab | ||
|
|
40d10d634a | ||
|
|
264a3a823f | ||
|
|
072ebf38a3 | ||
|
|
4d66de943e | ||
|
|
150f11b6a7 | ||
|
|
d7db64bec7 | ||
|
|
cdeef1967e | ||
|
|
aeb0cfe42f | ||
|
|
ef0c0faba8 | ||
|
|
93eb3d4e98 | ||
|
|
d1d4b53ee8 | ||
|
|
d93d758a8f | ||
|
|
48ad4fb930 | ||
|
|
cb5a300b44 | ||
|
|
b6acba8402 | ||
|
|
a82e2fff03 | ||
|
|
336d940acd | ||
|
|
cbf854fb2a | ||
|
|
bacef060fe | ||
|
|
a928382e51 | ||
|
|
e23d04a38c | ||
|
|
c6a50d307e | ||
|
|
436fa6c149 | ||
|
|
5dce5fd350 | ||
|
|
c37654a7d9 | ||
|
|
cc9044e091 | ||
|
|
0e2aa3830f | ||
|
|
d77187e9ec | ||
|
|
6adf6f726b | ||
|
|
f86af983e8 | ||
|
|
43fc8aaec0 | ||
|
|
8b543a2161 | ||
|
|
58896073c0 | ||
|
|
945543c078 | ||
|
|
3e1f447b20 | ||
|
|
497390c2ec | ||
|
|
b1365b202d | ||
|
|
70ffaa7357 | ||
|
|
84dc696ce9 | ||
|
|
714da456df | ||
|
|
fa67f30c49 | ||
|
|
68db6955a2 | ||
|
|
dc37719d46 | ||
|
|
ee453e170e | ||
|
|
d51463321b | ||
|
|
41ae47fbfe | ||
|
|
f7ad2ce41a | ||
|
|
bc76151926 | ||
|
|
af9f5b34dc | ||
|
|
f3489f8e95 | ||
|
|
39591eef12 | ||
|
|
2af7933a1b | ||
|
|
9542c05ea4 | ||
|
|
5f39923c27 | ||
|
|
13a70c2375 | ||
|
|
59ba70ca11 |
@@ -0,0 +1,36 @@
|
||||
name: Native macOS Recovery diagnostic on Ubuntu
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
macos-native-diagnostic:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1"
|
||||
DOTNET_NOLOGO: "1"
|
||||
steps:
|
||||
- name: Checkout diagnostic source
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET for the diagnostic helper
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: "10.0.x"
|
||||
|
||||
- name: Probe native macOS Recovery with existing Docker resources
|
||||
run: dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --run --output artifacts/native-macos
|
||||
|
||||
- name: Always clean up only this diagnostic's owned resources
|
||||
if: always()
|
||||
run: dotnet run --file tools/ci/MacOsNativeDiagnostic.cs -- --cleanup --output artifacts/native-macos
|
||||
|
||||
- name: Preserve native diagnostic evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: native-macos-recovery-diagnostic
|
||||
path: artifacts/native-macos/
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
@@ -24,37 +24,33 @@ jobs:
|
||||
|
||||
- name: Install Wine
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
sudo dpkg --add-architecture i386
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends wine64 wine32 winbind unzip
|
||||
if [ -d /usr/lib/wine ]; then
|
||||
echo "/usr/lib/wine" >> "${GITHUB_PATH}"
|
||||
export PATH="${PATH}:/usr/lib/wine"
|
||||
fi
|
||||
if command -v wine >/dev/null 2>&1; then
|
||||
WINE_BIN="$(command -v wine)"
|
||||
elif command -v wine64 >/dev/null 2>&1; then
|
||||
WINE_BIN="$(command -v wine64)"
|
||||
elif [ -x /usr/lib/wine/wine64 ]; then
|
||||
WINE_BIN="/usr/lib/wine/wine64"
|
||||
elif [ -x /usr/lib/wine/wine ]; then
|
||||
WINE_BIN="/usr/lib/wine/wine"
|
||||
else
|
||||
echo "No wine binary found after installation."
|
||||
ls -la /usr/lib/wine || true
|
||||
exit 1
|
||||
fi
|
||||
sudo apt-get install -y --no-install-recommends ca-certificates curl gnupg winbind unzip
|
||||
sudo install -d -m 0755 /etc/apt/keyrings
|
||||
curl -fsSL https://dl.winehq.org/wine-builds/winehq.key \
|
||||
| gpg --dearmor \
|
||||
| sudo tee /etc/apt/keyrings/winehq-archive.key >/dev/null
|
||||
. /etc/os-release
|
||||
curl -fsSL "https://dl.winehq.org/wine-builds/ubuntu/dists/${VERSION_CODENAME}/winehq-${VERSION_CODENAME}.sources" \
|
||||
| sudo tee /etc/apt/sources.list.d/winehq.sources >/dev/null
|
||||
sudo apt-get update
|
||||
# Wine 9 cannot enumerate the DOS_DOT patterns used by .NET 10's SDK pack lookup.
|
||||
sudo apt-get install -y --no-install-recommends "winehq-stable=11.0.0.0~${VERSION_CODENAME}-1"
|
||||
WINE_BIN="/opt/wine-stable/bin/wine"
|
||||
test -x "${WINE_BIN}"
|
||||
echo "WINE_BIN=${WINE_BIN}" >> "${GITHUB_ENV}"
|
||||
"${WINE_BIN}" --version
|
||||
|
||||
- name: Restore (Windows target)
|
||||
- name: Restore (portable tests for Windows/Wine)
|
||||
run: |
|
||||
dotnet restore MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
-r win-x64
|
||||
|
||||
- name: Build (Windows target)
|
||||
- name: Build (portable tests for Windows/Wine)
|
||||
run: |
|
||||
dotnet build MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-c Release \
|
||||
@@ -77,16 +73,110 @@ jobs:
|
||||
echo "WIN_DOTNET_DIR=${WIN_DOTNET_DIR}" >> "${GITHUB_ENV}"
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" --info
|
||||
|
||||
- name: Provision NuGet signature trust inside Wine
|
||||
run: |
|
||||
SDK_VERSION="$(dotnet --version)"
|
||||
SDK_TRUST_ROOT="${DOTNET_ROOT}/sdk/${SDK_VERSION}/trustedroots"
|
||||
dotnet publish scripts/wine-sdk-trust.cs \
|
||||
-c Release \
|
||||
-p:UseAppHost=false \
|
||||
-p:PublishAot=false \
|
||||
-o "${RUNNER_TEMP}/wine-sdk-trust" \
|
||||
--nologo
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" \
|
||||
"Z:${RUNNER_TEMP}/wine-sdk-trust/WineSdkTrust.dll" --import \
|
||||
"Z:${SDK_TRUST_ROOT}/codesignctl.pem" \
|
||||
"Z:${SDK_TRUST_ROOT}/timestampctl.pem"
|
||||
|
||||
- name: Diagnose Windows SDK targeting-pack resolution
|
||||
run: |
|
||||
find "${WIN_DOTNET_DIR}/packs" -maxdepth 4 -name PackageOverrides.txt -print
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" msbuild MeetingAssistant/MeetingAssistant.csproj \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
-p:TargetFramework=net10.0 \
|
||||
-getProperty:NetCoreRoot,NetCoreTargetingPackRoot,PrunePackageDataRoot,PrunePackageTargetingPackRoots,MSBuildSDKsPath,DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" msbuild MeetingAssistant/MeetingAssistant.csproj \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
-p:TargetFramework=net10.0 \
|
||||
-t:AddPrunePackageReferences \
|
||||
-v:normal
|
||||
|
||||
- name: Build Windows desktop target via Wine
|
||||
run: |
|
||||
rm -f MeetingAssistant/bin/Release/net10.0-windows10.0.19041.0/win-x64/MeetingAssistant.dll
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" build MeetingAssistant/MeetingAssistant.csproj \
|
||||
-c Release \
|
||||
-f net10.0-windows10.0.19041.0 \
|
||||
-r win-x64 \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
--nologo
|
||||
test -s MeetingAssistant/bin/Release/net10.0-windows10.0.19041.0/win-x64/MeetingAssistant.dll
|
||||
|
||||
- name: Run tests via Wine (Windows dotnet host)
|
||||
run: |
|
||||
rm -f artifacts/tests/wine.trx
|
||||
"${WINE_BIN}" "${WIN_DOTNET_DIR}/dotnet.exe" test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-c Release \
|
||||
-r win-x64 \
|
||||
--no-build \
|
||||
--logger "trx;LogFileName=wine.trx" \
|
||||
--results-directory artifacts/tests \
|
||||
--nologo
|
||||
test -s artifacts/tests/wine.trx
|
||||
|
||||
- name: Show test output folder on failure
|
||||
if: failure()
|
||||
run: |
|
||||
find artifacts/tests -type f -print || true
|
||||
find MeetingAssistant.Tests -type d -name TestResults -print || true
|
||||
find MeetingAssistant.Tests -type f -path "*/TestResults/*" -maxdepth 5 -print || true
|
||||
|
||||
portable-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1"
|
||||
DOTNET_NOLOGO: "1"
|
||||
TZ: "Europe/Berlin"
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: "10.0.x"
|
||||
|
||||
- name: Restore (portable target)
|
||||
run: |
|
||||
dotnet restore MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-p:EnableWindowsTargeting=true
|
||||
|
||||
- name: Build (portable target including macOS managed adapters)
|
||||
run: |
|
||||
dotnet build MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-c Release \
|
||||
-f net10.0 \
|
||||
--no-restore \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
--nologo
|
||||
|
||||
- name: Run portable and macOS managed behavior tests on Ubuntu
|
||||
run: |
|
||||
rm -f artifacts/tests/portable.trx
|
||||
dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj \
|
||||
-c Release \
|
||||
-f net10.0 \
|
||||
--no-build \
|
||||
-p:EnableWindowsTargeting=true \
|
||||
--logger "trx;LogFileName=portable.trx" \
|
||||
--results-directory artifacts/tests \
|
||||
--nologo
|
||||
test -s artifacts/tests/portable.trx
|
||||
|
||||
- name: Show test output folder on failure
|
||||
if: failure()
|
||||
run: |
|
||||
find artifacts/tests -type f -print || true
|
||||
find MeetingAssistant.Tests -type d -name TestResults -print || true
|
||||
find MeetingAssistant.Tests -type f -path "*/TestResults/*" -maxdepth 5 -print || true
|
||||
|
||||
@@ -110,6 +110,11 @@ public sealed class AzureSpeechStreamingTranscriptionProviderTests
|
||||
[Fact]
|
||||
public void ResolveKeyFallsBackToUserEnvironment()
|
||||
{
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var keyEnv = $"MEETING_ASSISTANT_AZURE_KEY_{Guid.NewGuid():N}";
|
||||
try
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
@@ -41,6 +42,30 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
|
||||
Assert.Equal("ok", body?.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStartupRegistersDetectedHostOperatingSystemOnce()
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
using var response = await client.GetAsync("/health");
|
||||
var first = factory.Services.GetRequiredService<HostOperatingSystem>();
|
||||
var second = factory.Services.GetRequiredService<HostOperatingSystem>();
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Same(first, second);
|
||||
Assert.Equal(HostOperatingSystem.DetectCurrent(), first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
|
||||
{
|
||||
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
|
||||
var result = await playback.QueueAsync(1, [1, 2]);
|
||||
|
||||
Assert.Contains("Windows build", result);
|
||||
Assert.DoesNotContain("Queued", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
|
||||
{
|
||||
|
||||
@@ -58,6 +58,64 @@ public sealed class LaunchProfileOptionsProviderTests
|
||||
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
|
||||
{
|
||||
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
|
||||
|
||||
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
|
||||
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
|
||||
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
|
||||
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
|
||||
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
|
||||
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
|
||||
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContradictorySpeakerSampleDurationsAreRejected()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
|
||||
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
|
||||
});
|
||||
|
||||
var exception = Assert.Throws<InvalidOperationException>(
|
||||
() => provider.GetRequiredProfile(null));
|
||||
|
||||
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
|
||||
Assert.Contains("MaximumSampleDuration", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DuplicateProfileHotkeysAreRejected()
|
||||
{
|
||||
|
||||
@@ -2,8 +2,6 @@ using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
@@ -284,15 +282,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
return PngTestFixture.Solid(width, height);
|
||||
}
|
||||
|
||||
private static string CreateNonStreamingTextResponse(string text)
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.Workflow;
|
||||
using MeetingAssistant.Notifications;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsDesktopControlManifestTests
|
||||
{
|
||||
[Fact]
|
||||
public void AutomaticWorkflowChangeCannotSilentlyLoseItsPromptOnMacOs()
|
||||
{
|
||||
IWorkflowRulesEditorWindowService service = new MacOsWorkflowRulesEditorWindowService(
|
||||
Options.Create(new MeetingAssistantOptions()));
|
||||
|
||||
Assert.Throws<PlatformNotSupportedException>(() => service.ShowWithPrompt("Update meeting rules"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsApplicationDeclinesUnavailableAutomaticApprovals()
|
||||
{
|
||||
using var factory = new WebApplicationFactory<Program>();
|
||||
var approval = factory.Services.GetRequiredService<AgentApprovalService>();
|
||||
|
||||
Assert.False(await approval.ConfirmAsync("Test approval", "Do not apply a change", CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MacOsApplicationRegistersARealInteractiveAgentWindow()
|
||||
{
|
||||
using var factory = new WebApplicationFactory<Program>();
|
||||
|
||||
var service = factory.Services.GetRequiredService<IWorkflowRulesEditorWindowService>();
|
||||
|
||||
Assert.IsType<MacOsWorkflowRulesEditorWindowService>(service);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManifestPreservesConfiguredWindowsEquivalentShortcutsAndControlEndpoints()
|
||||
{
|
||||
var profiles = new[]
|
||||
{
|
||||
Profile("default", "Ctrl+Alt+M", "Ctrl+Alt+Z", "Ctrl+Alt+S"),
|
||||
Profile("english", "Ctrl+Alt+L", "", "")
|
||||
};
|
||||
|
||||
var hotkeys = new[]
|
||||
{
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+M", LaunchProfileHotkeyAction.ToggleRecording),
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+Z", LaunchProfileHotkeyAction.AbortRecording),
|
||||
new LaunchProfileHotkey("default", "Ctrl+Alt+S", LaunchProfileHotkeyAction.CaptureScreenshot),
|
||||
new LaunchProfileHotkey("english", "Ctrl+Alt+L", LaunchProfileHotkeyAction.ToggleRecording)
|
||||
};
|
||||
|
||||
var manifest = MacOsDesktopControlManifest.Create(
|
||||
"http://127.0.0.1:5090",
|
||||
profiles,
|
||||
hotkeys);
|
||||
|
||||
Assert.Collection(
|
||||
manifest.Hotkeys,
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+M", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/recording/toggle", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+Z", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/recording/abort", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+S", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/default/meetings/screenshot/capture", hotkey.Endpoint);
|
||||
},
|
||||
hotkey =>
|
||||
{
|
||||
Assert.Equal("Ctrl+Alt+L", hotkey.Shortcut);
|
||||
Assert.Equal("/profiles/english/recording/toggle", hotkey.Endpoint);
|
||||
});
|
||||
Assert.Equal("http://127.0.0.1:5090/recording/status", manifest.StatusUrl);
|
||||
Assert.Equal("http://127.0.0.1:5090/diagnostics/settings-and-logs", manifest.OpenAgentUrl);
|
||||
}
|
||||
|
||||
private static LaunchProfile Profile(string name, string toggle, string abort, string screenshot)
|
||||
{
|
||||
return new LaunchProfile(name, new MeetingAssistantOptions
|
||||
{
|
||||
Hotkey = new HotkeyOptions
|
||||
{
|
||||
Toggle = toggle,
|
||||
Abort = abort
|
||||
},
|
||||
Screenshots = new ScreenshotOptions
|
||||
{
|
||||
Hotkey = screenshot
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsFactAttribute : FactAttribute
|
||||
{
|
||||
public MacOsFactAttribute()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
Skip = "Requires native macOS helpers and Apple system tools; run on macOS.";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using System.Net.Http.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsMeetingAudioSourceTests
|
||||
{
|
||||
[Fact]
|
||||
public void MacOsServiceLauncherDefinesStablePrivacyIdentity()
|
||||
{
|
||||
var repositoryDirectory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (repositoryDirectory is not null &&
|
||||
!File.Exists(Path.Combine(repositoryDirectory.FullName, "MeetingAssistant", "MeetingAssistant.csproj")))
|
||||
{
|
||||
repositoryDirectory = repositoryDirectory.Parent;
|
||||
}
|
||||
|
||||
Assert.NotNull(repositoryDirectory);
|
||||
var projectDirectory = Path.Combine(repositoryDirectory.FullName, "MeetingAssistant");
|
||||
var launcherDirectory = Path.Combine(projectDirectory, "Native", "MacOsLauncher");
|
||||
var infoPlistPath = Path.Combine(launcherDirectory, "Info.plist");
|
||||
var launcherSourcePath = Path.Combine(launcherDirectory, "main.swift");
|
||||
|
||||
Assert.True(File.Exists(infoPlistPath), $"Expected launcher Info.plist at '{infoPlistPath}'.");
|
||||
Assert.True(File.Exists(launcherSourcePath), $"Expected launcher source at '{launcherSourcePath}'.");
|
||||
|
||||
var infoPlist = File.ReadAllText(infoPlistPath);
|
||||
Assert.Contains("cloud.schweigert.meeting-assistant", infoPlist, StringComparison.Ordinal);
|
||||
Assert.Contains("NSMicrophoneUsageDescription", infoPlist, StringComparison.Ordinal);
|
||||
Assert.Contains("NSCalendarsFullAccessUsageDescription", infoPlist, StringComparison.Ordinal);
|
||||
|
||||
var launcherSource = File.ReadAllText(launcherSourcePath);
|
||||
Assert.Contains("set -a", launcherSource, StringComparison.Ordinal);
|
||||
Assert.Contains("Contents/Resources/app", launcherSource, StringComparison.Ordinal);
|
||||
|
||||
var projectFile = File.ReadAllText(Path.Combine(projectDirectory, "MeetingAssistant.csproj"));
|
||||
Assert.Contains(
|
||||
"designated => identifier "cloud.schweigert.meeting-assistant"",
|
||||
projectFile,
|
||||
StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[MacOsFact]
|
||||
public void NativeAudioCaptureIsPackagedAsSignedMacOsAppForPersistentPrivacyGrant()
|
||||
{
|
||||
var configuration = new DirectoryInfo(AppContext.BaseDirectory)
|
||||
.Parent?.Name ?? "Debug";
|
||||
var appPath = Path.GetFullPath(Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"MeetingAssistant",
|
||||
"bin",
|
||||
configuration,
|
||||
"net10.0",
|
||||
"Native",
|
||||
"MeetingAssistantAudioCapture.app"));
|
||||
var infoPlistPath = Path.Combine(appPath, "Contents", "Info.plist");
|
||||
var helperPath = Path.Combine(appPath, "Contents", "MacOS", "macos-meeting-audio-capture");
|
||||
|
||||
Assert.True(Directory.Exists(appPath), $"Expected macOS audio capture app at '{appPath}'.");
|
||||
Assert.True(File.Exists(infoPlistPath), $"Expected Info.plist at '{infoPlistPath}'.");
|
||||
Assert.True(File.Exists(helperPath), $"Expected native helper at '{helperPath}'.");
|
||||
|
||||
var infoPlist = File.ReadAllText(infoPlistPath);
|
||||
Assert.Contains("cloud.schweigert.meeting-assistant.audio-capture", infoPlist, StringComparison.Ordinal);
|
||||
Assert.Contains("NSMicrophoneUsageDescription", infoPlist, StringComparison.Ordinal);
|
||||
|
||||
using var verification = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo
|
||||
{
|
||||
FileName = "/usr/bin/codesign",
|
||||
ArgumentList = { "--verify", "--deep", "--strict", appPath },
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
});
|
||||
Assert.NotNull(verification);
|
||||
verification.WaitForExit();
|
||||
Assert.True(
|
||||
verification.ExitCode == 0,
|
||||
$"Expected a valid app-bundle signature: {verification.StandardError.ReadToEnd()}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsMicrophoneStreamsNativePcmUsingRunFormat()
|
||||
{
|
||||
var pcm = Pcm16(1_000, -2_000, 3_000, -4_000);
|
||||
var process = new StubMacOsAudioCaptureProcess(pcm, exitCode: 0);
|
||||
var processFactory = new StubMacOsAudioCaptureProcessFactory(process);
|
||||
IMeetingAudioSource source = new MacOsMicrophoneAudioSource(
|
||||
processFactory,
|
||||
NullLogger<MacOsMicrophoneAudioSource>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Recording =
|
||||
{
|
||||
SampleRate = 24_000,
|
||||
Channels = 2
|
||||
}
|
||||
};
|
||||
|
||||
var chunks = await ReadChunksAsync(source, options);
|
||||
|
||||
var chunk = Assert.Single(chunks);
|
||||
Assert.Equal(pcm, chunk.Pcm);
|
||||
Assert.Equal(24_000, chunk.SampleRate);
|
||||
Assert.Equal(2, chunk.Channels);
|
||||
Assert.Equal(
|
||||
new MacOsAudioCaptureRequest(MacOsAudioCaptureKind.Microphone, 24_000, 2),
|
||||
processFactory.Request);
|
||||
Assert.True(process.Disposed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsSystemAudioReportsNativePermissionFailure()
|
||||
{
|
||||
var process = new StubMacOsAudioCaptureProcess(
|
||||
[],
|
||||
exitCode: 77,
|
||||
standardError: "Screen Recording/System Audio permission is required.");
|
||||
IMeetingAudioSource source = new MacOsSystemAudioSource(
|
||||
new StubMacOsAudioCaptureProcessFactory(process),
|
||||
NullLogger<MacOsSystemAudioSource>.Instance);
|
||||
|
||||
var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
await ReadChunksAsync(source, new MeetingAssistantOptions()));
|
||||
|
||||
Assert.Contains("Screen Recording/System Audio permission", exception.Message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PortableRegistrationUsesMacOsCompositeOnlyForMacOs()
|
||||
{
|
||||
var macServices = new ServiceCollection().AddLogging();
|
||||
var unsupportedServices = new ServiceCollection().AddLogging();
|
||||
|
||||
PortableMeetingAudioRegistration.Add(macServices, isMacOs: true);
|
||||
PortableMeetingAudioRegistration.Add(unsupportedServices, isMacOs: false);
|
||||
|
||||
using var macProvider = macServices.BuildServiceProvider();
|
||||
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
|
||||
Assert.IsType<CompositeMeetingAudioSource>(macProvider.GetRequiredService<IMeetingAudioSource>());
|
||||
Assert.IsType<UnavailableMeetingAudioSource>(unsupportedProvider.GetRequiredService<IMeetingAudioSource>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses()
|
||||
{
|
||||
var testRoot = Path.Combine(Path.GetTempPath(), $"meeting-assistant-macos-endpoint-{Guid.NewGuid():N}");
|
||||
var nativeProcesses = new EndpointMacOsAudioCaptureProcessFactory(
|
||||
microphonePcm: Pcm16(2_000),
|
||||
systemPcm: Pcm16(10_000));
|
||||
var speechPipelines = new CapturingSpeechRecognitionPipelineFactory();
|
||||
var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:Vault:BaseFolder"] = testRoot,
|
||||
["MeetingAssistant:Vault:TranscriptsFolder"] = "Transcripts",
|
||||
["MeetingAssistant:Vault:MeetingNotesFolder"] = "Notes",
|
||||
["MeetingAssistant:Vault:AssistantContextFolder"] = "Context",
|
||||
["MeetingAssistant:Vault:SummariesFolder"] = "Summaries",
|
||||
["MeetingAssistant:Vault:ProjectsFolder"] = "Projects",
|
||||
["MeetingAssistant:Recording:TemporaryRecordingsFolder"] = Path.Combine(testRoot, "Recordings"),
|
||||
["MeetingAssistant:Recording:InactivitySafeguard:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(testRoot, "speakers.db"),
|
||||
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false"
|
||||
});
|
||||
});
|
||||
builder.ConfigureTestServices(services =>
|
||||
{
|
||||
// The endpoint fixture supplies its own transcription pipeline and does not warm external pyannote models.
|
||||
services.Remove(services.Single(descriptor =>
|
||||
descriptor.ServiceType == typeof(IHostedService) &&
|
||||
descriptor.ImplementationType == typeof(PyannoteDiarizationWarmupHostedService)));
|
||||
services.RemoveAll<IMeetingAudioSource>();
|
||||
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||
PortableMeetingAudioRegistration.Add(services, isMacOs: true);
|
||||
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||
services.AddSingleton<IMacOsAudioCaptureProcessFactory>(nativeProcesses);
|
||||
services.RemoveAll<ISpeechRecognitionPipelineFactory>();
|
||||
services.AddSingleton<ISpeechRecognitionPipelineFactory>(speechPipelines);
|
||||
services.RemoveAll<IMeetingNoteOpener>();
|
||||
services.AddSingleton<IMeetingNoteOpener, NoopMeetingNoteOpener>();
|
||||
services.RemoveAll<IMeetingSummaryPipeline>();
|
||||
services.AddSingleton<IMeetingSummaryPipeline, NoopMeetingSummaryPipeline>();
|
||||
});
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
using var startResponse = await client.PostAsync("/recording/start", content: null);
|
||||
var started = await startResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||
var mixedChunk = await speechPipelines.FirstChunk.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.True(startResponse.IsSuccessStatusCode);
|
||||
Assert.True(started?.IsRecording);
|
||||
Assert.Equal(12_000, BitConverter.ToInt16(mixedChunk.Pcm));
|
||||
|
||||
using var stopResponse = await client.PostAsync("/recording/stop", content: null);
|
||||
var stopped = await stopResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||
|
||||
Assert.True(stopResponse.IsSuccessStatusCode);
|
||||
Assert.False(stopped?.IsRecording);
|
||||
Assert.All(nativeProcesses.Processes, process => Assert.True(process.Terminated));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await factory.DisposeAsync();
|
||||
using var connection = new SqliteConnection($"Data Source={Path.Combine(testRoot, "speakers.db")}");
|
||||
SqliteConnection.ClearPool(connection);
|
||||
if (Directory.Exists(testRoot))
|
||||
{
|
||||
Directory.Delete(testRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<AudioChunk>> ReadChunksAsync(
|
||||
IMeetingAudioSource source,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var chunks = new List<AudioChunk>();
|
||||
await foreach (var chunk in source.CaptureAsync(options, CancellationToken.None))
|
||||
{
|
||||
chunks.Add(chunk);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private static byte[] Pcm16(params short[] samples)
|
||||
{
|
||||
var pcm = new byte[samples.Length * sizeof(short)];
|
||||
Buffer.BlockCopy(samples, 0, pcm, 0, pcm.Length);
|
||||
return pcm;
|
||||
}
|
||||
|
||||
private sealed class StubMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly IMacOsAudioCaptureProcess process;
|
||||
|
||||
public StubMacOsAudioCaptureProcessFactory(IMacOsAudioCaptureProcess process)
|
||||
{
|
||||
this.process = process;
|
||||
}
|
||||
|
||||
public MacOsAudioCaptureRequest? Request { get; private set; }
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
Request = request;
|
||||
return process;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly MemoryStream standardOutput;
|
||||
private readonly string standardError;
|
||||
|
||||
public StubMacOsAudioCaptureProcess(byte[] pcm, int exitCode, string standardError = "")
|
||||
{
|
||||
standardOutput = new MemoryStream(pcm);
|
||||
this.standardError = standardError;
|
||||
ExitCode = exitCode;
|
||||
}
|
||||
|
||||
public Stream StandardOutput => standardOutput;
|
||||
|
||||
public int ExitCode { get; }
|
||||
|
||||
public bool HasExited => true;
|
||||
|
||||
public bool Disposed { get; private set; }
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(standardError);
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Disposed = true;
|
||||
standardOutput.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class EndpointMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly IReadOnlyDictionary<MacOsAudioCaptureKind, byte[]> pcmByKind;
|
||||
|
||||
public EndpointMacOsAudioCaptureProcessFactory(byte[] microphonePcm, byte[] systemPcm)
|
||||
{
|
||||
pcmByKind = new Dictionary<MacOsAudioCaptureKind, byte[]>
|
||||
{
|
||||
[MacOsAudioCaptureKind.Microphone] = microphonePcm,
|
||||
[MacOsAudioCaptureKind.System] = systemPcm
|
||||
};
|
||||
}
|
||||
|
||||
public List<BlockingMacOsAudioCaptureProcess> Processes { get; } = [];
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
var process = new BlockingMacOsAudioCaptureProcess(pcmByKind[request.Kind]);
|
||||
Processes.Add(process);
|
||||
return process;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BlockingMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly TaskCompletionSource terminated = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public BlockingMacOsAudioCaptureProcess(byte[] pcm)
|
||||
{
|
||||
StandardOutput = new InitialPcmThenBlockingStream(pcm, terminated.Task);
|
||||
}
|
||||
|
||||
public Stream StandardOutput { get; }
|
||||
|
||||
public int ExitCode => Terminated ? 137 : throw new InvalidOperationException("Capture is still running.");
|
||||
|
||||
public bool HasExited => Terminated;
|
||||
|
||||
public bool Terminated => terminated.Task.IsCompleted;
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult("");
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return terminated.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
terminated.TrySetResult();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
StandardOutput.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InitialPcmThenBlockingStream : Stream
|
||||
{
|
||||
private readonly byte[] pcm;
|
||||
private readonly Task processExit;
|
||||
private bool read;
|
||||
|
||||
public InitialPcmThenBlockingStream(byte[] pcm, Task processExit)
|
||||
{
|
||||
this.pcm = pcm;
|
||||
this.processExit = processExit;
|
||||
}
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position
|
||||
{
|
||||
get => throw new NotSupportedException();
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public override async ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!read)
|
||||
{
|
||||
read = true;
|
||||
pcm.AsSpan().CopyTo(buffer.Span);
|
||||
return pcm.Length;
|
||||
}
|
||||
|
||||
await processExit.WaitAsync(cancellationToken);
|
||||
return 0;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
public override void Flush()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
|
||||
{
|
||||
public TaskCompletionSource<AudioChunk> FirstChunk { get; } =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public ISpeechRecognitionPipeline Create()
|
||||
{
|
||||
return new CapturingSpeechRecognitionPipeline(FirstChunk);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
|
||||
{
|
||||
private readonly TaskCompletionSource<AudioChunk> firstChunk;
|
||||
private readonly TaskCompletionSource completed = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public CapturingSpeechRecognitionPipeline(TaskCompletionSource<AudioChunk> firstChunk)
|
||||
{
|
||||
this.firstChunk = firstChunk;
|
||||
}
|
||||
|
||||
public Task InitializeAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task InitializeAsync(
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task WaitUntilReadyAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
firstChunk.TrySetResult(chunk);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public Task CompleteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
completed.TrySetResult();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await completed.Task.WaitAsync(cancellationToken);
|
||||
yield break;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
|
||||
string audioPath,
|
||||
SpeechRecognitionPipelineOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<TranscriptionSegment>>([]);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
completed.TrySetResult();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopMeetingNoteOpener : IMeetingNoteOpener
|
||||
{
|
||||
public Task OpenAsync(string notePath, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class NoopMeetingSummaryPipeline : IMeetingSummaryPipeline
|
||||
{
|
||||
public Task<MeetingSummaryRunResult> RunAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingSummaryRunResult(artifacts.SummaryPath, "", Succeeded: true));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
using System.Net.Http.Json;
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MacOsMeetingIntegrationTests
|
||||
{
|
||||
[Fact]
|
||||
public void PortableRegistrationUsesRealMacOsProvidersOnlyOnMacOs()
|
||||
{
|
||||
var macServices = new ServiceCollection().AddLogging();
|
||||
var unsupportedServices = new ServiceCollection().AddLogging();
|
||||
|
||||
MacOsMeetingIntegrationRegistration.Add(macServices, isMacOs: true);
|
||||
MacOsMeetingIntegrationRegistration.Add(unsupportedServices, isMacOs: false);
|
||||
|
||||
using var macProvider = macServices.BuildServiceProvider();
|
||||
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
|
||||
|
||||
Assert.IsType<MacOsMeetingMetadataProvider>(
|
||||
macProvider.GetRequiredService<IMeetingMetadataProvider>());
|
||||
Assert.IsType<MacOsCalendarMeetingProvider>(
|
||||
macProvider.GetRequiredService<ICalendarMeetingProvider>());
|
||||
Assert.IsType<MacOsMeetingStartPromptService>(
|
||||
macProvider.GetRequiredService<IMeetingStartPromptService>());
|
||||
Assert.IsType<MacOsDesktopIntegrationClient>(
|
||||
macProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
|
||||
Assert.IsType<MacOsDesktopIntegrationClient>(
|
||||
macProvider.GetRequiredService<IScreenshotImageCropper>());
|
||||
|
||||
Assert.IsType<NoopMeetingMetadataProvider>(
|
||||
unsupportedProvider.GetRequiredService<IMeetingMetadataProvider>());
|
||||
Assert.IsType<NoopCalendarMeetingProvider>(
|
||||
unsupportedProvider.GetRequiredService<ICalendarMeetingProvider>());
|
||||
Assert.IsType<NoopMeetingStartPromptService>(
|
||||
unsupportedProvider.GetRequiredService<IMeetingStartPromptService>());
|
||||
Assert.IsType<UnavailableActiveWindowScreenshotCapture>(
|
||||
unsupportedProvider.GetRequiredService<IActiveWindowScreenshotCapture>());
|
||||
Assert.IsType<UnavailableScreenshotImageCropper>(
|
||||
unsupportedProvider.GetRequiredService<IScreenshotImageCropper>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsCalendarProvidersReturnTeamsMetadataAndPromptCandidate()
|
||||
{
|
||||
var startedAt = new DateTimeOffset(2026, 7, 25, 10, 5, 0, TimeSpan.FromHours(2));
|
||||
var calendarEvent = new MacOsCalendarEvent(
|
||||
"event-1",
|
||||
"Project sync",
|
||||
startedAt.AddMinutes(-5),
|
||||
startedAt.AddMinutes(55),
|
||||
"Microsoft Teams",
|
||||
"Discuss launch readiness.\nJoin Microsoft Teams Meeting",
|
||||
"https://teams.microsoft.com/l/meetup-join/example",
|
||||
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
|
||||
IsCanceled: false);
|
||||
var client = new StubMacOsDesktopIntegrationClient(calendarEvent);
|
||||
IMeetingMetadataProvider metadataProvider = new MacOsMeetingMetadataProvider(
|
||||
client,
|
||||
NullLogger<MacOsMeetingMetadataProvider>.Instance);
|
||||
ICalendarMeetingProvider calendarProvider = new MacOsCalendarMeetingProvider(
|
||||
client,
|
||||
NullLogger<MacOsCalendarMeetingProvider>.Instance);
|
||||
|
||||
var metadata = await metadataProvider.GetCurrentMeetingAsync(
|
||||
startedAt,
|
||||
CancellationToken.None);
|
||||
var candidates = await calendarProvider.GetRecordingPromptCandidatesForDayAsync(
|
||||
DateOnly.FromDateTime(startedAt.Date),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.NotNull(metadata);
|
||||
Assert.Equal("Project sync", metadata.Title);
|
||||
Assert.Equal(
|
||||
["Ada Lovelace <ada@example.com>", "Grace Hopper <grace@example.com>"],
|
||||
metadata.Attendees);
|
||||
Assert.Equal("Discuss launch readiness.", metadata.Agenda);
|
||||
Assert.Equal(calendarEvent.End, metadata.ScheduledEnd);
|
||||
|
||||
var candidate = Assert.Single(candidates);
|
||||
Assert.Equal("event-1", candidate.Id);
|
||||
Assert.Equal("Project sync", candidate.Subject);
|
||||
Assert.NotNull(candidate.Metadata);
|
||||
Assert.Equal(metadata.Title, candidate.Metadata.Title);
|
||||
Assert.Equal(metadata.Attendees, candidate.Metadata.Attendees);
|
||||
Assert.Equal(metadata.Agenda, candidate.Metadata.Agenda);
|
||||
Assert.Equal(metadata.ScheduledEnd, candidate.Metadata.ScheduledEnd);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MacOsPromptAndScreenshotUseNativeClientResults()
|
||||
{
|
||||
var screenshot = new ActiveWindowScreenshot([137, 80, 78, 71], "Meeting deck");
|
||||
var client = new StubMacOsDesktopIntegrationClient(
|
||||
screenshot: screenshot,
|
||||
promptResponse: MeetingStartPromptResponse.Record);
|
||||
IMeetingStartPromptService prompt = new MacOsMeetingStartPromptService(
|
||||
client,
|
||||
NullLogger<MacOsMeetingStartPromptService>.Instance);
|
||||
IActiveWindowScreenshotCapture capture = client;
|
||||
IScreenshotImageCropper cropper = client;
|
||||
MeetingStartPromptResponse? callbackResponse = null;
|
||||
var request = new MeetingStartPromptRequest(new CalendarMeeting(
|
||||
"event-1",
|
||||
"Project sync",
|
||||
DateTimeOffset.Now,
|
||||
DateTimeOffset.Now.AddHours(1)));
|
||||
|
||||
await prompt.ShowPromptAsync(
|
||||
request,
|
||||
(response, _) =>
|
||||
{
|
||||
callbackResponse = response;
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
CancellationToken.None);
|
||||
var captured = await capture.CaptureAsync(CancellationToken.None);
|
||||
var croppedPath = await cropper.SaveCroppedScreenshotAsync(
|
||||
"meeting.png",
|
||||
new ScreenshotCropCoordinates(1, 2, 3, 4),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(MeetingStartPromptResponse.Record, callbackResponse);
|
||||
Assert.Equal(screenshot, captured);
|
||||
Assert.Equal("meeting.png.cropped.png", croppedPath);
|
||||
}
|
||||
|
||||
[MacOsFact]
|
||||
public async Task MacOsCapabilityEndpointReportsEnabledRealProviders()
|
||||
{
|
||||
await using var factory = new WebApplicationFactory<Program>();
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
var report = await client.GetFromJsonAsync<PlatformCapabilityReport>(
|
||||
"/diagnostics/platform-capabilities");
|
||||
|
||||
Assert.NotNull(report);
|
||||
Assert.Equal("macOS", report.Host);
|
||||
Assert.Equal(nameof(MacOsMeetingMetadataProvider), report.MeetingMetadataProvider);
|
||||
Assert.Equal(nameof(MacOsCalendarMeetingProvider), report.CalendarMeetingProvider);
|
||||
Assert.Equal(nameof(MacOsMeetingStartPromptService), report.MeetingStartPromptProvider);
|
||||
Assert.Equal(nameof(MacOsDesktopIntegrationClient), report.ScreenshotProvider);
|
||||
Assert.Equal(nameof(MacOsWorkflowRulesEditorWindowService), report.WorkflowEditorProvider);
|
||||
Assert.True(report.WorkflowEditorAvailable);
|
||||
Assert.True(report.CalendarRecordingPromptsEnabled);
|
||||
Assert.True(report.SpeakerIdentificationEnabled);
|
||||
Assert.True(report.ScreenshotOcrEnabled);
|
||||
Assert.True(report.FunAsrBackendEnabled);
|
||||
Assert.True(report.FunAsrDiarizationEnabled);
|
||||
Assert.True(report.WhisperLocalDiarizationEnabled);
|
||||
}
|
||||
|
||||
[MacOsFact]
|
||||
public async Task NativeHelperAdvertisesCalendarPromptAndScreenshotFeatures()
|
||||
{
|
||||
var runner = new ProcessCommandRunner(
|
||||
NullLogger<ProcessCommandRunner>.Instance);
|
||||
|
||||
var result = await runner.RunAsync(
|
||||
GetNativeHelperPath(),
|
||||
["self-test"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, result.ExitCode);
|
||||
using var json = System.Text.Json.JsonDocument.Parse(result.StandardOutput);
|
||||
Assert.Equal("ok", json.RootElement.GetProperty("status").GetString());
|
||||
Assert.Equal(
|
||||
["calendar", "prompt", "screenshot", "crop"],
|
||||
json.RootElement.GetProperty("features")
|
||||
.EnumerateArray()
|
||||
.Select(feature => feature.GetString()!)
|
||||
.ToArray());
|
||||
}
|
||||
|
||||
[MacOsFact]
|
||||
public async Task NativeHelperCropsPngUsingOcrPixelCoordinates()
|
||||
{
|
||||
var testRoot = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-native-crop-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(testRoot);
|
||||
var bitmapPath = Path.Combine(testRoot, "input.bmp");
|
||||
var inputPath = Path.Combine(testRoot, "input.png");
|
||||
var outputPath = Path.Combine(testRoot, "output.png");
|
||||
var outputBitmapPath = Path.Combine(testRoot, "output.bmp");
|
||||
try
|
||||
{
|
||||
await File.WriteAllBytesAsync(
|
||||
bitmapPath,
|
||||
CreateTopRedBottomBlueBitmap());
|
||||
var runner = new ProcessCommandRunner(
|
||||
NullLogger<ProcessCommandRunner>.Instance);
|
||||
var inputConversion = await runner.RunAsync(
|
||||
"/usr/bin/sips",
|
||||
["-s", "format", "png", bitmapPath, "--out", inputPath],
|
||||
CancellationToken.None);
|
||||
Assert.Equal(0, inputConversion.ExitCode);
|
||||
|
||||
var result = await runner.RunAsync(
|
||||
GetNativeHelperPath(),
|
||||
["crop", inputPath, outputPath, "0", "0", "2", "1"],
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, result.ExitCode);
|
||||
var png = await File.ReadAllBytesAsync(outputPath);
|
||||
Assert.Equal((2, 1), PngTestFixture.ReadDimensions(png));
|
||||
var outputConversion = await runner.RunAsync(
|
||||
"/usr/bin/sips",
|
||||
["-s", "format", "bmp", outputPath, "--out", outputBitmapPath],
|
||||
CancellationToken.None);
|
||||
Assert.Equal(0, outputConversion.ExitCode);
|
||||
Assert.Equal(
|
||||
(Red: (byte)255, Green: (byte)0, Blue: (byte)0),
|
||||
ReadFirstBitmapPixel(await File.ReadAllBytesAsync(outputBitmapPath)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(testRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CalendarProviderUsesActualLocalMidnightsAcrossDaylightSavingTransition()
|
||||
{
|
||||
if (TimeZoneInfo.Local.Id != "Europe/Berlin")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var client = new StubMacOsDesktopIntegrationClient();
|
||||
var provider = new MacOsCalendarMeetingProvider(
|
||||
client,
|
||||
NullLogger<MacOsCalendarMeetingProvider>.Instance);
|
||||
|
||||
await provider.GetRecordingPromptCandidatesForDayAsync(
|
||||
new DateOnly(2026, 3, 29),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(
|
||||
DateTimeOffset.Parse("2026-03-29T00:00:00+01:00"),
|
||||
client.CalendarFrom);
|
||||
Assert.Equal(
|
||||
DateTimeOffset.Parse("2026-03-30T00:00:00+02:00"),
|
||||
client.CalendarTo);
|
||||
Assert.Equal(TimeSpan.FromHours(23), client.CalendarTo - client.CalendarFrom);
|
||||
}
|
||||
|
||||
[MacOsFact]
|
||||
public async Task CalendarClientFallsBackToCalendarAutomationWhenEventKitIsDenied()
|
||||
{
|
||||
var runner = new SequenceCommandRunner(
|
||||
new CommandResult(
|
||||
MacOsIntegrationException.CalendarPermissionDeniedExitCode,
|
||||
"",
|
||||
"Calendar Full Access permission is required."),
|
||||
new CommandResult(
|
||||
0,
|
||||
"""
|
||||
[{"id":"fallback-event","subject":"Project sync","start":"2026-07-25T08:00:00Z","end":"2026-07-25T09:00:00Z","location":"Microsoft Teams","notes":"","url":"https://teams.microsoft.com/example","attendees":[],"isCanceled":false}]
|
||||
""",
|
||||
""));
|
||||
IMacOsCalendarClient client = new MacOsDesktopIntegrationClient(
|
||||
runner,
|
||||
Path.GetDirectoryName(GetNativeHelperPath())!);
|
||||
|
||||
var events = await client.GetCalendarEventsAsync(
|
||||
DateTimeOffset.Parse("2026-07-25T00:00:00Z"),
|
||||
DateTimeOffset.Parse("2026-07-26T00:00:00Z"),
|
||||
CancellationToken.None);
|
||||
|
||||
var calendarEvent = Assert.Single(events);
|
||||
Assert.Equal("fallback-event", calendarEvent.Id);
|
||||
Assert.Collection(
|
||||
runner.Commands,
|
||||
command => Assert.Equal("calendar", command.Arguments[0]),
|
||||
command =>
|
||||
{
|
||||
Assert.Equal("/usr/bin/osascript", command.FileName);
|
||||
Assert.Equal(["-l", "JavaScript"], command.Arguments.Take(2));
|
||||
});
|
||||
}
|
||||
|
||||
private static string GetNativeHelperPath()
|
||||
{
|
||||
var configuration = new DirectoryInfo(AppContext.BaseDirectory)
|
||||
.Parent?.Name ?? "Debug";
|
||||
return Path.GetFullPath(Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"MeetingAssistant",
|
||||
"bin",
|
||||
configuration,
|
||||
"net10.0",
|
||||
"Native",
|
||||
"macos-meeting-integrations"));
|
||||
}
|
||||
|
||||
private static byte[] CreateTopRedBottomBlueBitmap()
|
||||
{
|
||||
var bitmap = new byte[70];
|
||||
bitmap[0] = (byte)'B';
|
||||
bitmap[1] = (byte)'M';
|
||||
BitConverter.GetBytes(bitmap.Length).CopyTo(bitmap, 2);
|
||||
BitConverter.GetBytes(54).CopyTo(bitmap, 10);
|
||||
BitConverter.GetBytes(40).CopyTo(bitmap, 14);
|
||||
BitConverter.GetBytes(2).CopyTo(bitmap, 18);
|
||||
BitConverter.GetBytes(2).CopyTo(bitmap, 22);
|
||||
BitConverter.GetBytes((short)1).CopyTo(bitmap, 26);
|
||||
BitConverter.GetBytes((short)24).CopyTo(bitmap, 28);
|
||||
BitConverter.GetBytes(16).CopyTo(bitmap, 34);
|
||||
|
||||
// BMP rows are bottom-up and padded to four-byte boundaries.
|
||||
bitmap[54] = 255;
|
||||
bitmap[57] = 255;
|
||||
bitmap[64] = 255;
|
||||
bitmap[67] = 255;
|
||||
return bitmap;
|
||||
}
|
||||
|
||||
private static (byte Red, byte Green, byte Blue) ReadFirstBitmapPixel(byte[] bitmap)
|
||||
{
|
||||
var pixelOffset = BitConverter.ToInt32(bitmap, 10);
|
||||
return (
|
||||
Red: bitmap[pixelOffset + 2],
|
||||
Green: bitmap[pixelOffset + 1],
|
||||
Blue: bitmap[pixelOffset]);
|
||||
}
|
||||
|
||||
private sealed class StubMacOsDesktopIntegrationClient :
|
||||
IMacOsCalendarClient,
|
||||
IMacOsMeetingPromptClient,
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private readonly IReadOnlyList<MacOsCalendarEvent> events;
|
||||
private readonly ActiveWindowScreenshot screenshot;
|
||||
private readonly MeetingStartPromptResponse promptResponse;
|
||||
|
||||
public StubMacOsDesktopIntegrationClient(
|
||||
MacOsCalendarEvent? calendarEvent = null,
|
||||
ActiveWindowScreenshot? screenshot = null,
|
||||
MeetingStartPromptResponse promptResponse = MeetingStartPromptResponse.Skip)
|
||||
{
|
||||
events = calendarEvent is null ? [] : [calendarEvent];
|
||||
this.screenshot = screenshot ?? new ActiveWindowScreenshot([], null);
|
||||
this.promptResponse = promptResponse;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
CalendarFrom = from;
|
||||
CalendarTo = to;
|
||||
return Task.FromResult(events);
|
||||
}
|
||||
|
||||
public DateTimeOffset CalendarFrom { get; private set; }
|
||||
|
||||
public DateTimeOffset CalendarTo { get; private set; }
|
||||
|
||||
public Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(promptResponse);
|
||||
}
|
||||
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(screenshot);
|
||||
}
|
||||
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<string?>($"{screenshotPath}.cropped.png");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SequenceCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly Queue<CommandResult> results;
|
||||
|
||||
public SequenceCommandRunner(params CommandResult[] results)
|
||||
{
|
||||
this.results = new Queue<CommandResult>(results);
|
||||
}
|
||||
|
||||
public List<(string FileName, IReadOnlyList<string> Arguments)> Commands { get; } = [];
|
||||
|
||||
public Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
Commands.Add((fileName, arguments));
|
||||
return Task.FromResult(results.Dequeue());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,10 +9,10 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -206,7 +206,7 @@ public sealed class MeetingNoteStoreTests
|
||||
{
|
||||
var link = MeetingNoteActionLinks.CreateSummaryRetryLink(
|
||||
"http://localhost:5090/",
|
||||
"C:\\Vault\\Meetings\\Summaries\\summary with spaces.md");
|
||||
Path.Combine(Path.GetTempPath(), "summary with spaces.md"));
|
||||
|
||||
Assert.Equal(
|
||||
"[Retry summary generation](http://localhost:5090/meetings/summary/retry?summaryPath=summary%20with%20spaces.md)",
|
||||
|
||||
@@ -4,8 +4,6 @@ using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
|
||||
#pragma warning disable CA1416
|
||||
|
||||
@@ -117,11 +115,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
var croppedPath = Assert.Single(Directory.GetFiles(
|
||||
Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"),
|
||||
"*-cropped.png"));
|
||||
using (var cropped = new Bitmap(croppedPath))
|
||||
{
|
||||
Assert.Equal(2, cropped.Width);
|
||||
Assert.Equal(2, cropped.Height);
|
||||
}
|
||||
Assert.Equal((2, 2), PngTestFixture.ReadDimensions(await File.ReadAllBytesAsync(croppedPath)));
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var screenshotIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal);
|
||||
@@ -133,6 +127,38 @@ public sealed class MeetingScreenshotServiceTests
|
||||
Assert.Contains("Shared screen text", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureKeepsSuccessfulOcrTextWhenCropperRejectsCoordinates()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new CapturingScreenshotOcrClient(
|
||||
"Readable text survives.",
|
||||
new ScreenshotCropCoordinates(99, 99, 10, 10));
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture(CreatePngBytes(4, 4)),
|
||||
ocr,
|
||||
screenshotImageCropper: new RejectingScreenshotImageCropper());
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(
|
||||
fixture.Artifacts,
|
||||
TimeSpan.FromSeconds(5),
|
||||
CancellationToken.None);
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("Readable text survives.", context);
|
||||
Assert.DoesNotContain("_OCR failed", context);
|
||||
Assert.DoesNotContain("![Cropped screenshot]", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer()
|
||||
{
|
||||
@@ -454,7 +480,8 @@ public sealed class MeetingScreenshotServiceTests
|
||||
IActiveWindowScreenshotCapture capture,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
|
||||
IScreenshotImageCropper? screenshotImageCropper = null)
|
||||
{
|
||||
return new MeetingScreenshotService(
|
||||
capture,
|
||||
@@ -462,12 +489,28 @@ public sealed class MeetingScreenshotServiceTests
|
||||
NoteStore,
|
||||
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
|
||||
ocrClient,
|
||||
screenshotImageCropper ??
|
||||
capture as IScreenshotImageCropper ??
|
||||
new UnavailableScreenshotImageCropper(),
|
||||
NullLogger<MeetingScreenshotService>.Instance,
|
||||
meetingWorkflowEngine);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
private sealed class RejectingScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new InvalidDataException("The crop is outside image bounds.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedScreenshotCapture :
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private readonly byte[] bytes;
|
||||
|
||||
@@ -480,6 +523,21 @@ public sealed class MeetingScreenshotServiceTests
|
||||
{
|
||||
return Task.FromResult(new ActiveWindowScreenshot(bytes, "Demo Window"));
|
||||
}
|
||||
|
||||
public async Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
await File.WriteAllBytesAsync(
|
||||
croppedPath,
|
||||
PngTestFixture.Solid(2, 2),
|
||||
cancellationToken);
|
||||
return croppedPath;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingScreenshotOcrClient : IScreenshotOcrClient
|
||||
@@ -614,15 +672,7 @@ public sealed class MeetingScreenshotServiceTests
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
return PngTestFixture.Solid(width, height);
|
||||
}
|
||||
|
||||
private static string ExtractScreenshotOcrId(string context)
|
||||
|
||||
@@ -7,6 +7,37 @@ namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MeetingSummaryInstructionBuilderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
var artifacts = CreateArtifacts(root);
|
||||
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
|
||||
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
|
||||
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
Agent = new AgentOptions { InitialPrompt = "Custom base." },
|
||||
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
|
||||
}));
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.StartsWith("Custom base.", instructions);
|
||||
Assert.Contains("# Alpha", instructions);
|
||||
Assert.Contains("Alpha Platform", instructions);
|
||||
Assert.Contains("Builds meeting automation", instructions);
|
||||
Assert.Contains("PROJECT.md", instructions);
|
||||
Assert.Contains("256", instructions);
|
||||
Assert.Contains("request_project_association", instructions);
|
||||
Assert.Contains("request_workflow_change", instructions);
|
||||
Assert.DoesNotContain("Internal notes", instructions);
|
||||
Assert.DoesNotContain("Do not expose", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
|
||||
{
|
||||
@@ -75,7 +106,8 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.Equal("Custom summarizer instructions.", instructions);
|
||||
Assert.StartsWith("Custom summarizer instructions.", instructions);
|
||||
Assert.Contains("request_project_association", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -96,21 +128,12 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
var expected = """
|
||||
Base.
|
||||
|
||||
---
|
||||
projects:
|
||||
|
||||
# Alpha
|
||||
|
||||
Alpha instructions.
|
||||
|
||||
# Beta
|
||||
|
||||
Beta instructions.
|
||||
""";
|
||||
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
|
||||
Assert.StartsWith("Base.", instructions);
|
||||
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
|
||||
Assert.Contains("# Alpha", instructions);
|
||||
Assert.Contains("Alpha instructions.", instructions);
|
||||
Assert.Contains("# Beta", instructions);
|
||||
Assert.Contains("Beta instructions.", instructions);
|
||||
}
|
||||
|
||||
private static MeetingSessionArtifacts CreateArtifacts(string root)
|
||||
|
||||
@@ -50,6 +50,145 @@ public sealed class MicrophoneAudioSourceTests
|
||||
await cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
var devices = new SelectableCaptureDevices(selection);
|
||||
var prompts = new AcceptingPrompt();
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero, monitor, selection);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||
try
|
||||
{
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
|
||||
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
|
||||
var nextChunk = chunks.MoveNextAsync().AsTask();
|
||||
Assert.False(nextChunk.IsCompleted);
|
||||
Assert.Null(selection.SelectedDeviceId);
|
||||
|
||||
prompts.Response.SetResult(true);
|
||||
|
||||
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.Equal("new", selection.SelectedDeviceId);
|
||||
Assert.False(lifetime.IsCancellationRequested);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
|
||||
{
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var abandonedCapture = new DisposableAudioSource();
|
||||
var factory = new CallbackCaptureFactory(() =>
|
||||
{
|
||||
lifetime.Cancel();
|
||||
return abandonedCapture;
|
||||
});
|
||||
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
|
||||
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
|
||||
|
||||
Assert.False(await chunks.MoveNextAsync());
|
||||
Assert.True(abandonedCapture.Disposed);
|
||||
}
|
||||
|
||||
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
|
||||
{
|
||||
var selection = new MicrophoneDeviceSelection();
|
||||
var devices = new SwitchDuringCreationDevices(selection);
|
||||
var prompts = new AcceptingPrompt();
|
||||
prompts.Response.SetResult(true);
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
|
||||
TimeSpan.Zero, monitor, selection);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
|
||||
try
|
||||
{
|
||||
Assert.True(await chunks.MoveNextAsync());
|
||||
Assert.Equal("new", selection.SelectedDeviceId);
|
||||
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
|
||||
Assert.True(devices.AbandonedCapture.Disposed);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
|
||||
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
private MicrophoneDevice[] available = [new("old", "Built-in")];
|
||||
private int creations;
|
||||
private volatile bool switchObserved;
|
||||
public DisposableAudioSource AbandonedCapture { get; } = new();
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||
{
|
||||
if (selection.SelectedDeviceId == "new") switchObserved = true;
|
||||
return Volatile.Read(ref available);
|
||||
}
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(GetAvailableMicrophones(), available[0]);
|
||||
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
{
|
||||
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
|
||||
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
|
||||
// Let the acceptance callback finish before returning the already-opened source.
|
||||
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
|
||||
throw new TimeoutException("The new microphone was not selected during capture creation.");
|
||||
return AbandonedCapture;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
|
||||
{
|
||||
public bool Disposed { get; private set; }
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
|
||||
public void Dispose() => Disposed = true;
|
||||
}
|
||||
|
||||
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
Shown.TrySetResult(microphone);
|
||||
return Response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
|
||||
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
|
||||
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
|
||||
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
|
||||
}
|
||||
|
||||
private static byte[] Pcm16(short sample)
|
||||
{
|
||||
return BitConverter.GetBytes(sample);
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.Recording;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MicrophoneSwitchMonitorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
|
||||
{
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var devices = new MutableDevices();
|
||||
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
|
||||
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
|
||||
var switched = false;
|
||||
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||
devices.Available = [new("usb", "USB microphone")];
|
||||
|
||||
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(prompts.WasShown);
|
||||
Assert.False(switched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
|
||||
{
|
||||
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
|
||||
var newDevice = new MicrophoneDevice("new", "USB headset");
|
||||
var devices = new MutableDevices { Available = [oldDevice] };
|
||||
var prompts = new ControlledPrompts();
|
||||
var monitor = CreateMonitor(devices, prompts);
|
||||
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
|
||||
try
|
||||
{
|
||||
devices.Available = [oldDevice, newDevice];
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Equal(newDevice, prompt.Device);
|
||||
Assert.False(switched.Task.IsCompleted);
|
||||
await devices.WaitForPollsAsync();
|
||||
Assert.False(prompts.HasMore);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
await devices.WaitForPollsAsync();
|
||||
Assert.False(prompts.HasMore);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("declined")]
|
||||
[InlineData("removed")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("ended")]
|
||||
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
|
||||
{
|
||||
var device = new MicrophoneDevice("usb", "USB microphone");
|
||||
var devices = new MutableDevices();
|
||||
var prompts = new ControlledPrompts();
|
||||
var monitor = CreateMonitor(devices, prompts,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
|
||||
var switched = false;
|
||||
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
|
||||
try
|
||||
{
|
||||
devices.Available = [device];
|
||||
var prompt = await prompts.NextAsync();
|
||||
if (outcome == "declined")
|
||||
{
|
||||
prompt.Response.SetResult(false);
|
||||
await devices.WaitForPollsAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
if (outcome == "removed") devices.Available = [];
|
||||
if (outcome == "ended") await lifetime.CancelAsync();
|
||||
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
|
||||
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
}
|
||||
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
Assert.False(switched);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await lifetime.CancelAsync();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
|
||||
TimeSpan? promptLifetime = null) => new(devices, prompts,
|
||||
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
|
||||
promptLifetime ?? TimeSpan.FromMinutes(1));
|
||||
|
||||
private sealed class MutableDevices : IMicrophoneDeviceProvider
|
||||
{
|
||||
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
|
||||
public volatile MicrophoneDevice[] Available = [];
|
||||
public Action? BeforeRead { get; set; }
|
||||
|
||||
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
|
||||
{
|
||||
BeforeRead?.Invoke();
|
||||
polls.Writer.TryWrite(true);
|
||||
return Available;
|
||||
}
|
||||
|
||||
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
|
||||
=> new(Available, Available.FirstOrDefault());
|
||||
|
||||
public async Task WaitForPollsAsync()
|
||||
{
|
||||
while (polls.Reader.TryRead(out _)) { }
|
||||
for (var i = 0; i < 3; i++)
|
||||
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public bool WasShown { get; private set; }
|
||||
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
WasShown = true;
|
||||
beforeAcceptance();
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
|
||||
{
|
||||
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
|
||||
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
var prompt = new Prompt(microphone, cancellationToken);
|
||||
requests.Writer.TryWrite(prompt);
|
||||
return await prompt.Response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
|
||||
{
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -25,4 +25,16 @@ public sealed class NotificationActivationArgumentsTests
|
||||
Assert.Equal("abc123", arguments["promptId"]);
|
||||
Assert.Equal("continue", arguments["response"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
|
||||
{
|
||||
Assert.Contains(
|
||||
MeetingInactivityPromptActions.All,
|
||||
action => action.Content == "Pause transcription" &&
|
||||
action.ResponseArgument == "pause");
|
||||
Assert.Equal(
|
||||
MeetingInactivityPromptResponse.Pause,
|
||||
MeetingInactivityPromptActions.ParseResponse("pause"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
internal static class PngTestFixture
|
||||
{
|
||||
private const string Solid2X2 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAAqADAAQAAAABAAAAAgAAAADO0J6QAAAAEklEQVQIHWP8DwQMQMAEIkAAAD34BACALvQ5AAAAAElFTkSuQmCC";
|
||||
private const string Solid4X4 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAABKADAAQAAAABAAAABAAAAADmpNw4AAAAFUlEQVQIHWP8DwQMSIAJiQ1mEhYAAAZdBAQjjGwcAAAAAElFTkSuQmCC";
|
||||
private const string Solid8X6 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAYAAAD+Bd/7AAAAAXNSR0IArs4c6QAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAACKADAAQAAAABAAAABgAAAAAJfOcJAAAAFklEQVQIHWP8DwQMeAATHjmw1GBQAAAQiQQIVmhjoQAAAABJRU5ErkJggg==";
|
||||
|
||||
public static byte[] Solid(int width, int height)
|
||||
{
|
||||
var encoded = (width, height) switch
|
||||
{
|
||||
(2, 2) => Solid2X2,
|
||||
(4, 4) => Solid4X4,
|
||||
(8, 6) => Solid8X6,
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(width),
|
||||
$"No deterministic PNG fixture exists for {width}x{height}.")
|
||||
};
|
||||
return Convert.FromBase64String(encoded);
|
||||
}
|
||||
|
||||
public static (int Width, int Height) ReadDimensions(byte[] png)
|
||||
{
|
||||
if (!png.AsSpan(0, 8).SequenceEqual(
|
||||
new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }))
|
||||
{
|
||||
throw new InvalidDataException("The fixture is not a PNG.");
|
||||
}
|
||||
|
||||
return (
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(16, 4)),
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32BigEndian(png.AsSpan(20, 4)));
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,90 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ProjectKnowledgeToolTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("name: Incomplete project")]
|
||||
[InlineData("description: Missing name")]
|
||||
[InlineData("name: ' '\ndescription: Missing name")]
|
||||
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
|
||||
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
|
||||
using var catalog = JsonDocument.Parse(await tools.ListProjects());
|
||||
|
||||
var project = Assert.Single(catalog.RootElement.EnumerateArray());
|
||||
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
|
||||
Assert.Equal("", project.GetProperty("description").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectRoot = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
|
||||
var tools = new MeetingSummaryTools(artifacts,
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
|
||||
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
|
||||
|
||||
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||
$"description: {new string('b', 257)}", from: 3, to: 3);
|
||||
|
||||
Assert.StartsWith("Refused:", rejected);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
|
||||
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
|
||||
"description: Updated project purpose", from: 3, to: 3));
|
||||
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
|
||||
Assert.Contains("owner: Ada", updated);
|
||||
Assert.Contains("User-authored project notes.", updated);
|
||||
Assert.Contains("Updated project purpose", await tools.ListProjects());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
|
||||
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
|
||||
var tools = new MeetingSummaryTools(CreateArtifacts(root),
|
||||
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
|
||||
|
||||
var catalog = await tools.ListProjects();
|
||||
|
||||
using var json = JsonDocument.Parse(catalog);
|
||||
var projects = json.RootElement.EnumerateArray().ToArray();
|
||||
Assert.Equal(3, projects.Length);
|
||||
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
|
||||
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
|
||||
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
|
||||
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
|
||||
Assert.Equal("", projects[1].GetProperty("description").GetString());
|
||||
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
|
||||
Assert.Equal("", projects[2].GetProperty("description").GetString());
|
||||
Assert.DoesNotContain("Private", catalog);
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
|
||||
{
|
||||
@@ -48,7 +128,7 @@ public sealed class ProjectKnowledgeToolTests
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Contains("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
|
||||
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
|
||||
|
||||
@@ -56,15 +136,15 @@ public sealed class ProjectKnowledgeToolTests
|
||||
|
||||
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
|
||||
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
|
||||
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
||||
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
|
||||
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
|
||||
Assert.Equal(
|
||||
"README.md:2 Second alpha line",
|
||||
await tools.Search("alpha"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
|
||||
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
@@ -88,7 +168,8 @@ public sealed class ProjectKnowledgeToolTests
|
||||
}
|
||||
});
|
||||
|
||||
Assert.Equal("MeetingAssistant", await tools.ListProjects());
|
||||
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
|
||||
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -100,8 +181,11 @@ public sealed class ProjectKnowledgeToolTests
|
||||
Directory.CreateDirectory(projectRoot);
|
||||
var projectFile = Path.Combine(projectRoot, "notes.md");
|
||||
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||
var tools = new MeetingSummaryTools(
|
||||
CreateArtifacts(root),
|
||||
artifacts,
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
Vault =
|
||||
@@ -132,8 +216,13 @@ public sealed class ProjectKnowledgeToolTests
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var projectsRoot = Path.Combine(root, "Projects");
|
||||
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
|
||||
var artifacts = CreateArtifacts(root);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
|
||||
var outsidePath = Path.Combine(projectsRoot, "outside.md");
|
||||
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
|
||||
var tools = new MeetingSummaryTools(
|
||||
CreateArtifacts(root),
|
||||
artifacts,
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
Vault =
|
||||
@@ -144,6 +233,7 @@ public sealed class ProjectKnowledgeToolTests
|
||||
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
|
||||
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
|
||||
|
||||
@@ -3,11 +3,92 @@ using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.ExceptionServices;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task HostedServiceStopsSafelyWhenShutdownRequestsOverlap()
|
||||
{
|
||||
const int iterations = 64;
|
||||
const int shutdownCallers = 8;
|
||||
using var applicationLifetime = new CancellationTokenSource();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
new BlockingCommandRunner(),
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
using var callersReady = new Barrier(shutdownCallers + 1);
|
||||
var shutdownErrors = new ConcurrentQueue<ExceptionDispatchInfo>();
|
||||
PyannoteDiarizationWarmupHostedService? service = null;
|
||||
var shutdownTasks = Enumerable.Range(0, shutdownCallers)
|
||||
.Select(_ => Task.Factory.StartNew(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
for (var iteration = 0; iteration < iterations; iteration++)
|
||||
{
|
||||
WaitForShutdownCallers();
|
||||
try
|
||||
{
|
||||
service!.StopAsync(CancellationToken.None)
|
||||
.WaitAsync(TimeSpan.FromSeconds(5)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
shutdownErrors.Enqueue(ExceptionDispatchInfo.Capture(exception));
|
||||
}
|
||||
WaitForShutdownCallers();
|
||||
if (!shutdownErrors.IsEmpty)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
shutdownErrors.Enqueue(ExceptionDispatchInfo.Capture(exception));
|
||||
}
|
||||
}, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default))
|
||||
.ToArray();
|
||||
|
||||
try
|
||||
{
|
||||
for (var iteration = 0; iteration < iterations; iteration++)
|
||||
{
|
||||
service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
await service.StartAsync(applicationLifetime.Token);
|
||||
WaitForShutdownCallers();
|
||||
WaitForShutdownCallers();
|
||||
if (!shutdownErrors.IsEmpty)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await Task.WhenAll(shutdownTasks).WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
if (shutdownErrors.TryPeek(out var shutdownError))
|
||||
{
|
||||
shutdownError.Throw();
|
||||
}
|
||||
|
||||
void WaitForShutdownCallers()
|
||||
{
|
||||
Assert.True(
|
||||
callersReady.SignalAndWait(TimeSpan.FromSeconds(5)),
|
||||
"Timed out coordinating overlapping shutdown requests.");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceWarmsEnabledValidationRuntimeWithoutBlockingStartup()
|
||||
{
|
||||
@@ -18,31 +99,8 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote-validation:local",
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
@@ -54,23 +112,109 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
|
||||
Assert.True(commandRunner.RunCancellationWasObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
|
||||
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(
|
||||
[
|
||||
new LaunchProfile("english", namedOptions),
|
||||
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
|
||||
]),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await commandRunner.WaitForRunAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
|
||||
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
|
||||
{
|
||||
var commandRunner = new BlockingCommandRunner();
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new PyannoteDiarizationWarmupHostedService(
|
||||
finalizer,
|
||||
new FakeLaunchProfileOptionsProvider(options),
|
||||
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(50));
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
private static MeetingAssistantOptions CreateValidationOptions(string image)
|
||||
{
|
||||
return new MeetingAssistantOptions
|
||||
{
|
||||
Recording = { TranscriptionProvider = "azure-speech" },
|
||||
SpeakerIdentification =
|
||||
{
|
||||
PyannoteValidation =
|
||||
{
|
||||
Enabled = true,
|
||||
Diarization =
|
||||
{
|
||||
DockerCommand = "docker",
|
||||
Image = image,
|
||||
ModelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models"),
|
||||
Token = "hf_test",
|
||||
TokenEnv = "",
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
||||
{
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly IReadOnlyList<LaunchProfile> profiles;
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
|
||||
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
|
||||
{
|
||||
this.profiles = profiles;
|
||||
}
|
||||
|
||||
public LaunchProfile GetRequiredProfile(string? name)
|
||||
{
|
||||
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
|
||||
var profileName = string.IsNullOrWhiteSpace(name)
|
||||
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
|
||||
: name;
|
||||
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfile> GetProfiles()
|
||||
{
|
||||
return [GetRequiredProfile(null)];
|
||||
return profiles;
|
||||
}
|
||||
|
||||
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
||||
|
||||
@@ -2,6 +2,7 @@ using MeetingAssistant;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NAudio.Wave;
|
||||
@@ -57,17 +58,49 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Assert.Empty(commandRunner.Commands);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner(
|
||||
"""
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
|
||||
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
|
||||
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
|
||||
""");
|
||||
var modelsFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"models");
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
|
||||
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
|
||||
})
|
||||
.Build();
|
||||
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
|
||||
var validator = CreateValidator(commandRunner, configuredOptions);
|
||||
|
||||
var valid = await validator.ValidateSampleAsync(
|
||||
CreateWav(TimeSpan.FromSeconds(20)),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(valid);
|
||||
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
bool enabled = true)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
return CreateValidator(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
@@ -76,9 +109,8 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
Enabled = enabled,
|
||||
MinimumSingleSpeakerCoverage = 0.90,
|
||||
MinimumMatchingKnownSnippetRatio = 1,
|
||||
Diarization = new PyannoteDiarizationOptions
|
||||
Diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BuildImage = false,
|
||||
DockerCommand = "docker",
|
||||
Image = "meeting-assistant-pyannote:local",
|
||||
@@ -94,7 +126,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
|
||||
CapturingCommandRunner commandRunner,
|
||||
MeetingAssistantOptions configuredOptions)
|
||||
{
|
||||
var finalizer = new PyannoteTranscriptFinalizer(
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
return new PyannoteSpeakerIdentityMatchValidator(
|
||||
finalizer,
|
||||
Options.Create(configuredOptions),
|
||||
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
|
||||
}
|
||||
|
||||
|
||||
@@ -156,9 +156,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
commandRunner,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<PyannoteTranscriptFinalizer>.Instance);
|
||||
var explicitDiarization = new PyannoteDiarizationOptions
|
||||
var explicitDiarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-azure:local",
|
||||
@@ -169,7 +168,7 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
CommandTimeout = TimeSpan.FromMinutes(1)
|
||||
};
|
||||
|
||||
await finalizer.FinalizeAsync(
|
||||
await finalizer.FinalizeEnabledAsync(
|
||||
audioPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
|
||||
explicitDiarization,
|
||||
@@ -186,9 +185,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
{
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
|
||||
var diarization = new PyannoteDiarizationOptions
|
||||
var diarization = new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DockerCommand = "docker",
|
||||
BaseImage = "python:3.11-slim",
|
||||
Image = "meeting-assistant-pyannote-warmup:local",
|
||||
@@ -218,9 +216,8 @@ public sealed class PyannoteTranscriptFinalizerTests
|
||||
var commandRunner = new CapturingCommandRunner("");
|
||||
var finalizer = CreateFinalizer(commandRunner, token: null);
|
||||
|
||||
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
|
||||
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
|
||||
{
|
||||
Enabled = true,
|
||||
Token = "",
|
||||
TokenEnv = "",
|
||||
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,596 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(7, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Single(fixture.Encoder.Requests);
|
||||
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Sabrina", saved.CanonicalName);
|
||||
Assert.Equal(3, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Empty(saved.Snippets);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(8, saved.VoiceVectors.Count);
|
||||
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options =>
|
||||
{
|
||||
options.OutlierPruningMinimumVectors = 20;
|
||||
options.OutlierPruningNeighborSimilarity = 0.90;
|
||||
options.OutlierPruningMinimumNeighbors = 3;
|
||||
options.OutlierPruningMinimumClusterRatio = 0.60;
|
||||
});
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal(16, saved.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
saved.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
"Chris",
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
var currentRunVectors = Enumerable.Range(0, 8)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var liveResult = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5),
|
||||
CancellationToken.None);
|
||||
fixture.Encoder.Vectors = currentRunVectors;
|
||||
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
|
||||
{
|
||||
KnownSpeakerMappings = liveResult.SpeakerMappings
|
||||
};
|
||||
if (transcriptAlreadyRelabeled)
|
||||
{
|
||||
finalRequest = finalRequest with { Segments = liveResult.Segments };
|
||||
}
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(10, saved.VoiceVectors.Count);
|
||||
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
|
||||
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors =
|
||||
[
|
||||
UnitVector(0),
|
||||
UnitVector(0, 2, 0.01f),
|
||||
UnitVector(0, 3, 0.02f),
|
||||
UnitVector(0, 4, 0.03f),
|
||||
UnitVector(0, 5, 0.04f)
|
||||
];
|
||||
var service = fixture.CreateService();
|
||||
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
|
||||
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal(6, saved.VoiceVectors.Count);
|
||||
Assert.Single(saved.References);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyFinishedSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments[0].Speaker);
|
||||
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
|
||||
Assert.All(
|
||||
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Equal("John", saved.CanonicalName);
|
||||
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
|
||||
Assert.Equal("John", result.Segments.Single().Speaker);
|
||||
Assert.All(saved.References, reference =>
|
||||
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync(
|
||||
null,
|
||||
[UnitVector(0), UnitVector(0, 20, 0.02f)],
|
||||
candidates: ["John", "Mike"]);
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(identity.Id);
|
||||
Assert.Null(saved.CanonicalName);
|
||||
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, index + 2, 0.03f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ProcessFinishedTranscriptAsync(
|
||||
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadOnlyIdentityAsync();
|
||||
Assert.Equal("Manuel", saved.CanonicalName);
|
||||
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync(
|
||||
configureSpeaker: options => options.MaxMatchCandidates = 1);
|
||||
await fixture.AddIdentityAsync("Legacy WAV identity", []);
|
||||
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
|
||||
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
var result = await service.IdentifyKnownSpeakersAsync(
|
||||
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Chris", result.Segments.Single().Speaker);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
|
||||
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
|
||||
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
|
||||
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.ToList();
|
||||
var service = fixture.CreateService();
|
||||
|
||||
await service.ApplySpeakerOverrideAsync(
|
||||
request,
|
||||
"Guest-01",
|
||||
"Sabrina",
|
||||
CancellationToken.None);
|
||||
|
||||
var saved = await fixture.LoadIdentityAsync(target.Id);
|
||||
Assert.Single(saved.Snippets);
|
||||
Assert.Equal(5, saved.VoiceVectors.Count);
|
||||
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
|
||||
Assert.Single(await context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private sealed class Fixture : IAsyncDisposable
|
||||
{
|
||||
private readonly string directory;
|
||||
private readonly string databasePath;
|
||||
private readonly SpeakerIdentificationOptions speakerOptions;
|
||||
|
||||
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
|
||||
{
|
||||
this.directory = directory;
|
||||
this.databasePath = databasePath;
|
||||
this.speakerOptions = speakerOptions;
|
||||
}
|
||||
|
||||
public FakeEncoder Encoder { get; } = new();
|
||||
|
||||
public FakeSnippetExtractor SnippetExtractor { get; } = new();
|
||||
|
||||
public string DatabasePath => databasePath;
|
||||
|
||||
public static async Task<Fixture> CreateAsync(
|
||||
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
|
||||
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
|
||||
{
|
||||
var directory = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
var databasePath = Path.Combine(directory, "speaker-identities.db");
|
||||
var speakerOptions = new SpeakerIdentificationOptions
|
||||
{
|
||||
DatabasePath = databasePath,
|
||||
MatchBatchSize = 6,
|
||||
MaxMatchCandidates = 100,
|
||||
MatchIdentityActiveAge = TimeSpan.FromDays(365),
|
||||
MinimumSampleSpeechDuration = TimeSpan.Zero,
|
||||
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
}
|
||||
};
|
||||
configure?.Invoke(speakerOptions.Resemblyzer);
|
||||
configureSpeaker?.Invoke(speakerOptions);
|
||||
await using var context = new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
return new Fixture(directory, databasePath, speakerOptions);
|
||||
}
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService CreateService()
|
||||
{
|
||||
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
|
||||
return new ResemblyzerSpeakerIdentificationService(
|
||||
new TestDbContextFactory(databasePath),
|
||||
SnippetExtractor,
|
||||
Encoder,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
speakerOptions.Resemblyzer,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(appOptions),
|
||||
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
|
||||
}
|
||||
|
||||
public SpeakerIdentificationRequest CreateRequest(
|
||||
string speaker,
|
||||
int sampleCount,
|
||||
IReadOnlyList<string>? attendees = null,
|
||||
int segmentCount = 1)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, "transcript.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
var segments = Enumerable.Range(0, segmentCount)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 30),
|
||||
TimeSpan.FromSeconds((index + 1) * 30),
|
||||
speaker,
|
||||
"enough useful words for a speaker sample"))
|
||||
.ToList();
|
||||
var segment = segments[0];
|
||||
return new SpeakerIdentificationRequest(
|
||||
Path.Combine(directory, "meeting.wav"),
|
||||
new MeetingNote(
|
||||
Path.Combine(directory, "meeting.md"),
|
||||
new MeetingNoteFrontmatter
|
||||
{
|
||||
Title = "Test",
|
||||
Attendees = attendees?.ToList() ?? ["Chris"],
|
||||
Transcript = transcriptPath,
|
||||
AssistantContext = Path.Combine(directory, "context.md"),
|
||||
Summary = Path.Combine(directory, "summary.md")
|
||||
},
|
||||
""),
|
||||
segments,
|
||||
Enumerable.Range(0, sampleCount)
|
||||
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
|
||||
.ToList());
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> AddIdentityAsync(
|
||||
string? name,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
IReadOnlyList<string>? candidates = null,
|
||||
IReadOnlyList<string>? aliases = null)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
|
||||
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = speakerOptions.Resemblyzer.ModelId,
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"known-{index}",
|
||||
CreatedAt = now.AddMinutes(index)
|
||||
}).ToList()
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
return identity;
|
||||
}
|
||||
|
||||
public async Task AddCurrentRunCandidateAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string candidateName)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
|
||||
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "older-model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(UnitVector(1)),
|
||||
Fingerprint = "older-model-vector",
|
||||
CreatedAt = now
|
||||
}
|
||||
],
|
||||
References =
|
||||
[
|
||||
SpeakerIdentityReferences.Create(
|
||||
request.MeetingNote.Path,
|
||||
request.MeetingNote.Frontmatter.Transcript,
|
||||
now)
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync(identity => identity.Id == id);
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
|
||||
{
|
||||
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
|
||||
return await context.SpeakerIdentities
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.References)
|
||||
.SingleAsync();
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public IReadOnlyList<float[]> Vectors { get; set; } = [];
|
||||
|
||||
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
|
||||
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
|
||||
{
|
||||
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
|
||||
|
||||
public Task<byte[]> ExtractSnippetAsync(
|
||||
string audioPath,
|
||||
IReadOnlyList<TranscriptionSegment> speakerSegments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Requests.Add(speakerSegments.ToList());
|
||||
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
var databasePath = Path.Combine(directory, "identities.db");
|
||||
var factory = new TestDbContextFactory(databasePath);
|
||||
await using (var context = factory.CreateDbContext())
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris",
|
||||
DateTimeOffset.UtcNow.AddMonths(-2),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 20, 0.02f))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Chris duplicate",
|
||||
DateTimeOffset.UtcNow.AddDays(-1),
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.02f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 40, 0.02f)))
|
||||
.ToList(),
|
||||
directory));
|
||||
context.SpeakerIdentities.Add(CreateIdentity(
|
||||
"Decoy",
|
||||
DateTimeOffset.UtcNow.AddMonths(-1),
|
||||
[UnitVector(0)],
|
||||
directory));
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
Enabled = true,
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MaxVectorsPerIdentity = 1000,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained"
|
||||
};
|
||||
var service = new ResemblyzerSpeakerIdentityMergeService(
|
||||
factory,
|
||||
new ResemblyzerVoiceClusterMatcher(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
resemblyzerOptions,
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
{
|
||||
MergeRecentIdentityAge = TimeSpan.FromDays(14),
|
||||
MaxMatchCandidates = 1,
|
||||
MaxSnippetsPerSpeaker = 3,
|
||||
Resemblyzer = resemblyzerOptions
|
||||
}
|
||||
}),
|
||||
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
|
||||
|
||||
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.MatchAttempts);
|
||||
Assert.Equal(1, result.MergedPairs);
|
||||
await using var verification = factory.CreateDbContext();
|
||||
var saved = await verification.SpeakerIdentities
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.SingleAsync(identity => identity.CanonicalName == "Chris");
|
||||
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
|
||||
Assert.Equal("Chris", saved.CanonicalName);
|
||||
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
|
||||
Assert.Equal(20, saved.VoiceVectors.Count);
|
||||
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static SpeakerIdentity CreateIdentity(
|
||||
string name,
|
||||
DateTimeOffset createdAt,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
string directory)
|
||||
{
|
||||
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
|
||||
File.WriteAllText(transcriptPath, "Transcript");
|
||||
return new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = name,
|
||||
CreatedAt = createdAt,
|
||||
UpdatedAt = createdAt,
|
||||
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = ToBytes(vector),
|
||||
Fingerprint = $"{name}-{index}",
|
||||
CreatedAt = createdAt.AddMinutes(index)
|
||||
}).ToList(),
|
||||
References =
|
||||
[
|
||||
new SpeakerIdentityReference
|
||||
{
|
||||
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
|
||||
TranscriptPath = transcriptPath,
|
||||
CreatedAt = createdAt
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primary] = 1;
|
||||
if (secondary is { } index)
|
||||
{
|
||||
vector[index] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
private static byte[] ToBytes(float[] vector)
|
||||
{
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static float[] ToVector(byte[] bytes)
|
||||
{
|
||||
var vector = new float[bytes.Length / sizeof(float)];
|
||||
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
|
||||
return vector;
|
||||
}
|
||||
|
||||
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
|
||||
{
|
||||
private readonly string databasePath;
|
||||
|
||||
public TestDbContextFactory(string databasePath)
|
||||
{
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
|
||||
public SpeakerIdentityDbContext CreateDbContext()
|
||||
{
|
||||
return new SpeakerIdentityDbContext(
|
||||
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||
.Options);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcherTests
|
||||
{
|
||||
[Fact]
|
||||
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
|
||||
{
|
||||
var matcher = CreateMatcher();
|
||||
var query = Enumerable.Range(0, 5)
|
||||
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
|
||||
.ToList();
|
||||
var candidates = new[]
|
||||
{
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
|
||||
};
|
||||
|
||||
var result = matcher.Match(query, candidates);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
Assert.True(result.Cohesion >= 0.99);
|
||||
Assert.True(result.BestSimilarity >= 0.99);
|
||||
Assert.True(result.RunnerUpSimilarity < 0.1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FourVectorsDoNotTriggerAutomaticMatching()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 4).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("4/5", result.Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("cohesion", result.Reason);
|
||||
Assert.Null(result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarityBelowThresholdIsRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("best similarity", result.Reason);
|
||||
Assert.Equal(0, result.BestSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimilarCandidatesWithinMarginAreRejected()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
|
||||
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
|
||||
]);
|
||||
|
||||
Assert.False(result.Accepted);
|
||||
Assert.Contains("margin", result.Reason);
|
||||
Assert.NotNull(result.RunnerUpSimilarity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
|
||||
{
|
||||
var result = CreateMatcher().Match(
|
||||
Enumerable.Repeat(UnitVector(0), 5).ToList(),
|
||||
[
|
||||
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
|
||||
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
|
||||
]);
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.Equal(42, result.IdentityId);
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
|
||||
{
|
||||
return new ResemblyzerVoiceClusterMatcher(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
RequiredVectorsPerSpeaker = 5,
|
||||
MinimumClusterCohesion = 0.75,
|
||||
MinimumIdentitySimilarity = 0.75,
|
||||
MinimumSimilarityMargin = 0.05
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int? secondaryDimension = null,
|
||||
float secondaryValue = 0)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
if (secondaryDimension is { } dimension)
|
||||
{
|
||||
vector[dimension] = secondaryValue;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
|
||||
{
|
||||
[Fact]
|
||||
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(16, identity.VoiceVectors.Count);
|
||||
Assert.All(
|
||||
identity.VoiceVectors,
|
||||
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AmbiguousDenseClustersPreserveAllVectors()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 10)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(20, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 15)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(0, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PruningPreservesOtherModelsAndMalformedStoredRows()
|
||||
{
|
||||
var identity = IdentityWithVectors(
|
||||
Enumerable.Range(0, 16)
|
||||
.Select(index => UnitVector(0, index + 2, 0.04f))
|
||||
.Concat(Enumerable.Range(0, 4)
|
||||
.Select(index => UnitVector(1, index + 30, 0.04f)))
|
||||
.ToList());
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
[UnitVector(2, 60, 0.04f)],
|
||||
"older-model",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = [1],
|
||||
Fingerprint = "malformed",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "zero-magnitude",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
var pruner = CreatePruner();
|
||||
|
||||
var result = pruner.Prune(identity);
|
||||
|
||||
Assert.Equal(4, result);
|
||||
Assert.Equal(19, identity.VoiceVectors.Count);
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
|
||||
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
|
||||
}
|
||||
|
||||
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
|
||||
{
|
||||
return new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
OutlierPruningMinimumVectors = 20,
|
||||
OutlierPruningNeighborSimilarity = 0.90,
|
||||
OutlierPruningMinimumNeighbors = 3,
|
||||
OutlierPruningMinimumClusterRatio = 0.60
|
||||
},
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
|
||||
}
|
||||
|
||||
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var identity = new SpeakerIdentity();
|
||||
SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
"resemblyzer-0.1.4-pretrained",
|
||||
1000,
|
||||
DateTimeOffset.UtcNow);
|
||||
return identity;
|
||||
}
|
||||
|
||||
private static float[] UnitVector(
|
||||
int primaryDimension,
|
||||
int secondaryDimension,
|
||||
float secondaryValue)
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[primaryDimension] = 1;
|
||||
vector[secondaryDimension] = secondaryValue;
|
||||
return vector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.Enabled = true;
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(options),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
await encoder.WaitForWarmupAsync();
|
||||
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(encoder.CancellationObserved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisabledWarmupDoesNotInvokeEncoder()
|
||||
{
|
||||
var encoder = new BlockingEncoder();
|
||||
var service = new ResemblyzerWarmupHostedService(
|
||||
encoder,
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<ResemblyzerWarmupHostedService>.Instance);
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.False(encoder.WarmupStarted);
|
||||
}
|
||||
|
||||
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool WarmupStarted { get; private set; }
|
||||
|
||||
public bool CancellationObserved { get; private set; }
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
WarmupStarted = true;
|
||||
started.TrySetResult();
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
CancellationObserved = true;
|
||||
}
|
||||
}
|
||||
|
||||
public Task WaitForWarmupAsync()
|
||||
{
|
||||
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,13 +8,22 @@ namespace MeetingAssistant.Tests;
|
||||
public sealed class SpeakerAudioSampleCollectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
|
||||
public void DefaultSampleDurationsAreTenAndSixtySeconds()
|
||||
{
|
||||
var options = new SpeakerIdentificationOptions();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
|
||||
@@ -37,8 +46,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1));
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
|
||||
|
||||
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
|
||||
@@ -48,6 +57,61 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
|
||||
|
||||
var sample = Assert.Single(collector.Snapshot());
|
||||
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
|
||||
|
||||
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
|
||||
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
|
||||
|
||||
Assert.Empty(collector.Snapshot());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60));
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
|
||||
|
||||
var sample = collector.TryAdd(Segment(
|
||||
0,
|
||||
70,
|
||||
"Guest01",
|
||||
"one two three four five six seven eight nine ten."));
|
||||
|
||||
Assert.NotNull(sample);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
|
||||
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
|
||||
{
|
||||
@@ -55,8 +119,8 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 3,
|
||||
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSegmentGap: TimeSpan.FromSeconds(1),
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
logger: logger);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
|
||||
|
||||
@@ -69,6 +133,47 @@ public sealed class SpeakerAudioSampleCollectorTests
|
||||
Assert.Contains("minimum duration", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
|
||||
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
|
||||
|
||||
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
|
||||
|
||||
Assert.Collection(
|
||||
samples,
|
||||
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
|
||||
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
|
||||
{
|
||||
var collector = new SpeakerAudioSampleCollector(
|
||||
TimeSpan.FromMinutes(2),
|
||||
maxSamplesPerSpeaker: 5,
|
||||
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
|
||||
maximumSampleDuration: TimeSpan.FromSeconds(60),
|
||||
requireNonOverlappingSamples: true);
|
||||
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
|
||||
|
||||
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
|
||||
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
|
||||
|
||||
Assert.Single(collector.Snapshot());
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(
|
||||
double start,
|
||||
double end,
|
||||
|
||||
@@ -107,6 +107,9 @@ public sealed class SpeakerIdentityMergeServiceTests
|
||||
return new SpeakerIdentityMergeService(
|
||||
new TestSpeakerIdentityDbContextFactory(dbPath),
|
||||
Matcher,
|
||||
new ResemblyzerVoiceVectorOutlierPruner(
|
||||
new ResemblyzerSpeakerRecognitionOptions(),
|
||||
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
SpeakerIdentification = new SpeakerIdentificationOptions
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerIdentityMergerTests
|
||||
{
|
||||
[Fact]
|
||||
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var target = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("oldest", 1, now.AddMinutes(-4)),
|
||||
Vector("duplicate", 2, now.AddMinutes(-3))
|
||||
]
|
||||
};
|
||||
var source = new SpeakerIdentity
|
||||
{
|
||||
VoiceVectors =
|
||||
[
|
||||
Vector("duplicate", 2, now.AddMinutes(-2)),
|
||||
Vector("newer", 3, now.AddMinutes(-1)),
|
||||
Vector("newest", 4, now)
|
||||
]
|
||||
};
|
||||
|
||||
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
|
||||
|
||||
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
|
||||
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "model",
|
||||
Dimensions = 256,
|
||||
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerRecognitionFeatureSelectionTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
|
||||
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
|
||||
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
|
||||
bool enabled,
|
||||
Type expectedIdentificationType,
|
||||
Type expectedMergeType,
|
||||
int expectedMinimumSamples,
|
||||
bool expectedNonOverlappingSamples)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
try
|
||||
{
|
||||
await using var factory = new WebApplicationFactory<Program>()
|
||||
.WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
|
||||
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
|
||||
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
|
||||
});
|
||||
});
|
||||
builder.ConfigureTestServices(services =>
|
||||
{
|
||||
services.RemoveAll<IResemblyzerVoiceEncoder>();
|
||||
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
|
||||
});
|
||||
});
|
||||
|
||||
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
|
||||
|
||||
Assert.IsType(expectedIdentificationType, service);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
|
||||
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
|
||||
Assert.IsType(expectedMergeType, mergeService);
|
||||
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
|
||||
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
|
||||
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
|
||||
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
|
||||
}
|
||||
finally
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
public Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<float[]>>([]);
|
||||
}
|
||||
|
||||
public Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerSampleSpanSelectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13),
|
||||
Segment(20, 70)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Equal(3, span.Count);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60));
|
||||
|
||||
Assert.Empty(span);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
|
||||
{
|
||||
var segments = Enumerable.Range(0, 6)
|
||||
.Select(index => new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(index * 10),
|
||||
TimeSpan.FromSeconds((index + 1) * 10),
|
||||
"Guest01",
|
||||
$"segment {index} has enough words"))
|
||||
.ToList();
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
segments,
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(5, spans.Count);
|
||||
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
|
||||
Assert.All(
|
||||
spans.Zip(spans.Skip(1)),
|
||||
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 10),
|
||||
Segment(9, 19),
|
||||
Segment(20, 30)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Equal(2, spans.Count);
|
||||
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(7, 13)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.Zero, span[0].Start);
|
||||
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[
|
||||
Segment(0, 4),
|
||||
Segment(20, 25)
|
||||
],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
Assert.Empty(spans);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
|
||||
{
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
[Segment(0, 70)],
|
||||
"Guest01",
|
||||
TimeSpan.FromSeconds(10),
|
||||
TimeSpan.FromSeconds(60),
|
||||
maxSpans: 5);
|
||||
|
||||
var span = Assert.Single(spans);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
|
||||
}
|
||||
|
||||
private static TranscriptionSegment Segment(double start, double end)
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
TimeSpan.FromSeconds(start),
|
||||
TimeSpan.FromSeconds(end),
|
||||
"Guest01",
|
||||
"enough words for this segment");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using MeetingAssistant.Speakers;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SpeakerVoiceVectorPersistenceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
context.SpeakerIdentities.Add(new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
VoiceVectors =
|
||||
[
|
||||
new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-a",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}
|
||||
]
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
var identity = await context.SpeakerIdentities
|
||||
.Include(candidate => candidate.VoiceVectors)
|
||||
.SingleAsync();
|
||||
var vector = Assert.Single(identity.VoiceVectors);
|
||||
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
|
||||
Assert.Equal(256, vector.Dimensions);
|
||||
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
|
||||
|
||||
context.SpeakerIdentities.Remove(identity);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using (var context = new SpeakerIdentityDbContext(options))
|
||||
{
|
||||
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
|
||||
{
|
||||
var databasePath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"speaker-identities.db");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
|
||||
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
|
||||
.UseSqlite($"Data Source={databasePath}")
|
||||
.Options;
|
||||
await using var context = new SpeakerIdentityDbContext(options);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = "Chris",
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
context.SpeakerVoiceVectors.AddRange(
|
||||
CreateVector(identity.Id, "same-vector"),
|
||||
CreateVector(identity.Id, "same-vector"));
|
||||
|
||||
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
|
||||
}
|
||||
|
||||
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
|
||||
{
|
||||
return new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identityId,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
using System.Threading.Channels;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Notifications;
|
||||
using MeetingAssistant.Summary;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class SummaryAgentRequestTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("denied")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("shutdown")]
|
||||
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
|
||||
{
|
||||
var prompts = new ControlledPrompts();
|
||||
var editor = new RecordingEditor();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
await requests.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
|
||||
var prompt = await prompts.NextAsync();
|
||||
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
}
|
||||
finally { await requests.StopAsync(CancellationToken.None); }
|
||||
Assert.Empty(editor.Prompts);
|
||||
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InvalidRequestsAreRefusedBeforeNotification()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
|
||||
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
|
||||
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
|
||||
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
|
||||
Assert.False(prompts.HasMore);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var prompts = new ControlledPrompts();
|
||||
var editor = new RecordingEditor();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
|
||||
await requests.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
|
||||
|
||||
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
|
||||
|
||||
Assert.StartsWith("Requested", result);
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
|
||||
Assert.Empty(editor.Prompts);
|
||||
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
|
||||
Assert.True(tools.SummaryWasWritten);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
Assert.False(prompt.Response.TrySetResult(true));
|
||||
Assert.Single(editor.Prompts);
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
finally { await requests.StopAsync(CancellationToken.None); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("denied")]
|
||||
[InlineData("expired")]
|
||||
[InlineData("canceled")]
|
||||
[InlineData("removed")]
|
||||
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
|
||||
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
|
||||
var prompt = await prompts.NextAsync();
|
||||
|
||||
if (outcome == "denied") prompt.Response.SetResult(false);
|
||||
if (outcome == "canceled") await cancellation.CancelAsync();
|
||||
if (outcome == "removed")
|
||||
{
|
||||
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
|
||||
prompt.Response.SetResult(true);
|
||||
}
|
||||
if (outcome == "canceled")
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
|
||||
else
|
||||
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
|
||||
|
||||
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
prompt.Response.TrySetResult(true);
|
||||
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
|
||||
{
|
||||
var fixture = await CreateAsync();
|
||||
var prompts = new ControlledPrompts();
|
||||
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
|
||||
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
|
||||
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
|
||||
var prompt = await prompts.NextAsync();
|
||||
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
|
||||
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
|
||||
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
|
||||
Assert.False(request.IsCompleted);
|
||||
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
|
||||
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
|
||||
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
|
||||
|
||||
prompt.Response.SetResult(true);
|
||||
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.StartsWith("Approved", result);
|
||||
Assert.Contains("Alpha instructions", result);
|
||||
Assert.Contains("Alpha Platform", result);
|
||||
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
|
||||
Assert.Contains("Edited by user", note);
|
||||
Assert.Contains("custom: keep-me", note);
|
||||
Assert.Contains("User's latest notes.", note);
|
||||
Assert.Contains("Existing", note);
|
||||
Assert.Contains("Alpha", note);
|
||||
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
|
||||
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
|
||||
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
|
||||
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
|
||||
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
|
||||
Assert.False(prompts.HasMore);
|
||||
}
|
||||
|
||||
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var project = Path.Combine(root, "Projects", "Alpha");
|
||||
Directory.CreateDirectory(project);
|
||||
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
|
||||
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
|
||||
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
|
||||
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
|
||||
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
|
||||
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
|
||||
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
|
||||
}
|
||||
|
||||
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
|
||||
{
|
||||
public List<string> Prompts { get; } = [];
|
||||
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public void Show() { }
|
||||
public void ShowWithPrompt(string initialPrompt)
|
||||
{
|
||||
Prompts.Add(initialPrompt);
|
||||
Started.TrySetResult(initialPrompt);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ControlledPrompts : IAgentApprovalPromptService
|
||||
{
|
||||
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
|
||||
public bool HasMore => requests.Reader.TryPeek(out _);
|
||||
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
|
||||
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new PendingPrompt(prompt, cancellationToken);
|
||||
requests.Writer.TryWrite(request);
|
||||
try { return await request.Response.Task.WaitAsync(cancellationToken); }
|
||||
finally { request.Removed.TrySetResult(); }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
|
||||
{
|
||||
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,8 @@ public sealed class TaskbarIconTests
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
@@ -94,10 +95,32 @@ public sealed class TaskbarIconTests
|
||||
menu,
|
||||
("Open agent", MeetingTaskbarAction.EditRules, false),
|
||||
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, true),
|
||||
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
|
||||
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
|
||||
("Exit", MeetingTaskbarAction.Exit, true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
|
||||
{
|
||||
var runningMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
|
||||
[Profile("default")]);
|
||||
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
|
||||
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
|
||||
[Profile("default")]);
|
||||
|
||||
Assert.Contains(runningMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.PauseTranscription &&
|
||||
item.Text == "Pause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
|
||||
item.Text == "Unpause transcription");
|
||||
Assert.Contains(pausedMenu.Items, item =>
|
||||
item.Action == MeetingTaskbarAction.StopRecording &&
|
||||
item.Text == "Finish meeting");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessingMenuShowsSummarizingButAllowsStartingNewRecordings()
|
||||
{
|
||||
@@ -158,6 +181,11 @@ public sealed class TaskbarIconTests
|
||||
[SupportedOSPlatform("windows")]
|
||||
public void TaskbarIconGlyphsAreVisuallyCentered()
|
||||
{
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var state in new[]
|
||||
{
|
||||
RecordingProcessState.Idle,
|
||||
@@ -200,7 +228,8 @@ public sealed class TaskbarIconTests
|
||||
private static RecordingStatus Status(
|
||||
bool isRecording = false,
|
||||
RecordingProcessState state = RecordingProcessState.Idle,
|
||||
string? profile = null)
|
||||
string? profile = null,
|
||||
bool isPaused = false)
|
||||
{
|
||||
return new RecordingStatus(
|
||||
isRecording,
|
||||
@@ -209,7 +238,8 @@ public sealed class TaskbarIconTests
|
||||
state == RecordingProcessState.Idle ? null : "context.md",
|
||||
state == RecordingProcessState.Idle ? null : "summary.md",
|
||||
state,
|
||||
profile);
|
||||
profile,
|
||||
isPaused);
|
||||
}
|
||||
|
||||
[SupportedOSPlatform("windows")]
|
||||
|
||||
@@ -7,16 +7,16 @@ public sealed class VaultPathTests
|
||||
[Fact]
|
||||
public void ResolveUsesVaultBaseFolderForRelativePaths()
|
||||
{
|
||||
var baseFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-vault");
|
||||
var vault = new VaultOptions
|
||||
{
|
||||
BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta"
|
||||
BaseFolder = baseFolder
|
||||
};
|
||||
|
||||
var resolved = VaultPath.Resolve(vault, @"Meetings\Transcripts");
|
||||
var relativePath = Path.Combine("Meetings", "Transcripts");
|
||||
var resolved = VaultPath.Resolve(vault, relativePath);
|
||||
|
||||
Assert.Equal(
|
||||
@"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta\Meetings\Transcripts",
|
||||
resolved);
|
||||
Assert.Equal(Path.Combine(baseFolder, relativePath), resolved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -24,11 +24,12 @@ public sealed class VaultPathTests
|
||||
{
|
||||
var vault = new VaultOptions
|
||||
{
|
||||
BaseFolder = @"C:\Users\masc3\OpenCloud\Persönlich\Vault\Exxeta"
|
||||
BaseFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-vault")
|
||||
};
|
||||
var absolutePath = Path.Combine(Path.GetTempPath(), "meeting-assistant-other");
|
||||
|
||||
var resolved = VaultPath.Resolve(vault, @"C:\Other\Path");
|
||||
var resolved = VaultPath.Resolve(vault, absolutePath);
|
||||
|
||||
Assert.Equal(@"C:\Other\Path", resolved);
|
||||
Assert.Equal(absolutePath, resolved);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
using MeetingAssistant;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class VenvResemblyzerVoiceEncoderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
|
||||
{
|
||||
using var fixture = new EncoderFixture();
|
||||
|
||||
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.FileName == "python-test"
|
||||
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("torch==2.14.0+cpu")
|
||||
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
|
||||
Assert.Contains(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Contains("Resemblyzer==0.1.4")
|
||||
&& command.Arguments.Contains("--no-deps"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
warmupExitCode: 21,
|
||||
warmupError: "incompatible environment");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
|
||||
|
||||
Assert.Empty(Directory.EnumerateFiles(
|
||||
fixture.RuntimeFolder,
|
||||
".ready",
|
||||
SearchOption.AllDirectories));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
|
||||
{
|
||||
var first = new float[256];
|
||||
first[0] = 2;
|
||||
var second = new float[256];
|
||||
second[1] = 3;
|
||||
using var fixture = new EncoderFixture(VectorOutput([first, second]));
|
||||
|
||||
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, vectors.Count);
|
||||
Assert.Equal(1f, vectors[0][0], 5);
|
||||
Assert.Equal(1f, vectors[1][1], 5);
|
||||
var encodingCommand = Assert.Single(
|
||||
fixture.Runner.Commands,
|
||||
command => command.Arguments.Any(
|
||||
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
|
||||
Assert.EndsWith(
|
||||
OperatingSystem.IsWindows()
|
||||
? Path.Combine("Scripts", "python.exe")
|
||||
: Path.Combine("bin", "python"),
|
||||
encodingCommand.FileName,
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal(2, encodingCommand.Arguments.Count);
|
||||
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongVectorDimension()
|
||||
{
|
||||
var vector = new float[255];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("255 dimensions", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsMalformedVectorJson()
|
||||
{
|
||||
using var fixture = new EncoderFixture(
|
||||
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsNonFiniteVector()
|
||||
{
|
||||
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
|
||||
using var fixture = new EncoderFixture(
|
||||
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsWrongResultCount()
|
||||
{
|
||||
var vector = new float[256];
|
||||
vector[0] = 1;
|
||||
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
|
||||
{
|
||||
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidDataException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("zero magnitude", error.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EncodeAsyncReportsFailedLocalCommand()
|
||||
{
|
||||
using var fixture = new EncoderFixture("", 17, "runtime failed");
|
||||
|
||||
var error = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
|
||||
|
||||
Assert.Contains("exit code 17", error.Message);
|
||||
Assert.Contains("runtime failed", error.Message);
|
||||
}
|
||||
|
||||
private static string VectorOutput(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
return $"""
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
|
||||
{JsonSerializer.Serialize(vectors)}
|
||||
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
|
||||
""";
|
||||
}
|
||||
|
||||
private sealed class EncoderFixture : IDisposable
|
||||
{
|
||||
public EncoderFixture(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
RuntimeFolder = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"meeting-assistant-tests",
|
||||
Guid.NewGuid().ToString("N"),
|
||||
"resemblyzer");
|
||||
Runner = new CapturingCommandRunner(
|
||||
encodingOutput,
|
||||
encodingExitCode,
|
||||
encodingError,
|
||||
warmupExitCode,
|
||||
warmupError);
|
||||
var options = new MeetingAssistantOptions();
|
||||
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
|
||||
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
|
||||
Encoder = new VenvResemblyzerVoiceEncoder(
|
||||
Runner,
|
||||
Options.Create(options),
|
||||
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
|
||||
}
|
||||
|
||||
public CapturingCommandRunner Runner { get; }
|
||||
|
||||
public VenvResemblyzerVoiceEncoder Encoder { get; }
|
||||
|
||||
public string RuntimeFolder { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(RuntimeFolder))
|
||||
{
|
||||
Directory.Delete(RuntimeFolder, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingCommandRunner : ICommandRunner
|
||||
{
|
||||
private readonly string encodingOutput;
|
||||
private readonly int encodingExitCode;
|
||||
private readonly string encodingError;
|
||||
private readonly int warmupExitCode;
|
||||
private readonly string warmupError;
|
||||
|
||||
public CapturingCommandRunner(
|
||||
string encodingOutput = "",
|
||||
int encodingExitCode = 0,
|
||||
string encodingError = "",
|
||||
int warmupExitCode = 0,
|
||||
string warmupError = "")
|
||||
{
|
||||
this.encodingOutput = encodingOutput;
|
||||
this.encodingExitCode = encodingExitCode;
|
||||
this.encodingError = encodingError;
|
||||
this.warmupExitCode = warmupExitCode;
|
||||
this.warmupError = warmupError;
|
||||
}
|
||||
|
||||
public List<CapturedCommand> Commands { get; } = [];
|
||||
|
||||
public Task<CommandResult> RunAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
|
||||
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
|
||||
if (isEncoding)
|
||||
{
|
||||
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
|
||||
}
|
||||
|
||||
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
|
||||
return Task.FromResult(isWarmup
|
||||
? new CommandResult(warmupExitCode, string.Empty, warmupError)
|
||||
: new CommandResult(0, string.Empty, string.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
|
||||
}
|
||||
@@ -732,6 +732,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var rulesPath = Path.Combine(root, "rules.yaml");
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
HostOperatingSystem.DetectCurrent(),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions
|
||||
@@ -749,10 +750,61 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Contains("list_recent_summaries", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesWindowsCommandsAndConceptsOnWindows()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.Windows),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: Windows", instructions);
|
||||
Assert.Contains("PowerShell", instructions);
|
||||
Assert.Contains("Windows-style paths", instructions);
|
||||
Assert.Contains("Windows services", instructions);
|
||||
Assert.Contains("Task Manager", instructions);
|
||||
Assert.Contains("Do not use macOS-only commands or concepts", instructions);
|
||||
Assert.Contains("does not make unavailable integrations available", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesMacOSCommandsAndConceptsOnMacOS()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: macOS", instructions);
|
||||
Assert.Contains("zsh", instructions);
|
||||
Assert.Contains("POSIX-style paths", instructions);
|
||||
Assert.Contains("launchd and LaunchAgents", instructions);
|
||||
Assert.Contains("Activity Monitor", instructions);
|
||||
Assert.Contains("Do not use Windows-only commands or concepts", instructions);
|
||||
Assert.Contains("does not make unavailable integrations available", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderUsesPortableConceptsOnUnsupportedHosts()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.Unsupported),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
|
||||
Assert.Contains("Detected host operating system: unsupported", instructions);
|
||||
Assert.Contains("portable .NET concepts", instructions);
|
||||
Assert.Contains("Do not assume Windows or macOS", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
HostOperatingSystem.DetectCurrent(),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
|
||||
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
|
||||
@@ -767,6 +819,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
|
||||
{
|
||||
var builder = new WorkflowRulesEditorInstructionBuilder(
|
||||
new HostOperatingSystem(HostOperatingSystemKind.MacOS),
|
||||
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
@@ -784,6 +837,8 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Contains("read the matching assistant context", instructions);
|
||||
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
|
||||
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
|
||||
Assert.Contains("Detected host operating system: macOS", instructions);
|
||||
Assert.Contains("zsh", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -809,6 +864,32 @@ public sealed class WorkflowRulesEditorTests
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
|
||||
{
|
||||
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
|
||||
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
|
||||
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentityId = identity.Id,
|
||||
ModelId = "resemblyzer-0.1.4-pretrained",
|
||||
Dimensions = 256,
|
||||
VectorBytes = new byte[256 * sizeof(float)],
|
||||
Fingerprint = "vector-1",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
await fixture.Context.SaveChangesAsync();
|
||||
var tools = fixture.CreateTools();
|
||||
|
||||
var searchResult = await tools.SearchIdentities("Sabrina");
|
||||
var readResult = await tools.ReadIdentity(identity.Id);
|
||||
|
||||
Assert.Contains("\"sampleCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
|
||||
Assert.Contains("\"voiceVectorCount\": 1", readResult);
|
||||
Assert.DoesNotContain("vector-1", readResult);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
|
||||
{
|
||||
@@ -817,7 +898,7 @@ public sealed class WorkflowRulesEditorTests
|
||||
|
||||
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
|
||||
|
||||
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
|
||||
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
|
||||
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
|
||||
}
|
||||
|
||||
@@ -914,6 +995,30 @@ public sealed class WorkflowRulesEditorTests
|
||||
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
|
||||
{
|
||||
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
|
||||
{
|
||||
Draft = "My unfinished draft"
|
||||
};
|
||||
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
|
||||
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
|
||||
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
|
||||
|
||||
var task = requested.SendPromptAsync(prompt);
|
||||
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
|
||||
Assert.True(requested.IsThinking);
|
||||
Assert.Empty(pipeline.LastConversation);
|
||||
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
|
||||
Assert.Equal("My unfinished draft", existing.Draft);
|
||||
Assert.Empty(existing.Messages);
|
||||
pipeline.Release.SetResult();
|
||||
await task.WaitAsync(TimeSpan.FromSeconds(2));
|
||||
Assert.False(requested.IsThinking);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ViewModelShowsAgentErrorWhenRequestTimesOut()
|
||||
{
|
||||
|
||||
@@ -52,14 +52,14 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
|
||||
cachedMeetingDay = today;
|
||||
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
|
||||
logger.LogInformation(
|
||||
"Synced {MeetingCount} Outlook Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
|
||||
"Synced {MeetingCount} Teams calendar meetings for {Day}; next sync at {NextSyncAt}",
|
||||
cachedMeetings.Count,
|
||||
today,
|
||||
nextSyncAt);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not read today's Outlook Teams meetings for recording prompts");
|
||||
logger.LogWarning(exception, "Could not read today's Teams calendar meetings for recording prompts");
|
||||
nextSyncAt = now.Add(GetSyncInterval(promptOptions));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -32,7 +32,14 @@ Calendar-Feature is refined, Golem feature needs more work.
|
||||
|
||||
PROJECT.md should use this structure:
|
||||
|
||||
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: Project display name
|
||||
description: Concise project purpose and distinguishing context.
|
||||
---
|
||||
|
||||
# Executive Summary
|
||||
|
||||
<Elevator Pitch>
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace MeetingAssistant;
|
||||
|
||||
public enum HostOperatingSystemKind
|
||||
{
|
||||
Windows,
|
||||
MacOS,
|
||||
Unsupported
|
||||
}
|
||||
|
||||
public sealed record HostOperatingSystem(HostOperatingSystemKind Kind)
|
||||
{
|
||||
public static HostOperatingSystem DetectCurrent()
|
||||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.Windows);
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsMacOS())
|
||||
{
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.MacOS);
|
||||
}
|
||||
|
||||
return new HostOperatingSystem(HostOperatingSystemKind.Unsupported);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.LaunchProfiles;
|
||||
|
||||
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
var options = BindDefaultOptions();
|
||||
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ValidateSpeakerSampleDurations(options, DefaultProfileName);
|
||||
return new LaunchProfile(DefaultProfileName, options);
|
||||
}
|
||||
|
||||
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
|
||||
profileSection.Bind(options);
|
||||
ApplyArrayOverrides(profileSection, options);
|
||||
ValidateSpeakerSampleDurations(options, profileName);
|
||||
return new LaunchProfile(profileName, options);
|
||||
}
|
||||
|
||||
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
: name.Trim();
|
||||
}
|
||||
|
||||
private static void ValidateSpeakerSampleDurations(
|
||||
MeetingAssistantOptions options,
|
||||
string profileName)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification,
|
||||
$"Launch profile '{profileName}' has invalid speaker sample durations");
|
||||
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
|
||||
options.SpeakerIdentification.Resemblyzer,
|
||||
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
|
||||
}
|
||||
|
||||
private static void ApplyArrayOverrides(
|
||||
IConfigurationSection profileSection,
|
||||
MeetingAssistantOptions options)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed record MacOsDesktopControlManifest(
|
||||
string BaseUrl,
|
||||
string StatusUrl,
|
||||
string OpenAgentUrl,
|
||||
IReadOnlyList<MacOsDesktopProfile> Profiles,
|
||||
IReadOnlyList<MacOsDesktopHotkey> Hotkeys)
|
||||
{
|
||||
public static MacOsDesktopControlManifest Create(
|
||||
string baseUrl,
|
||||
IReadOnlyList<LaunchProfile> profiles,
|
||||
IReadOnlyList<LaunchProfileHotkey> hotkeys)
|
||||
{
|
||||
var normalizedBaseUrl = baseUrl.TrimEnd('/');
|
||||
return new MacOsDesktopControlManifest(
|
||||
normalizedBaseUrl,
|
||||
$"{normalizedBaseUrl}/recording/status",
|
||||
$"{normalizedBaseUrl}/diagnostics/settings-and-logs",
|
||||
profiles
|
||||
.Select(profile => new MacOsDesktopProfile(
|
||||
profile.Name,
|
||||
profile.Options.Hotkey.Toggle))
|
||||
.ToArray(),
|
||||
hotkeys.Select(CreateHotkey).ToArray());
|
||||
}
|
||||
|
||||
private static MacOsDesktopHotkey CreateHotkey(LaunchProfileHotkey hotkey)
|
||||
{
|
||||
var endpoint = hotkey.Action switch
|
||||
{
|
||||
LaunchProfileHotkeyAction.ToggleRecording => "recording/toggle",
|
||||
LaunchProfileHotkeyAction.AbortRecording => "recording/abort",
|
||||
LaunchProfileHotkeyAction.CaptureScreenshot => "meetings/screenshot/capture",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(hotkey))
|
||||
};
|
||||
return new MacOsDesktopHotkey(
|
||||
hotkey.Hotkey,
|
||||
$"/profiles/{Uri.EscapeDataString(hotkey.ProfileName)}/{endpoint}");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MacOsDesktopProfile(string Name, string ToggleShortcut);
|
||||
|
||||
public sealed record MacOsDesktopHotkey(string Shortcut, string Endpoint);
|
||||
@@ -0,0 +1,107 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed class MacOsDesktopControlService : IHostedService, IDisposable
|
||||
{
|
||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||
private readonly IOptions<MeetingAssistantOptions> options;
|
||||
private readonly IHostApplicationLifetime applicationLifetime;
|
||||
private readonly ILogger<MacOsDesktopControlService> logger;
|
||||
private Process? helper;
|
||||
private string? manifestPath;
|
||||
|
||||
public MacOsDesktopControlService(
|
||||
ILaunchProfileOptionsProvider launchProfiles,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
IHostApplicationLifetime applicationLifetime,
|
||||
ILogger<MacOsDesktopControlService> logger)
|
||||
{
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.options = options;
|
||||
this.applicationLifetime = applicationLifetime;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-desktop-controls");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
logger.LogWarning("macOS desktop controls helper was not found at {HelperPath}", helperPath);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
var manifest = MacOsDesktopControlManifest.Create(
|
||||
options.Value.Api.PublicBaseUrl,
|
||||
launchProfiles.GetProfiles(),
|
||||
launchProfiles.GetHotkeys());
|
||||
manifestPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-desktop-{Environment.ProcessId}.json");
|
||||
File.WriteAllText(
|
||||
manifestPath,
|
||||
JsonSerializer.Serialize(
|
||||
manifest,
|
||||
new JsonSerializerOptions(JsonSerializerDefaults.Web)));
|
||||
|
||||
helper = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = helperPath,
|
||||
ArgumentList = { manifestPath },
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
});
|
||||
|
||||
if (helper is null)
|
||||
{
|
||||
throw new InvalidOperationException("Could not start the macOS desktop controls helper.");
|
||||
}
|
||||
|
||||
helper.EnableRaisingEvents = true;
|
||||
helper.Exited += (_, _) =>
|
||||
{
|
||||
if (!applicationLifetime.ApplicationStopping.IsCancellationRequested)
|
||||
{
|
||||
logger.LogWarning("macOS desktop controls helper exited unexpectedly");
|
||||
}
|
||||
};
|
||||
logger.LogInformation("Started macOS menu-bar controls with {HotkeyCount} global hotkeys", manifest.Hotkeys.Count);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (helper is { HasExited: false })
|
||||
{
|
||||
helper.Kill();
|
||||
helper.WaitForExit(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
DeleteManifest();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
helper?.Dispose();
|
||||
DeleteManifest();
|
||||
}
|
||||
|
||||
private void DeleteManifest()
|
||||
{
|
||||
if (manifestPath is not null && File.Exists(manifestPath))
|
||||
{
|
||||
File.Delete(manifestPath);
|
||||
manifestPath = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
using System.Text.Json;
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
internal sealed record MacOsCalendarEvent(
|
||||
string Id,
|
||||
string Subject,
|
||||
DateTimeOffset Start,
|
||||
DateTimeOffset End,
|
||||
string Location,
|
||||
string Notes,
|
||||
string Url,
|
||||
IReadOnlyList<string> Attendees,
|
||||
bool IsCanceled);
|
||||
|
||||
internal interface IMacOsCalendarClient
|
||||
{
|
||||
Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal interface IMacOsMeetingPromptClient
|
||||
{
|
||||
Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class MacOsDesktopIntegrationClient :
|
||||
IMacOsCalendarClient,
|
||||
IMacOsMeetingPromptClient,
|
||||
IActiveWindowScreenshotCapture,
|
||||
IScreenshotImageCropper
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly string nativeDirectory;
|
||||
|
||||
public MacOsDesktopIntegrationClient(ICommandRunner commandRunner)
|
||||
: this(commandRunner, Path.Combine(AppContext.BaseDirectory, "Native"))
|
||||
{
|
||||
}
|
||||
|
||||
internal MacOsDesktopIntegrationClient(
|
||||
ICommandRunner commandRunner,
|
||||
string nativeDirectory)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.nativeDirectory = nativeDirectory;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<MacOsCalendarEvent>> GetCalendarEventsAsync(
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fromValue = from.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
|
||||
var toValue = to.ToString("O", System.Globalization.CultureInfo.InvariantCulture);
|
||||
CommandResult result;
|
||||
try
|
||||
{
|
||||
result = await RunHelperAsync(
|
||||
["calendar", fromValue, toValue],
|
||||
cancellationToken);
|
||||
}
|
||||
catch (MacOsIntegrationException exception)
|
||||
when (exception.ExitCode == MacOsIntegrationException.CalendarPermissionDeniedExitCode)
|
||||
{
|
||||
result = await RunCalendarAutomationFallbackAsync(
|
||||
fromValue,
|
||||
toValue,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return JsonSerializer.Deserialize<MacOsCalendarEvent[]>(result.StandardOutput, JsonOptions) ?? [];
|
||||
}
|
||||
|
||||
public async Task<MeetingStartPromptResponse> ShowMeetingStartPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requestPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-prompt-{Guid.NewGuid():N}.json");
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(
|
||||
requestPath,
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
subject = request.Meeting.Subject,
|
||||
start = request.Meeting.Start
|
||||
}, JsonOptions),
|
||||
cancellationToken);
|
||||
var result = await RunHelperAsync(["prompt", requestPath], cancellationToken);
|
||||
var response = JsonSerializer.Deserialize<MacOsPromptResult>(
|
||||
result.StandardOutput,
|
||||
JsonOptions);
|
||||
return string.Equals(response?.Response, "record", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingStartPromptResponse.Record
|
||||
: MeetingStartPromptResponse.Skip;
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(requestPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<ActiveWindowScreenshot> CaptureAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var outputPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"meeting-assistant-screenshot-{Guid.NewGuid():N}.png");
|
||||
try
|
||||
{
|
||||
var result = await RunHelperAsync(["screenshot", outputPath], cancellationToken);
|
||||
var response = JsonSerializer.Deserialize<MacOsScreenshotResult>(
|
||||
result.StandardOutput,
|
||||
JsonOptions);
|
||||
return new ActiveWindowScreenshot(
|
||||
await File.ReadAllBytesAsync(outputPath, cancellationToken),
|
||||
response?.WindowTitle);
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.Delete(outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
try
|
||||
{
|
||||
await RunHelperAsync(
|
||||
[
|
||||
"crop",
|
||||
screenshotPath,
|
||||
croppedPath,
|
||||
crop.X.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Y.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Width.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
crop.Height.ToString(System.Globalization.CultureInfo.InvariantCulture)
|
||||
],
|
||||
cancellationToken);
|
||||
}
|
||||
catch (MacOsIntegrationException exception)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
|
||||
exception);
|
||||
}
|
||||
return croppedPath;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunHelperAsync(
|
||||
IReadOnlyList<string> arguments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
throw new PlatformNotSupportedException(
|
||||
"The native macOS desktop integrations can only run on macOS.");
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(nativeDirectory, "macos-meeting-integrations");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS meeting integrations helper was not found at '{helperPath}'. Build or publish Meeting Assistant on macOS.",
|
||||
helperPath);
|
||||
}
|
||||
|
||||
var result = await commandRunner.RunAsync(helperPath, arguments, cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(result.StandardError)
|
||||
? "The helper did not provide an error message."
|
||||
: result.StandardError.Trim();
|
||||
throw new MacOsIntegrationException(
|
||||
result.ExitCode,
|
||||
$"macOS meeting integration '{arguments[0]}' failed with exit code {result.ExitCode}. {detail}");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunCalendarAutomationFallbackAsync(
|
||||
string from,
|
||||
string to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var scriptPath = Path.Combine(nativeDirectory, "macos-calendar-events.js");
|
||||
if (!File.Exists(scriptPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS Calendar automation fallback was not found at '{scriptPath}'.",
|
||||
scriptPath);
|
||||
}
|
||||
|
||||
var result = await commandRunner.RunAsync(
|
||||
"/usr/bin/osascript",
|
||||
["-l", "JavaScript", scriptPath, from, to],
|
||||
cancellationToken);
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(result.StandardError)
|
||||
? "Calendar automation did not provide an error message."
|
||||
: result.StandardError.Trim();
|
||||
throw new InvalidOperationException(
|
||||
$"macOS calendar access failed through both EventKit and Calendar automation. {detail}");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private sealed record MacOsPromptResult(string Response);
|
||||
|
||||
private sealed record MacOsScreenshotResult(string? WindowTitle);
|
||||
}
|
||||
|
||||
internal sealed class MacOsIntegrationException : InvalidOperationException
|
||||
{
|
||||
public const int CalendarPermissionDeniedExitCode = 77;
|
||||
|
||||
public MacOsIntegrationException(int exitCode, string message)
|
||||
: base(message)
|
||||
{
|
||||
ExitCode = exitCode;
|
||||
}
|
||||
|
||||
public int ExitCode { get; }
|
||||
}
|
||||
|
||||
internal sealed class MacOsMeetingMetadataProvider :
|
||||
IMeetingMetadataProvider,
|
||||
IMeetingMetadataDiagnosticProvider
|
||||
{
|
||||
private readonly IMacOsCalendarClient client;
|
||||
private readonly ILogger<MacOsMeetingMetadataProvider> logger;
|
||||
|
||||
public MacOsMeetingMetadataProvider(
|
||||
IMacOsCalendarClient client,
|
||||
ILogger<MacOsMeetingMetadataProvider> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<MeetingMetadata?> GetCurrentMeetingAsync(
|
||||
DateTimeOffset startedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await GetCurrentMeetingCoreAsync(startedAt, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogDebug(exception, "macOS calendar meeting metadata was not available");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<MeetingMetadataDiagnosticResult> DiagnoseCurrentMeetingAsync(
|
||||
DateTimeOffset startedAt,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await MeetingMetadataDiagnosticRunner.RunAsync(
|
||||
nameof(MacOsMeetingMetadataProvider),
|
||||
startedAt,
|
||||
timeout,
|
||||
token => GetCurrentMeetingCoreAsync(startedAt, token),
|
||||
"No matching Teams calendar event was found.",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<MeetingMetadata?> GetCurrentMeetingCoreAsync(
|
||||
DateTimeOffset startedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var events = await client.GetCalendarEventsAsync(
|
||||
startedAt.AddHours(-4),
|
||||
startedAt.AddMinutes(5),
|
||||
cancellationToken);
|
||||
var candidates = events
|
||||
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
|
||||
.Where(calendarEvent => !calendarEvent.IsCanceled)
|
||||
.ToList();
|
||||
var selected = OutlookMeetingCandidateSelector.Select(
|
||||
candidates,
|
||||
startedAt.LocalDateTime,
|
||||
calendarEvent => calendarEvent.Start.LocalDateTime,
|
||||
calendarEvent => calendarEvent.End.LocalDateTime);
|
||||
return selected is null ? null : MacOsCalendarEventMapper.CreateMetadata(selected);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsCalendarMeetingProvider : ICalendarMeetingProvider
|
||||
{
|
||||
private readonly IMacOsCalendarClient client;
|
||||
private readonly ILogger<MacOsCalendarMeetingProvider> logger;
|
||||
|
||||
public MacOsCalendarMeetingProvider(
|
||||
IMacOsCalendarClient client,
|
||||
ILogger<MacOsCalendarMeetingProvider> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CalendarMeeting>> GetRecordingPromptCandidatesForDayAsync(
|
||||
DateOnly day,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var dayStart = AtLocalMidnight(day);
|
||||
var dayEnd = AtLocalMidnight(day.AddDays(1));
|
||||
try
|
||||
{
|
||||
var events = await client.GetCalendarEventsAsync(dayStart, dayEnd, cancellationToken);
|
||||
return events
|
||||
.Where(calendarEvent => !calendarEvent.IsCanceled)
|
||||
.Where(MacOsCalendarEventMapper.IsTeamsEvent)
|
||||
.Where(calendarEvent => calendarEvent.Start >= dayStart && calendarEvent.Start < dayEnd)
|
||||
.OrderBy(calendarEvent => calendarEvent.Start)
|
||||
.Select(calendarEvent => new CalendarMeeting(
|
||||
calendarEvent.Id,
|
||||
CalendarAppointmentMetadata.ReadSubject(calendarEvent.Subject),
|
||||
calendarEvent.Start,
|
||||
calendarEvent.End,
|
||||
MacOsCalendarEventMapper.CreateMetadata(calendarEvent),
|
||||
calendarEvent.IsCanceled))
|
||||
.ToList();
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogDebug(exception, "macOS calendar meetings were not available");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private static DateTimeOffset AtLocalMidnight(DateOnly day)
|
||||
{
|
||||
var localMidnight = day.ToDateTime(TimeOnly.MinValue);
|
||||
return new DateTimeOffset(
|
||||
localMidnight,
|
||||
TimeZoneInfo.Local.GetUtcOffset(localMidnight));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsMeetingStartPromptService : IMeetingStartPromptService
|
||||
{
|
||||
private readonly IMacOsMeetingPromptClient client;
|
||||
private readonly ILogger<MacOsMeetingStartPromptService> logger;
|
||||
|
||||
public MacOsMeetingStartPromptService(
|
||||
IMacOsMeetingPromptClient client,
|
||||
ILogger<MacOsMeetingStartPromptService> logger)
|
||||
{
|
||||
this.client = client;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task ShowPromptAsync(
|
||||
MeetingStartPromptRequest request,
|
||||
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await client.ShowMeetingStartPromptAsync(request, cancellationToken);
|
||||
await handleResponseAsync(response, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(exception, "Failed to display native macOS meeting-start prompt");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class MacOsMeetingIntegrationRegistration
|
||||
{
|
||||
public static IServiceCollection Add(
|
||||
IServiceCollection services,
|
||||
bool isMacOs)
|
||||
{
|
||||
if (isMacOs)
|
||||
{
|
||||
services.TryAddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||
services.TryAddSingleton<MacOsDesktopIntegrationClient>();
|
||||
services.TryAddSingleton<IMacOsCalendarClient>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.TryAddSingleton<IMacOsMeetingPromptClient>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.AddSingleton<IMeetingMetadataProvider, MacOsMeetingMetadataProvider>();
|
||||
services.AddSingleton<ICalendarMeetingProvider, MacOsCalendarMeetingProvider>();
|
||||
services.AddSingleton<IMeetingStartPromptService, MacOsMeetingStartPromptService>();
|
||||
services.AddSingleton<IActiveWindowScreenshotCapture>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
services.AddSingleton<IScreenshotImageCropper>(services =>
|
||||
services.GetRequiredService<MacOsDesktopIntegrationClient>());
|
||||
return services;
|
||||
}
|
||||
|
||||
services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||
services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
|
||||
services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
|
||||
services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
services.AddSingleton<IScreenshotImageCropper, UnavailableScreenshotImageCropper>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class MacOsCalendarEventMapper
|
||||
{
|
||||
public static bool IsTeamsEvent(MacOsCalendarEvent calendarEvent)
|
||||
{
|
||||
return CalendarAppointmentMetadata.IsTeamsAppointment(
|
||||
calendarEvent.Subject,
|
||||
calendarEvent.Location,
|
||||
calendarEvent.Notes,
|
||||
calendarEvent.Url);
|
||||
}
|
||||
|
||||
public static MeetingMetadata CreateMetadata(MacOsCalendarEvent calendarEvent)
|
||||
{
|
||||
return CalendarAppointmentMetadata.Create(
|
||||
calendarEvent.Subject,
|
||||
calendarEvent.Attendees,
|
||||
calendarEvent.Notes,
|
||||
calendarEvent.End);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
using System.Diagnostics;
|
||||
using MeetingAssistant.Workflow;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.MacOs;
|
||||
|
||||
public sealed class MacOsWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
|
||||
{
|
||||
private readonly IOptions<MeetingAssistantOptions> options;
|
||||
|
||||
public MacOsWorkflowRulesEditorWindowService(IOptions<MeetingAssistantOptions> options)
|
||||
{
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void Show()
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var url = $"{options.Value.Api.PublicBaseUrl.TrimEnd('/')}/diagnostics/settings-and-logs";
|
||||
Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = "/usr/bin/open",
|
||||
ArgumentList = { url },
|
||||
UseShellExecute = false
|
||||
});
|
||||
}
|
||||
|
||||
public void ShowWithPrompt(string initialPrompt)
|
||||
{
|
||||
throw new PlatformNotSupportedException(
|
||||
"Automatic workflow-change conversations require the Windows approval UI. Use the interactive macOS agent instead.");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MacOsWorkflowRulesEditorSession
|
||||
{
|
||||
private readonly WorkflowRulesEditorChatViewModel viewModel;
|
||||
private readonly SemaphoreSlim sendLock = new(1, 1);
|
||||
|
||||
public MacOsWorkflowRulesEditorSession(WorkflowRulesEditorChatViewModel viewModel)
|
||||
{
|
||||
this.viewModel = viewModel;
|
||||
}
|
||||
|
||||
public async Task<MacOsWorkflowRulesEditorChatResponse> SendAsync(
|
||||
string? message,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(message))
|
||||
{
|
||||
return Snapshot();
|
||||
}
|
||||
|
||||
await sendLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
viewModel.Draft = message;
|
||||
await viewModel.SendAsync(cancellationToken);
|
||||
return Snapshot();
|
||||
}
|
||||
finally
|
||||
{
|
||||
sendLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private MacOsWorkflowRulesEditorChatResponse Snapshot()
|
||||
{
|
||||
return new MacOsWorkflowRulesEditorChatResponse(
|
||||
viewModel.Messages
|
||||
.Where(item => item.Kind == WorkflowRulesEditorConversationItemKind.Message)
|
||||
.Select(item => new MacOsWorkflowRulesEditorMessage(
|
||||
item.Message!.Role.ToString().ToLowerInvariant(),
|
||||
item.Message.Content))
|
||||
.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorChatRequest(string? Message);
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorChatResponse(
|
||||
IReadOnlyList<MacOsWorkflowRulesEditorMessage> Messages);
|
||||
|
||||
public sealed record MacOsWorkflowRulesEditorMessage(string Role, string Content);
|
||||
|
||||
public static class MacOsWorkflowRulesEditorPage
|
||||
{
|
||||
public const string Html = """
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Meeting Summary Agent</title>
|
||||
<style>
|
||||
:root { color-scheme: dark; font: 15px -apple-system, BlinkMacSystemFont, sans-serif; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: #171719; color: #f5f5f7; }
|
||||
main { height: 100vh; display: grid; grid-template-rows: auto 1fr auto; padding: 18px; gap: 14px; }
|
||||
h1 { margin: 0; font-size: 18px; }
|
||||
#messages { overflow: auto; display: flex; flex-direction: column; gap: 10px; }
|
||||
.message { max-width: 82%; padding: 10px 12px; border: 1px solid #414147; border-radius: 10px; white-space: pre-wrap; }
|
||||
.user { align-self: flex-end; background: #293650; }
|
||||
.agent { align-self: flex-start; background: #232326; }
|
||||
form { display: grid; grid-template-columns: 1fr auto; gap: 10px; }
|
||||
textarea { min-height: 78px; resize: vertical; border-radius: 9px; border: 1px solid #55555d; background: #222225; color: inherit; padding: 10px; font: inherit; }
|
||||
button { min-width: 96px; border: 0; border-radius: 9px; background: #0a84ff; color: white; font-weight: 600; }
|
||||
button:disabled { opacity: .55; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Meeting Summary Agent</h1>
|
||||
<section id="messages"></section>
|
||||
<form id="chat">
|
||||
<textarea id="prompt" placeholder="ask me what I can do for you"></textarea>
|
||||
<button id="send">Send</button>
|
||||
</form>
|
||||
</main>
|
||||
<script>
|
||||
const form = document.querySelector('#chat');
|
||||
const prompt = document.querySelector('#prompt');
|
||||
const send = document.querySelector('#send');
|
||||
const messages = document.querySelector('#messages');
|
||||
const render = items => {
|
||||
messages.replaceChildren(...items.map(item => {
|
||||
const card = document.createElement('div');
|
||||
card.className = `message ${item.role}`;
|
||||
card.textContent = item.content;
|
||||
return card;
|
||||
}));
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
};
|
||||
form.addEventListener('submit', async event => {
|
||||
event.preventDefault();
|
||||
const message = prompt.value.trim();
|
||||
if (!message) return;
|
||||
prompt.value = '';
|
||||
send.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/diagnostics/settings-and-logs/chat', {
|
||||
method: 'POST',
|
||||
headers: {'content-type': 'application/json'},
|
||||
body: JSON.stringify({message})
|
||||
});
|
||||
const body = await response.json();
|
||||
render(body.messages);
|
||||
} finally {
|
||||
send.disabled = false;
|
||||
prompt.focus();
|
||||
}
|
||||
});
|
||||
prompt.addEventListener('keydown', event => {
|
||||
if (event.key === 'Enter' && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
form.requestSubmit();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
""";
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||
<MicrosoftSpeechVersion>1.51.1</MicrosoftSpeechVersion>
|
||||
<MicrosoftSpeechVersion>1.51.2</MicrosoftSpeechVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
@@ -15,21 +15,42 @@
|
||||
<UseWPF>true</UseWPF>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="$([MSBuild]::IsOSPlatform('OSX')) and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||
<MacOsNativeHelpersEnabled>true</MacOsNativeHelpersEnabled>
|
||||
<MacOsAudioCaptureSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/main.swift</MacOsAudioCaptureSource>
|
||||
<MacOsAudioCaptureInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/Info.plist</MacOsAudioCaptureInfoPlist>
|
||||
<MacOsAudioCaptureBundlePath>Native/MeetingAssistantAudioCapture.app</MacOsAudioCaptureBundlePath>
|
||||
<MacOsAudioCaptureOutputPath>$(MacOsAudioCaptureBundlePath)/Contents/MacOS/macos-meeting-audio-capture</MacOsAudioCaptureOutputPath>
|
||||
<MacOsAudioCaptureRid Condition="'$(RuntimeIdentifier)' != ''">$(RuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||
<MacOsAudioCaptureRid Condition="'$(MacOsAudioCaptureRid)' == ''">$(NETCoreSdkRuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureRid)' == 'osx-x64'">x86_64</MacOsAudioCaptureArchitecture>
|
||||
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureArchitecture)' == ''">arm64</MacOsAudioCaptureArchitecture>
|
||||
<MacOsDesktopControlsSource>$(MSBuildProjectDirectory)/Native/MacOsDesktopControls/main.swift</MacOsDesktopControlsSource>
|
||||
<MacOsDesktopControlsOutputPath>Native/macos-desktop-controls</MacOsDesktopControlsOutputPath>
|
||||
<MacOsMeetingIntegrationsSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/main.swift</MacOsMeetingIntegrationsSource>
|
||||
<MacOsMeetingIntegrationsInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsMeetingIntegrations/Info.plist</MacOsMeetingIntegrationsInfoPlist>
|
||||
<MacOsMeetingIntegrationsOutputPath>Native/macos-meeting-integrations</MacOsMeetingIntegrationsOutputPath>
|
||||
<MacOsLauncherSource>$(MSBuildProjectDirectory)/Native/MacOsLauncher/main.swift</MacOsLauncherSource>
|
||||
<MacOsLauncherInfoPlist>$(MSBuildProjectDirectory)/Native/MacOsLauncher/Info.plist</MacOsLauncherInfoPlist>
|
||||
<MacOsLauncherOutputPath>Native/macos-meeting-assistant-launcher</MacOsLauncherOutputPath>
|
||||
<MacOsApplicationBundlePath>MeetingAssistant.app</MacOsApplicationBundlePath>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.17.0" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
|
||||
<PackageReference Include="DiffPlex" Version="1.9.0" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
|
||||
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
|
||||
<PackageReference Include="NAudio" Version="2.3.0" />
|
||||
<PackageReference Include="NCalcSync" Version="7.0.2" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
|
||||
<PackageReference Include="NAudio" Version="3.1.0" />
|
||||
<PackageReference Include="NCalcSync" Version="7.2.0" />
|
||||
<PackageReference Include="RazorLight" Version="2.3.1" />
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.10" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
|
||||
<PackageReference Include="Whisper.net" Version="1.9.1" />
|
||||
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
|
||||
<PackageReference Include="YamlDotNet" Version="18.1.0" />
|
||||
@@ -37,7 +58,7 @@
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
<PackageReference Include="CommunityToolkit.WinUI.Notifications" Version="7.1.2" />
|
||||
<PackageReference Include="H.NotifyIcon.Uno.WinUI" Version="2.4.1" />
|
||||
<PackageReference Include="H.NotifyIcon" Version="2.4.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -47,6 +68,7 @@
|
||||
<None Include="..\docs\meeting-assistant-configuration.md" Link="docs\meeting-assistant-configuration.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="Content\Project-AGENTS.md" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="..\openspec\specs\**\*.md" Link="openspec\specs\%(RecursiveDir)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<None Include="Native\MacOsMeetingIntegrations\calendar-events.js" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" TargetPath="Native\macos-calendar-events.js" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
|
||||
@@ -61,6 +83,52 @@
|
||||
<Copy SourceFiles="@(MeetingAssistantRootConfig)" DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="BuildMacOsNativeHelpers"
|
||||
AfterTargets="Build"
|
||||
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
|
||||
<MakeDir Directories="$(TargetDir)Native" />
|
||||
<MakeDir Directories="$(TargetDir)$(MacOsAudioCaptureBundlePath)/Contents/MacOS" />
|
||||
<Copy SourceFiles="$(MacOsAudioCaptureInfoPlist)" DestinationFiles="$(TargetDir)$(MacOsAudioCaptureBundlePath)/Contents/Info.plist" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AVFoundation -framework CoreMedia -framework ScreenCaptureKit -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$(MacOsAudioCaptureInfoPlist)" "$(MacOsAudioCaptureSource)" -o "$(TargetDir)$(MacOsAudioCaptureOutputPath)"" />
|
||||
<Exec Command="/usr/bin/codesign --force --deep --sign - "$(TargetDir)$(MacOsAudioCaptureBundlePath)"" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework Carbon -framework WebKit "$(MacOsDesktopControlsSource)" -o "$(TargetDir)$(MacOsDesktopControlsOutputPath)"" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AppKit -framework CoreGraphics -framework EventKit -framework ImageIO -framework UniformTypeIdentifiers -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$(MacOsMeetingIntegrationsInfoPlist)" "$(MacOsMeetingIntegrationsSource)" -o "$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)"" />
|
||||
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 "$(MacOsLauncherSource)" -o "$(TargetDir)$(MacOsLauncherOutputPath)"" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="PackageMacOsApplication"
|
||||
AfterTargets="Publish"
|
||||
DependsOnTargets="CopyMeetingAssistantConfigToPublishRoot;CopyMacOsNativeHelpersToPublish"
|
||||
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
|
||||
<RemoveDir Directories="$(PublishDir)$(MacOsApplicationBundlePath)" />
|
||||
<MakeDir Directories="$(PublishDir)$(MacOsApplicationBundlePath)/Contents/MacOS" />
|
||||
<MakeDir Directories="$(PublishDir)$(MacOsApplicationBundlePath)/Contents/Resources/app" />
|
||||
<ItemGroup>
|
||||
<MacOsPublishedRuntimeFiles Include="$(PublishDir)**/*" Exclude="$(PublishDir)$(MacOsApplicationBundlePath)/**/*" />
|
||||
</ItemGroup>
|
||||
<Copy
|
||||
SourceFiles="@(MacOsPublishedRuntimeFiles)"
|
||||
DestinationFiles="@(MacOsPublishedRuntimeFiles->'$(PublishDir)$(MacOsApplicationBundlePath)/Contents/Resources/app/%(RecursiveDir)%(Filename)%(Extension)')" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsLauncherOutputPath)" DestinationFiles="$(PublishDir)$(MacOsApplicationBundlePath)/Contents/MacOS/MeetingAssistant" />
|
||||
<Copy SourceFiles="$(MacOsLauncherInfoPlist)" DestinationFiles="$(PublishDir)$(MacOsApplicationBundlePath)/Contents/Info.plist" />
|
||||
<Exec Command="/usr/bin/codesign --force --deep --sign - --requirements '=designated => identifier "cloud.schweigert.meeting-assistant"' "$(PublishDir)$(MacOsApplicationBundlePath)"" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CopyMacOsNativeHelpersToPublish"
|
||||
AfterTargets="Publish"
|
||||
Condition="'$(MacOsNativeHelpersEnabled)' == 'true'">
|
||||
<MakeDir Directories="$(PublishDir)Native" />
|
||||
<MakeDir Directories="$(PublishDir)$(MacOsAudioCaptureBundlePath)/Contents/MacOS" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsAudioCaptureOutputPath)" DestinationFiles="$(PublishDir)$(MacOsAudioCaptureOutputPath)" />
|
||||
<Copy SourceFiles="$(MacOsAudioCaptureInfoPlist)" DestinationFiles="$(PublishDir)$(MacOsAudioCaptureBundlePath)/Contents/Info.plist" />
|
||||
<Exec Command="/usr/bin/codesign --force --deep --sign - "$(PublishDir)$(MacOsAudioCaptureBundlePath)"" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsDesktopControlsOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||
<Copy SourceFiles="$(TargetDir)$(MacOsMeetingIntegrationsOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||
</Target>
|
||||
|
||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||
|
||||
@@ -165,6 +165,8 @@ public sealed class RecordingInactivitySafeguardOptions
|
||||
|
||||
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
|
||||
|
||||
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
|
||||
|
||||
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
|
||||
@@ -181,10 +183,8 @@ public sealed class WhisperLocalOptions
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions
|
||||
public class PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string DockerCommand { get; set; } = "docker";
|
||||
|
||||
public string BaseImage { get; set; } = "python:3.11-slim";
|
||||
@@ -208,6 +208,11 @@ public sealed class PyannoteDiarizationOptions
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
||||
}
|
||||
|
||||
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
|
||||
public sealed class AzureSpeechOptions
|
||||
{
|
||||
public string Endpoint { get; set; } = "";
|
||||
@@ -340,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
|
||||
|
||||
public int MaxSnippetsPerSpeaker { get; set; } = 3;
|
||||
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
|
||||
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
|
||||
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
|
||||
|
||||
@@ -355,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
|
||||
public AzureSpeechOptions AzureSpeech { get; set; } = new();
|
||||
|
||||
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
|
||||
|
||||
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerSpeakerRecognitionOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public int RequiredVectorsPerSpeaker { get; set; } = 5;
|
||||
|
||||
public int MaxVectorsPerIdentity { get; set; } = 1000;
|
||||
|
||||
public int OutlierPruningMinimumVectors { get; set; } = 20;
|
||||
|
||||
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
|
||||
|
||||
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
|
||||
|
||||
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
|
||||
|
||||
public double MinimumClusterCohesion { get; set; } = 0.75;
|
||||
|
||||
public double MinimumIdentitySimilarity { get; set; } = 0.75;
|
||||
|
||||
public double MinimumSimilarityMargin { get; set; } = 0.05;
|
||||
|
||||
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
|
||||
|
||||
public string PackageVersion { get; set; } = "0.1.4";
|
||||
|
||||
public string PythonCommand { get; set; } = "python";
|
||||
|
||||
public string TorchVersion { get; set; } = "2.14.0+cpu";
|
||||
|
||||
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
|
||||
|
||||
public string WebRtcVadVersion { get; set; } = "2.0.14";
|
||||
|
||||
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
|
||||
|
||||
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
|
||||
}
|
||||
|
||||
public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
@@ -365,9 +411,8 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
|
||||
|
||||
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
|
||||
|
||||
public PyannoteDiarizationOptions Diarization { get; set; } = new()
|
||||
public PyannoteRuntimeOptions Diarization { get; set; } = new()
|
||||
{
|
||||
Enabled = true,
|
||||
CommandTimeout = TimeSpan.FromHours(1),
|
||||
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
|
||||
};
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
using MeetingAssistant.Calendar;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
|
||||
internal static class CalendarAppointmentMetadata
|
||||
{
|
||||
public static MeetingMetadata Create(
|
||||
string? subject,
|
||||
IEnumerable<string> attendees,
|
||||
string? body,
|
||||
DateTimeOffset end)
|
||||
{
|
||||
return new MeetingMetadata(
|
||||
ReadSubject(subject),
|
||||
NormalizeAttendees(attendees),
|
||||
ExtractAgenda(body ?? ""),
|
||||
end);
|
||||
}
|
||||
|
||||
public static string ReadSubject(string? subject)
|
||||
{
|
||||
var trimmed = subject?.Trim();
|
||||
return string.IsNullOrWhiteSpace(trimmed) ? "Teams meeting" : trimmed;
|
||||
}
|
||||
|
||||
public static bool IsTeamsAppointment(
|
||||
string? subject,
|
||||
string? location,
|
||||
string? body,
|
||||
string? url = null)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(
|
||||
subject ?? "",
|
||||
location ?? "",
|
||||
string.Join(Environment.NewLine, body, url));
|
||||
}
|
||||
|
||||
public static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || TeamsMeetingMarkerDetector.ContainsTeamsMarker(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim()
|
||||
.Replace("mailto:", "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
return emailStart > 0 && emailEnd > emailStart
|
||||
? attendee[(emailStart + 1)..emailEnd].Trim()
|
||||
: attendee.Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
|
||||
internal static class MeetingMetadataDiagnosticRunner
|
||||
{
|
||||
public static async Task<MeetingMetadataDiagnosticResult> RunAsync(
|
||||
string provider,
|
||||
DateTimeOffset startedAt,
|
||||
TimeSpan timeout,
|
||||
Func<CancellationToken, Task<MeetingMetadata?>> lookupAsync,
|
||||
string noMatchError,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var lookup = lookupAsync(cancellationToken);
|
||||
var metadata = timeout > TimeSpan.Zero
|
||||
? await lookup.WaitAsync(timeout, cancellationToken)
|
||||
: await lookup.WaitAsync(cancellationToken);
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
metadata is not null,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
metadata,
|
||||
metadata is null ? noMatchError : null);
|
||||
}
|
||||
catch (TimeoutException exception)
|
||||
{
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
false,
|
||||
true,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
provider,
|
||||
startedAt,
|
||||
false,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,18 +7,17 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
{
|
||||
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
|
||||
{
|
||||
var title = ReadSubject(appointment);
|
||||
return new MeetingMetadata(
|
||||
title,
|
||||
return CalendarAppointmentMetadata.Create(
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")),
|
||||
ReadAttendees(appointment),
|
||||
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")),
|
||||
OutlookClassicCom.ToLocalOffset(end));
|
||||
}
|
||||
|
||||
public static string ReadSubject(object appointment)
|
||||
{
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
|
||||
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
|
||||
return CalendarAppointmentMetadata.ReadSubject(
|
||||
Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")));
|
||||
}
|
||||
|
||||
public static bool IsTeamsAppointment(object appointment)
|
||||
@@ -26,29 +25,12 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
|
||||
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
|
||||
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
|
||||
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
|
||||
return CalendarAppointmentMetadata.IsTeamsAppointment(subject, location, body);
|
||||
}
|
||||
|
||||
public static string ExtractAgenda(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
|
||||
var agendaLines = new List<string>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
agendaLines.Add(line);
|
||||
}
|
||||
|
||||
return string.Join(Environment.NewLine, agendaLines).Trim();
|
||||
return CalendarAppointmentMetadata.ExtractAgenda(body);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> ReadAttendees(object appointment)
|
||||
@@ -93,66 +75,7 @@ internal static class OutlookClassicAppointmentMetadata
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => group
|
||||
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
|
||||
.ThenBy(attendee => attendee.Length)
|
||||
.First())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool IsTeamsSeparator(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
return trimmed.Length >= 8 &&
|
||||
trimmed.All(character => character is '_' or '-' or '*' or ' ');
|
||||
}
|
||||
|
||||
private static bool IsTeamsJoinLine(string line)
|
||||
{
|
||||
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
var normalized = attendee.Trim();
|
||||
var mailtoPrefix = "mailto:";
|
||||
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = normalized.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
var name = normalized[..emailStart].Trim();
|
||||
var email = normalized[(emailStart + 1)..emailEnd].Trim();
|
||||
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return email;
|
||||
}
|
||||
|
||||
return $"{name} <{email}>";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static string GetAttendeeDeduplicationKey(string attendee)
|
||||
{
|
||||
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
|
||||
var emailEnd = attendee.LastIndexOf('>');
|
||||
if (emailStart > 0 && emailEnd > emailStart)
|
||||
{
|
||||
return attendee[(emailStart + 1)..emailEnd].Trim();
|
||||
}
|
||||
|
||||
return attendee.Trim();
|
||||
return CalendarAppointmentMetadata.NormalizeAttendees(attendees);
|
||||
}
|
||||
|
||||
private static string FormatRecipient(object recipient)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Diagnostics;
|
||||
using MeetingAssistant.Calendar;
|
||||
|
||||
namespace MeetingAssistant.MeetingNotes;
|
||||
@@ -36,54 +35,16 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var lookup = OutlookClassicCom.RunStaAsync(
|
||||
return await MeetingMetadataDiagnosticRunner.RunAsync(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
timeout,
|
||||
token => OutlookClassicCom.RunStaAsync(
|
||||
() => GetCurrentMeeting(startedAt),
|
||||
cancellationToken,
|
||||
"Meeting Assistant Outlook COM Lookup");
|
||||
var metadata = timeout > TimeSpan.Zero
|
||||
? await lookup.WaitAsync(timeout, cancellationToken)
|
||||
: await lookup.WaitAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
metadata is not null,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
metadata,
|
||||
metadata is null ? "No matching Teams appointment was found." : null);
|
||||
}
|
||||
catch (TimeoutException exception)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
false,
|
||||
true,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new MeetingMetadataDiagnosticResult(
|
||||
nameof(OutlookClassicMeetingMetadataProvider),
|
||||
startedAt,
|
||||
false,
|
||||
false,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
exception.Message);
|
||||
}
|
||||
token,
|
||||
"Meeting Assistant Outlook COM Lookup"),
|
||||
"No matching Teams appointment was found.",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private MeetingMetadata? GetCurrentMeeting(DateTimeOffset startedAt)
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import AppKit
|
||||
import Carbon
|
||||
import Foundation
|
||||
import WebKit
|
||||
|
||||
struct Manifest: Decodable {
|
||||
let baseUrl: String
|
||||
let statusUrl: String
|
||||
let openAgentUrl: String
|
||||
let profiles: [Profile]
|
||||
let hotkeys: [Hotkey]
|
||||
}
|
||||
|
||||
struct Profile: Decodable {
|
||||
let name: String
|
||||
let toggleShortcut: String
|
||||
}
|
||||
|
||||
struct Hotkey: Decodable {
|
||||
let shortcut: String
|
||||
let endpoint: String
|
||||
}
|
||||
|
||||
struct RecordingStatus: Decodable {
|
||||
let isRecording: Bool
|
||||
let state: String
|
||||
let launchProfile: String?
|
||||
}
|
||||
|
||||
@main
|
||||
final class DesktopControls: NSObject, NSApplicationDelegate, NSMenuDelegate {
|
||||
private let manifest: Manifest
|
||||
private var status = RecordingStatus(isRecording: false, state: "Idle", launchProfile: nil)
|
||||
private var statusItem: NSStatusItem!
|
||||
private var registeredHotkeys: [UInt32: Hotkey] = [:]
|
||||
private var timer: Timer?
|
||||
private var agentWindow: NSWindow?
|
||||
|
||||
override init() {
|
||||
guard CommandLine.arguments.count == 2,
|
||||
let data = try? Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[1])),
|
||||
let decoded = try? JSONDecoder().decode(Manifest.self, from: data) else {
|
||||
fputs("Usage: macos-desktop-controls <manifest.json>\n", stderr)
|
||||
exit(2)
|
||||
}
|
||||
manifest = decoded
|
||||
super.init()
|
||||
}
|
||||
|
||||
static func main() {
|
||||
let application = NSApplication.shared
|
||||
let delegate = DesktopControls()
|
||||
application.delegate = delegate
|
||||
application.setActivationPolicy(.accessory)
|
||||
application.run()
|
||||
}
|
||||
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.squareLength)
|
||||
statusItem.button?.image = NSImage(
|
||||
systemSymbolName: "waveform.circle",
|
||||
accessibilityDescription: "Meeting Assistant")
|
||||
let menu = NSMenu()
|
||||
menu.delegate = self
|
||||
statusItem.menu = menu
|
||||
registerHotkeys()
|
||||
refreshStatus()
|
||||
timer = Timer.scheduledTimer(withTimeInterval: 2, repeats: true) { [weak self] _ in
|
||||
self?.refreshStatus()
|
||||
}
|
||||
}
|
||||
|
||||
func applicationWillTerminate(_ notification: Notification) {
|
||||
timer?.invalidate()
|
||||
}
|
||||
|
||||
func menuWillOpen(_ menu: NSMenu) {
|
||||
rebuildMenu(menu)
|
||||
refreshStatus()
|
||||
}
|
||||
|
||||
private func rebuildMenu(_ menu: NSMenu) {
|
||||
menu.removeAllItems()
|
||||
let stateItem = NSMenuItem(title: tooltip, action: nil, keyEquivalent: "")
|
||||
stateItem.isEnabled = false
|
||||
menu.addItem(stateItem)
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(item("Open agent", #selector(openAgent)))
|
||||
|
||||
if status.isRecording {
|
||||
menu.addItem(item("Stop meeting recording and transcribe", #selector(stopRecording)))
|
||||
menu.addItem(item("Cancel meeting recording and discard", #selector(abortRecording)))
|
||||
} else {
|
||||
for (index, profile) in manifest.profiles.enumerated() {
|
||||
let entry = item(
|
||||
"Start meeting recording (\(profile.name))",
|
||||
#selector(startProfile(_:)))
|
||||
entry.tag = index
|
||||
entry.keyEquivalent = menuKey(from: profile.toggleShortcut)
|
||||
entry.keyEquivalentModifierMask = menuModifiers(from: profile.toggleShortcut)
|
||||
menu.addItem(entry)
|
||||
}
|
||||
}
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(item("Exit", #selector(exitApplication)))
|
||||
}
|
||||
|
||||
private var tooltip: String {
|
||||
if status.isRecording {
|
||||
return status.launchProfile.map { "Meeting Assistant — recording (\($0))" }
|
||||
?? "Meeting Assistant — recording"
|
||||
}
|
||||
return status.state.lowercased() == "summarizing"
|
||||
? "Meeting Assistant — summarizing"
|
||||
: "Meeting Assistant — idle"
|
||||
}
|
||||
|
||||
private func item(_ title: String, _ action: Selector) -> NSMenuItem {
|
||||
let result = NSMenuItem(title: title, action: action, keyEquivalent: "")
|
||||
result.target = self
|
||||
return result
|
||||
}
|
||||
|
||||
@objc private func openAgent() {
|
||||
if agentWindow == nil {
|
||||
let webView = WKWebView(frame: .zero)
|
||||
let window = NSWindow(
|
||||
contentRect: NSRect(x: 0, y: 0, width: 840, height: 760),
|
||||
styleMask: [.titled, .closable, .miniaturizable, .resizable],
|
||||
backing: .buffered,
|
||||
defer: false)
|
||||
window.title = "Meeting Summary Agent"
|
||||
window.contentView = webView
|
||||
window.center()
|
||||
window.isReleasedWhenClosed = false
|
||||
webView.load(URLRequest(url: URL(string: manifest.openAgentUrl)!))
|
||||
agentWindow = window
|
||||
}
|
||||
NSApplication.shared.activate(ignoringOtherApps: true)
|
||||
agentWindow?.makeKeyAndOrderFront(nil)
|
||||
}
|
||||
|
||||
@objc private func startProfile(_ sender: NSMenuItem) {
|
||||
let profile = manifest.profiles[sender.tag]
|
||||
post(path: "/profiles/\(escaped(profile.name))/recording/start")
|
||||
}
|
||||
|
||||
@objc private func stopRecording() {
|
||||
post(path: "/recording/stop")
|
||||
}
|
||||
|
||||
@objc private func abortRecording() {
|
||||
post(path: "/recording/abort")
|
||||
}
|
||||
|
||||
@objc private func exitApplication() {
|
||||
post(path: "/application/exit")
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) {
|
||||
NSApplication.shared.terminate(nil)
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshStatus() {
|
||||
guard let url = URL(string: manifest.statusUrl) else { return }
|
||||
URLSession.shared.dataTask(with: url) { [weak self] data, _, _ in
|
||||
guard let data,
|
||||
let next = try? JSONDecoder().decode(RecordingStatus.self, from: data) else { return }
|
||||
DispatchQueue.main.async {
|
||||
self?.status = next
|
||||
self?.statusItem.button?.image = NSImage(
|
||||
systemSymbolName: next.isRecording ? "record.circle.fill" : "waveform.circle",
|
||||
accessibilityDescription: self?.tooltip)
|
||||
}
|
||||
}.resume()
|
||||
}
|
||||
|
||||
private func post(path: String) {
|
||||
post(url: manifest.baseUrl + path)
|
||||
}
|
||||
|
||||
private func post(url value: String) {
|
||||
guard let url = URL(string: value) else { return }
|
||||
var request = URLRequest(url: url)
|
||||
request.httpMethod = "POST"
|
||||
URLSession.shared.dataTask(with: request) { [weak self] _, _, _ in
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) {
|
||||
self?.refreshStatus()
|
||||
}
|
||||
}.resume()
|
||||
}
|
||||
|
||||
private func registerHotkeys() {
|
||||
var eventType = EventTypeSpec(
|
||||
eventClass: OSType(kEventClassKeyboard),
|
||||
eventKind: OSType(kEventHotKeyPressed))
|
||||
InstallEventHandler(
|
||||
GetApplicationEventTarget(),
|
||||
{ _, event, context in
|
||||
guard let event, let context else { return noErr }
|
||||
var identifier = EventHotKeyID()
|
||||
GetEventParameter(
|
||||
event,
|
||||
EventParamName(kEventParamDirectObject),
|
||||
EventParamType(typeEventHotKeyID),
|
||||
nil,
|
||||
MemoryLayout<EventHotKeyID>.size,
|
||||
nil,
|
||||
&identifier)
|
||||
let owner = Unmanaged<DesktopControls>.fromOpaque(context).takeUnretainedValue()
|
||||
if let hotkey = owner.registeredHotkeys[identifier.id] {
|
||||
owner.post(path: hotkey.endpoint)
|
||||
}
|
||||
return noErr
|
||||
},
|
||||
1,
|
||||
&eventType,
|
||||
Unmanaged.passUnretained(self).toOpaque(),
|
||||
nil)
|
||||
|
||||
for (index, hotkey) in manifest.hotkeys.enumerated() {
|
||||
guard let keyCode = carbonKeyCode(from: hotkey.shortcut) else { continue }
|
||||
let identifier = EventHotKeyID(signature: fourCC("MTGA"), id: UInt32(index + 1))
|
||||
var reference: EventHotKeyRef?
|
||||
let result = RegisterEventHotKey(
|
||||
keyCode,
|
||||
carbonModifiers(from: hotkey.shortcut),
|
||||
identifier,
|
||||
GetApplicationEventTarget(),
|
||||
0,
|
||||
&reference)
|
||||
if result == noErr {
|
||||
registeredHotkeys[identifier.id] = hotkey
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func escaped(_ value: String) -> String {
|
||||
value.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? value
|
||||
}
|
||||
}
|
||||
|
||||
private func tokens(_ shortcut: String) -> [String] {
|
||||
shortcut.uppercased().split(separator: "+").map(String.init)
|
||||
}
|
||||
|
||||
private func carbonModifiers(from shortcut: String) -> UInt32 {
|
||||
let parts = tokens(shortcut)
|
||||
var result: UInt32 = 0
|
||||
if parts.contains("CTRL") || parts.contains("CONTROL") { result |= UInt32(controlKey) }
|
||||
if parts.contains("ALT") || parts.contains("OPTION") { result |= UInt32(optionKey) }
|
||||
if parts.contains("SHIFT") { result |= UInt32(shiftKey) }
|
||||
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
|
||||
result |= UInt32(cmdKey)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private func menuModifiers(from shortcut: String) -> NSEvent.ModifierFlags {
|
||||
let parts = tokens(shortcut)
|
||||
var result: NSEvent.ModifierFlags = []
|
||||
if parts.contains("CTRL") || parts.contains("CONTROL") { result.insert(.control) }
|
||||
if parts.contains("ALT") || parts.contains("OPTION") { result.insert(.option) }
|
||||
if parts.contains("SHIFT") { result.insert(.shift) }
|
||||
if parts.contains("WIN") || parts.contains("WINDOWS") || parts.contains("CMD") || parts.contains("COMMAND") {
|
||||
result.insert(.command)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private func menuKey(from shortcut: String) -> String {
|
||||
tokens(shortcut).last?.lowercased() ?? ""
|
||||
}
|
||||
|
||||
private func carbonKeyCode(from shortcut: String) -> UInt32? {
|
||||
guard let key = tokens(shortcut).last, key.count == 1, let scalar = key.unicodeScalars.first else {
|
||||
return nil
|
||||
}
|
||||
let codes: [Character: UInt32] = [
|
||||
"A": 0, "S": 1, "D": 2, "F": 3, "H": 4, "G": 5, "Z": 6, "X": 7,
|
||||
"C": 8, "V": 9, "B": 11, "Q": 12, "W": 13, "E": 14, "R": 15,
|
||||
"Y": 16, "T": 17, "M": 46, "L": 37, "N": 45, "P": 35, "O": 31,
|
||||
"I": 34, "U": 32, "J": 38, "K": 40
|
||||
]
|
||||
return codes[Character(String(scalar))]
|
||||
}
|
||||
|
||||
private func fourCC(_ value: String) -> OSType {
|
||||
value.utf8.reduce(0) { ($0 << 8) + OSType($1) }
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>MeetingAssistant</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>cloud.schweigert.meeting-assistant</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>Meeting Assistant</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSUIElement</key>
|
||||
<true/>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Meeting Assistant reads calendar metadata for meeting notes and recording prompts.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Meeting Assistant records microphone audio for live meeting transcription.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,70 @@
|
||||
import Darwin
|
||||
import Dispatch
|
||||
import Foundation
|
||||
|
||||
@main
|
||||
private struct MeetingAssistantLauncher {
|
||||
private static let serviceDirectoryName = "MeetingAssistant"
|
||||
|
||||
static func main() {
|
||||
let runtimeDirectory = Bundle.main.bundleURL
|
||||
.appendingPathComponent("Contents/Resources/app", isDirectory: true)
|
||||
let assemblyPath = runtimeDirectory.appendingPathComponent("MeetingAssistant.dll").path
|
||||
let environmentFile = FileManager.default.homeDirectoryForCurrentUser
|
||||
.appendingPathComponent("Library/Application Support")
|
||||
.appendingPathComponent(serviceDirectoryName)
|
||||
.appendingPathComponent("config/meeting-assistant.env")
|
||||
|
||||
guard FileManager.default.fileExists(atPath: assemblyPath) else {
|
||||
fail("Meeting Assistant runtime not found at \(assemblyPath)")
|
||||
}
|
||||
|
||||
guard FileManager.default.fileExists(atPath: environmentFile.path) else {
|
||||
fail("Meeting Assistant environment file not found at \(environmentFile.path)")
|
||||
}
|
||||
|
||||
let process = Process()
|
||||
process.executableURL = URL(fileURLWithPath: "/bin/zsh")
|
||||
process.arguments = [
|
||||
"-c",
|
||||
"set -a; source \"$1\"; set +a; exec /usr/local/share/dotnet/dotnet \"$2\" --contentRoot \"$3\"",
|
||||
"meeting-assistant-launcher",
|
||||
environmentFile.path,
|
||||
assemblyPath,
|
||||
runtimeDirectory.path
|
||||
]
|
||||
process.currentDirectoryURL = runtimeDirectory
|
||||
|
||||
signal(SIGTERM, SIG_IGN)
|
||||
signal(SIGINT, SIG_IGN)
|
||||
|
||||
let terminationSignal = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .global())
|
||||
terminationSignal.setEventHandler {
|
||||
if process.isRunning {
|
||||
process.terminate()
|
||||
}
|
||||
}
|
||||
terminationSignal.resume()
|
||||
|
||||
let interruptSignal = DispatchSource.makeSignalSource(signal: SIGINT, queue: .global())
|
||||
interruptSignal.setEventHandler {
|
||||
if process.isRunning {
|
||||
kill(process.processIdentifier, SIGINT)
|
||||
}
|
||||
}
|
||||
interruptSignal.resume()
|
||||
|
||||
do {
|
||||
try process.run()
|
||||
process.waitUntilExit()
|
||||
Darwin.exit(process.terminationStatus)
|
||||
} catch {
|
||||
fail("Meeting Assistant launcher failed: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
private static func fail(_ message: String) -> Never {
|
||||
FileHandle.standardError.write(Data("\(message)\n".utf8))
|
||||
Darwin.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>macos-meeting-audio-capture</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>cloud.schweigert.meeting-assistant.audio-capture</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>Meeting Assistant Audio Capture</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSUIElement</key>
|
||||
<true/>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Meeting Assistant records microphone audio for live meeting transcription.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,328 @@
|
||||
import AVFoundation
|
||||
import CoreMedia
|
||||
import Darwin
|
||||
import Foundation
|
||||
import ScreenCaptureKit
|
||||
|
||||
private enum CaptureFailure: Error, CustomStringConvertible {
|
||||
case usage
|
||||
case invalidFormat
|
||||
case microphonePermission
|
||||
case noDisplay
|
||||
case audioConverter(String)
|
||||
case systemAudio(String)
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .usage:
|
||||
return "Usage: macos-meeting-audio-capture <microphone|system> <sample-rate> <channels>"
|
||||
case .invalidFormat:
|
||||
return "Sample rate must be positive and channels must be 1 or 2."
|
||||
case .microphonePermission:
|
||||
return "Microphone permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Microphone."
|
||||
case .noDisplay:
|
||||
return "ScreenCaptureKit could not find a display for system-audio capture."
|
||||
case .audioConverter(let detail):
|
||||
return "Native PCM conversion failed: \(detail)"
|
||||
case .systemAudio(let detail):
|
||||
return "Screen Recording/System Audio permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Screen & System Audio Recording. \(detail)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final class ErrorReporter: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
|
||||
func write(_ message: String) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
let line = message.hasSuffix("\n") ? message : message + "\n"
|
||||
if let data = line.data(using: .utf8) {
|
||||
try? FileHandle.standardError.write(contentsOf: data)
|
||||
}
|
||||
}
|
||||
|
||||
func exit(with failure: CaptureFailure, code: Int32) -> Never {
|
||||
write(failure.description)
|
||||
Darwin.exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
private final class PcmWriter: @unchecked Sendable {
|
||||
private let targetFormat: AVAudioFormat
|
||||
private let output = FileHandle.standardOutput
|
||||
private let reporter: ErrorReporter
|
||||
private let lock = NSLock()
|
||||
private var converter: AVAudioConverter?
|
||||
|
||||
init(sampleRate: Int, channels: Int, reporter: ErrorReporter) throws {
|
||||
guard let format = AVAudioFormat(
|
||||
commonFormat: .pcmFormatInt16,
|
||||
sampleRate: Double(sampleRate),
|
||||
channels: AVAudioChannelCount(channels),
|
||||
interleaved: true
|
||||
) else {
|
||||
throw CaptureFailure.invalidFormat
|
||||
}
|
||||
|
||||
targetFormat = format
|
||||
self.reporter = reporter
|
||||
}
|
||||
|
||||
func write(_ input: AVAudioPCMBuffer) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
|
||||
do {
|
||||
let outputBuffer = try convert(input)
|
||||
let buffers = UnsafeMutableAudioBufferListPointer(outputBuffer.mutableAudioBufferList)
|
||||
guard let audioBuffer = buffers.first, let bytes = audioBuffer.mData else {
|
||||
return
|
||||
}
|
||||
|
||||
let data = Data(bytes: bytes, count: Int(audioBuffer.mDataByteSize))
|
||||
try output.write(contentsOf: data)
|
||||
} catch {
|
||||
reporter.exit(with: .audioConverter(error.localizedDescription), code: 70)
|
||||
}
|
||||
}
|
||||
|
||||
private func convert(_ input: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer {
|
||||
let sourceFormat = input.format
|
||||
let activeConverter: AVAudioConverter
|
||||
if let converter {
|
||||
activeConverter = converter
|
||||
} else {
|
||||
guard let created = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
|
||||
throw CaptureFailure.audioConverter("AVAudioConverter could not convert \(sourceFormat) to \(targetFormat).")
|
||||
}
|
||||
converter = created
|
||||
activeConverter = created
|
||||
}
|
||||
|
||||
let ratio = targetFormat.sampleRate / sourceFormat.sampleRate
|
||||
let capacity = max(1, AVAudioFrameCount(ceil(Double(input.frameLength) * ratio)) + 32)
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
|
||||
throw CaptureFailure.audioConverter("Could not allocate the target PCM buffer.")
|
||||
}
|
||||
|
||||
var suppliedInput = false
|
||||
var conversionError: NSError?
|
||||
let status = activeConverter.convert(to: converted, error: &conversionError) { _, inputStatus in
|
||||
if suppliedInput {
|
||||
inputStatus.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
|
||||
suppliedInput = true
|
||||
inputStatus.pointee = .haveData
|
||||
return input
|
||||
}
|
||||
|
||||
if status == .error {
|
||||
throw conversionError ?? CaptureFailure.audioConverter("AVAudioConverter returned an unknown error.")
|
||||
}
|
||||
|
||||
return converted
|
||||
}
|
||||
}
|
||||
|
||||
private final class MicrophoneCapture: @unchecked Sendable {
|
||||
private let engine = AVAudioEngine()
|
||||
private let writer: PcmWriter
|
||||
|
||||
init(writer: PcmWriter) {
|
||||
self.writer = writer
|
||||
}
|
||||
|
||||
func run() async throws -> Never {
|
||||
let authorized = await withCheckedContinuation { continuation in
|
||||
AVCaptureDevice.requestAccess(for: .audio) { granted in
|
||||
continuation.resume(returning: granted)
|
||||
}
|
||||
}
|
||||
guard authorized else {
|
||||
throw CaptureFailure.microphonePermission
|
||||
}
|
||||
|
||||
let input = engine.inputNode
|
||||
let inputFormat = input.outputFormat(forBus: 0)
|
||||
input.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [writer] buffer, _ in
|
||||
writer.write(buffer)
|
||||
}
|
||||
engine.prepare()
|
||||
try engine.start()
|
||||
|
||||
while true {
|
||||
try await Task.sleep(for: .seconds(3_600))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final class SystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
|
||||
private let writer: PcmWriter
|
||||
private let reporter: ErrorReporter
|
||||
private let sampleQueue = DispatchQueue(label: "meeting-assistant.system-audio")
|
||||
private var stream: SCStream?
|
||||
|
||||
init(writer: PcmWriter, reporter: ErrorReporter) {
|
||||
self.writer = writer
|
||||
self.reporter = reporter
|
||||
}
|
||||
|
||||
func run(sampleRate: Int, channels: Int) async throws -> Never {
|
||||
let content: SCShareableContent
|
||||
do {
|
||||
content = try await SCShareableContent.excludingDesktopWindows(
|
||||
false,
|
||||
onScreenWindowsOnly: false
|
||||
)
|
||||
} catch {
|
||||
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||
}
|
||||
|
||||
guard let display = content.displays.first else {
|
||||
throw CaptureFailure.noDisplay
|
||||
}
|
||||
|
||||
let filter = SCContentFilter(
|
||||
display: display,
|
||||
excludingApplications: [],
|
||||
exceptingWindows: []
|
||||
)
|
||||
let configuration = SCStreamConfiguration()
|
||||
configuration.capturesAudio = true
|
||||
configuration.excludesCurrentProcessAudio = false
|
||||
configuration.sampleRate = sampleRate
|
||||
configuration.channelCount = channels
|
||||
configuration.width = 2
|
||||
configuration.height = 2
|
||||
configuration.showsCursor = false
|
||||
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
|
||||
|
||||
let captureStream = SCStream(filter: filter, configuration: configuration, delegate: self)
|
||||
try captureStream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
|
||||
do {
|
||||
try await captureStream.startCapture()
|
||||
} catch {
|
||||
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||
}
|
||||
stream = captureStream
|
||||
|
||||
while true {
|
||||
try await Task.sleep(for: .seconds(3_600))
|
||||
}
|
||||
}
|
||||
|
||||
func stream(
|
||||
_ stream: SCStream,
|
||||
didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
|
||||
of outputType: SCStreamOutputType
|
||||
) {
|
||||
guard outputType == .audio,
|
||||
sampleBuffer.isValid,
|
||||
CMSampleBufferDataIsReady(sampleBuffer),
|
||||
let description = CMSampleBufferGetFormatDescription(sampleBuffer),
|
||||
let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description)
|
||||
else {
|
||||
return
|
||||
}
|
||||
|
||||
let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
|
||||
guard frameCount > 0,
|
||||
let sourceFormat = AVAudioFormat(streamDescription: streamDescription)
|
||||
else {
|
||||
return
|
||||
}
|
||||
|
||||
var bufferListSize = 0
|
||||
var retainedBlockBuffer: CMBlockBuffer?
|
||||
var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||
sampleBuffer,
|
||||
bufferListSizeNeededOut: &bufferListSize,
|
||||
bufferListOut: nil,
|
||||
bufferListSize: 0,
|
||||
blockBufferAllocator: nil,
|
||||
blockBufferMemoryAllocator: nil,
|
||||
flags: 0,
|
||||
blockBufferOut: &retainedBlockBuffer
|
||||
)
|
||||
guard status == noErr else {
|
||||
reporter.exit(
|
||||
with: .audioConverter("CoreMedia could not describe system audio (OSStatus \(status))."),
|
||||
code: 71
|
||||
)
|
||||
}
|
||||
|
||||
let rawBufferList = UnsafeMutableRawPointer.allocate(
|
||||
byteCount: bufferListSize,
|
||||
alignment: MemoryLayout<AudioBufferList>.alignment
|
||||
)
|
||||
defer { rawBufferList.deallocate() }
|
||||
let audioBufferList = rawBufferList.bindMemory(to: AudioBufferList.self, capacity: 1)
|
||||
status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||
sampleBuffer,
|
||||
bufferListSizeNeededOut: nil,
|
||||
bufferListOut: audioBufferList,
|
||||
bufferListSize: bufferListSize,
|
||||
blockBufferAllocator: nil,
|
||||
blockBufferMemoryAllocator: nil,
|
||||
flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment),
|
||||
blockBufferOut: &retainedBlockBuffer
|
||||
)
|
||||
guard status == noErr,
|
||||
let buffer = AVAudioPCMBuffer(
|
||||
pcmFormat: sourceFormat,
|
||||
bufferListNoCopy: audioBufferList,
|
||||
deallocator: nil
|
||||
)
|
||||
else {
|
||||
reporter.exit(
|
||||
with: .audioConverter("CoreMedia could not expose system audio (OSStatus \(status))."),
|
||||
code: 71
|
||||
)
|
||||
}
|
||||
|
||||
buffer.frameLength = frameCount
|
||||
writer.write(buffer)
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
||||
reporter.exit(with: .systemAudio(error.localizedDescription), code: 72)
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
private struct MacOsMeetingAudioCapture {
|
||||
static func main() async {
|
||||
let reporter = ErrorReporter()
|
||||
do {
|
||||
let arguments = CommandLine.arguments
|
||||
guard arguments.count == 4,
|
||||
let sampleRate = Int(arguments[2]),
|
||||
let channels = Int(arguments[3])
|
||||
else {
|
||||
throw CaptureFailure.usage
|
||||
}
|
||||
guard sampleRate > 0, channels == 1 || channels == 2 else {
|
||||
throw CaptureFailure.invalidFormat
|
||||
}
|
||||
|
||||
let writer = try PcmWriter(sampleRate: sampleRate, channels: channels, reporter: reporter)
|
||||
switch arguments[1] {
|
||||
case "microphone":
|
||||
try await MicrophoneCapture(writer: writer).run()
|
||||
case "system":
|
||||
try await SystemAudioCapture(writer: writer, reporter: reporter)
|
||||
.run(sampleRate: sampleRate, channels: channels)
|
||||
default:
|
||||
throw CaptureFailure.usage
|
||||
}
|
||||
} catch let failure as CaptureFailure {
|
||||
reporter.exit(with: failure, code: 64)
|
||||
} catch {
|
||||
reporter.write(error.localizedDescription)
|
||||
Darwin.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>cloud.schweigert.meeting-assistant.integrations</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>Meeting Assistant Integrations</string>
|
||||
<key>NSCalendarsUsageDescription</key>
|
||||
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Meeting Assistant reads calendar events to enrich meeting notes and offer recording prompts.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,68 @@
|
||||
function safe(read, fallback) {
|
||||
try {
|
||||
const value = read();
|
||||
return value === null || value === undefined ? fallback : value;
|
||||
} catch (_) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function participantText(participant) {
|
||||
const name = String(safe(() => participant.displayName(), "")).trim();
|
||||
const email = String(safe(() => participant.email(), "")).trim();
|
||||
if (!name) {
|
||||
return email;
|
||||
}
|
||||
if (!email || name.toLowerCase() === email.toLowerCase()) {
|
||||
return name;
|
||||
}
|
||||
return `${name} <${email}>`;
|
||||
}
|
||||
|
||||
function run(argv) {
|
||||
if (argv.length !== 2) {
|
||||
throw new Error("Usage: calendar-events.js <from-iso8601> <to-iso8601>");
|
||||
}
|
||||
|
||||
const from = new Date(argv[0]);
|
||||
const to = new Date(argv[1]);
|
||||
const calendar = Application("Calendar");
|
||||
const result = [];
|
||||
|
||||
for (const sourceCalendar of calendar.calendars()) {
|
||||
const events = sourceCalendar.events.whose({
|
||||
_and: [
|
||||
{ startDate: { _lessThan: to } },
|
||||
{ endDate: { _greaterThan: from } }
|
||||
]
|
||||
})();
|
||||
for (const event of events) {
|
||||
const start = event.startDate();
|
||||
const end = event.endDate();
|
||||
const attendees = safe(() => event.attendees(), [])
|
||||
.map(participantText)
|
||||
.filter(value => value.length > 0);
|
||||
const organizer = String(safe(() => event.organizer(), "")).trim();
|
||||
if (organizer && !attendees.includes(organizer)) {
|
||||
attendees.unshift(organizer);
|
||||
}
|
||||
|
||||
result.push({
|
||||
id: String(safe(() => event.uid(), `${start.toISOString()}|${event.summary()}`)),
|
||||
subject: String(safe(() => event.summary(), "")),
|
||||
start: start.toISOString(),
|
||||
end: end.toISOString(),
|
||||
location: String(safe(() => event.location(), "")),
|
||||
notes: String(safe(() => event.description(), "")),
|
||||
url: String(safe(() => event.url(), "")),
|
||||
attendees: attendees,
|
||||
isCanceled: String(safe(() => event.status(), ""))
|
||||
.toLowerCase()
|
||||
.includes("cancel")
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
result.sort((left, right) => left.start.localeCompare(right.start));
|
||||
return JSON.stringify(result);
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import EventKit
|
||||
import Foundation
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
private struct CalendarEventOutput: Encodable {
|
||||
let id: String
|
||||
let subject: String
|
||||
let start: Date
|
||||
let end: Date
|
||||
let location: String
|
||||
let notes: String
|
||||
let url: String
|
||||
let attendees: [String]
|
||||
let isCanceled: Bool
|
||||
}
|
||||
|
||||
private struct PromptRequest: Decodable {
|
||||
let subject: String
|
||||
let start: Date
|
||||
}
|
||||
|
||||
private struct PromptOutput: Encodable {
|
||||
let response: String
|
||||
}
|
||||
|
||||
private struct ScreenshotOutput: Encodable {
|
||||
let windowTitle: String?
|
||||
}
|
||||
|
||||
private struct SelfTestOutput: Encodable {
|
||||
let status: String
|
||||
let features: [String]
|
||||
}
|
||||
|
||||
private enum IntegrationError: LocalizedError {
|
||||
case usage
|
||||
case invalidDate(String)
|
||||
case calendarPermission
|
||||
case screenRecordingPermission
|
||||
case noFrontmostApplication
|
||||
case noActiveWindow
|
||||
case screenshotCapture
|
||||
case screenshotWrite
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .usage:
|
||||
return "Usage: macos-meeting-integrations self-test | calendar <from-iso8601> <to-iso8601> | prompt <request.json> | screenshot <output.png> | crop <input.png> <output.png> <x> <y> <width> <height>"
|
||||
case .invalidDate(let value):
|
||||
return "Invalid ISO-8601 date: \(value)"
|
||||
case .calendarPermission:
|
||||
return "Calendar Full Access permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Calendars."
|
||||
case .screenRecordingPermission:
|
||||
return "Screen & System Audio Recording permission is required. Enable Meeting Assistant under System Settings > Privacy & Security."
|
||||
case .noFrontmostApplication:
|
||||
return "Could not determine the frontmost macOS application."
|
||||
case .noActiveWindow:
|
||||
return "Could not find an on-screen window for the frontmost macOS application."
|
||||
case .screenshotCapture:
|
||||
return "macOS could not capture the active application window."
|
||||
case .screenshotWrite:
|
||||
return "macOS could not write the active-window screenshot as PNG."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
private enum MacOsMeetingIntegrations {
|
||||
@MainActor
|
||||
static func main() async {
|
||||
do {
|
||||
let arguments = Array(CommandLine.arguments.dropFirst())
|
||||
guard let command = arguments.first else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
|
||||
switch command {
|
||||
case "self-test":
|
||||
try writeJson(SelfTestOutput(
|
||||
status: "ok",
|
||||
features: ["calendar", "prompt", "screenshot", "crop"]))
|
||||
case "calendar":
|
||||
guard arguments.count == 3 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
let from = try parseDate(arguments[1])
|
||||
let to = try parseDate(arguments[2])
|
||||
try await writeCalendar(from: from, to: to)
|
||||
case "prompt":
|
||||
guard arguments.count == 2 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try showPrompt(requestPath: arguments[1])
|
||||
case "screenshot":
|
||||
guard arguments.count == 2 else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try captureScreenshot(outputPath: arguments[1])
|
||||
case "crop":
|
||||
guard arguments.count == 7,
|
||||
let x = Int(arguments[3]),
|
||||
let y = Int(arguments[4]),
|
||||
let width = Int(arguments[5]),
|
||||
let height = Int(arguments[6]) else {
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
try cropScreenshot(
|
||||
inputPath: arguments[1],
|
||||
outputPath: arguments[2],
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height)
|
||||
default:
|
||||
throw IntegrationError.usage
|
||||
}
|
||||
} catch {
|
||||
fputs("\(error.localizedDescription)\n", stderr)
|
||||
if let integrationError = error as? IntegrationError,
|
||||
case .calendarPermission = integrationError {
|
||||
exit(77)
|
||||
}
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
private static func parseDate(_ value: String) throws -> Date {
|
||||
let formatter = ISO8601DateFormatter()
|
||||
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
|
||||
if let date = formatter.date(from: value) {
|
||||
return date
|
||||
}
|
||||
|
||||
formatter.formatOptions = [.withInternetDateTime]
|
||||
guard let date = formatter.date(from: value) else {
|
||||
throw IntegrationError.invalidDate(value)
|
||||
}
|
||||
return date
|
||||
}
|
||||
|
||||
private static func writeCalendar(from: Date, to: Date) async throws {
|
||||
let store = EKEventStore()
|
||||
guard try await requestCalendarAccess(store) else {
|
||||
throw IntegrationError.calendarPermission
|
||||
}
|
||||
|
||||
let predicate = store.predicateForEvents(withStart: from, end: to, calendars: nil)
|
||||
let events = store.events(matching: predicate)
|
||||
.map { event in
|
||||
CalendarEventOutput(
|
||||
id: event.eventIdentifier ?? "\(event.startDate.timeIntervalSince1970)|\(event.title ?? "")",
|
||||
subject: event.title ?? "",
|
||||
start: event.startDate,
|
||||
end: event.endDate,
|
||||
location: event.location ?? "",
|
||||
notes: event.notes ?? "",
|
||||
url: event.url?.absoluteString ?? "",
|
||||
attendees: attendees(from: event),
|
||||
isCanceled: event.status == .canceled)
|
||||
}
|
||||
.sorted { $0.start < $1.start }
|
||||
try writeJson(events)
|
||||
}
|
||||
|
||||
private static func requestCalendarAccess(_ store: EKEventStore) async throws -> Bool {
|
||||
if #available(macOS 14.0, *) {
|
||||
return try await store.requestFullAccessToEvents()
|
||||
}
|
||||
|
||||
return try await withCheckedThrowingContinuation { continuation in
|
||||
store.requestAccess(to: .event) { granted, error in
|
||||
if let error {
|
||||
continuation.resume(throwing: error)
|
||||
} else {
|
||||
continuation.resume(returning: granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func attendees(from event: EKEvent) -> [String] {
|
||||
var values: [String] = []
|
||||
if let organizer = event.organizer,
|
||||
let value = participantText(organizer) {
|
||||
values.append(value)
|
||||
}
|
||||
for participant in event.attendees ?? [] {
|
||||
if let value = participantText(participant) {
|
||||
values.append(value)
|
||||
}
|
||||
}
|
||||
return Array(NSOrderedSet(array: values)) as? [String] ?? values
|
||||
}
|
||||
|
||||
private static func participantText(_ participant: EKParticipant) -> String? {
|
||||
let name = participant.name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let email = participant.url
|
||||
.absoluteString
|
||||
.replacingOccurrences(of: "mailto:", with: "", options: .caseInsensitive)
|
||||
if name.isEmpty {
|
||||
return email.isEmpty ? nil : email
|
||||
}
|
||||
if email.isEmpty || name.caseInsensitiveCompare(email) == .orderedSame {
|
||||
return name
|
||||
}
|
||||
return "\(name) <\(email)>"
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private static func showPrompt(requestPath: String) throws {
|
||||
let data = try Data(contentsOf: URL(fileURLWithPath: requestPath))
|
||||
let decoder = JSONDecoder()
|
||||
decoder.dateDecodingStrategy = .iso8601
|
||||
let request = try decoder.decode(PromptRequest.self, from: data)
|
||||
|
||||
let application = NSApplication.shared
|
||||
application.setActivationPolicy(.accessory)
|
||||
application.activate(ignoringOtherApps: true)
|
||||
|
||||
let alert = NSAlert()
|
||||
alert.messageText = "Record meeting?"
|
||||
alert.informativeText = "\(request.subject) starts at \(request.start.formatted(date: .omitted, time: .shortened))."
|
||||
alert.alertStyle = .informational
|
||||
alert.addButton(withTitle: "Record")
|
||||
alert.addButton(withTitle: "Skip")
|
||||
let response = alert.runModal() == .alertFirstButtonReturn ? "record" : "skip"
|
||||
try writeJson(PromptOutput(response: response))
|
||||
}
|
||||
|
||||
private static func captureScreenshot(outputPath: String) throws {
|
||||
guard CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() else {
|
||||
throw IntegrationError.screenRecordingPermission
|
||||
}
|
||||
guard let frontmost = NSWorkspace.shared.frontmostApplication else {
|
||||
throw IntegrationError.noFrontmostApplication
|
||||
}
|
||||
guard let windows = CGWindowListCopyWindowInfo(
|
||||
[.optionOnScreenOnly, .excludeDesktopElements],
|
||||
kCGNullWindowID) as? [[String: Any]] else {
|
||||
throw IntegrationError.noActiveWindow
|
||||
}
|
||||
|
||||
let candidate = windows.first { window in
|
||||
let ownerPid = window[kCGWindowOwnerPID as String] as? Int32
|
||||
let layer = window[kCGWindowLayer as String] as? Int
|
||||
let alpha = window[kCGWindowAlpha as String] as? Double
|
||||
guard ownerPid == frontmost.processIdentifier,
|
||||
layer == 0,
|
||||
(alpha ?? 1) > 0,
|
||||
let bounds = window[kCGWindowBounds as String] as? [String: Any],
|
||||
let width = bounds["Width"] as? Double,
|
||||
let height = bounds["Height"] as? Double else {
|
||||
return false
|
||||
}
|
||||
return width > 1 && height > 1
|
||||
}
|
||||
guard let candidate,
|
||||
let windowNumber = candidate[kCGWindowNumber as String] as? UInt32 else {
|
||||
throw IntegrationError.noActiveWindow
|
||||
}
|
||||
guard let image = CGWindowListCreateImage(
|
||||
.null,
|
||||
.optionIncludingWindow,
|
||||
CGWindowID(windowNumber),
|
||||
[.boundsIgnoreFraming, .bestResolution]) else {
|
||||
throw IntegrationError.screenshotCapture
|
||||
}
|
||||
|
||||
try writePng(image, to: outputPath)
|
||||
|
||||
try writeJson(ScreenshotOutput(
|
||||
windowTitle: candidate[kCGWindowName as String] as? String))
|
||||
}
|
||||
|
||||
private static func cropScreenshot(
|
||||
inputPath: String,
|
||||
outputPath: String,
|
||||
x: Int,
|
||||
y: Int,
|
||||
width: Int,
|
||||
height: Int
|
||||
) throws {
|
||||
guard x >= 0,
|
||||
y >= 0,
|
||||
width > 0,
|
||||
height > 0,
|
||||
let source = CGImageSourceCreateWithURL(
|
||||
URL(fileURLWithPath: inputPath) as CFURL,
|
||||
nil),
|
||||
let image = CGImageSourceCreateImageAtIndex(source, 0, nil),
|
||||
x + width <= image.width,
|
||||
y + height <= image.height,
|
||||
let cropped = image.cropping(to: CGRect(
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height)) else {
|
||||
throw IntegrationError.screenshotCapture
|
||||
}
|
||||
|
||||
try writePng(cropped, to: outputPath)
|
||||
try writeJson(["status": "ok"])
|
||||
}
|
||||
|
||||
private static func writePng(_ image: CGImage, to outputPath: String) throws {
|
||||
let outputUrl = URL(fileURLWithPath: outputPath) as CFURL
|
||||
guard let destination = CGImageDestinationCreateWithURL(
|
||||
outputUrl,
|
||||
UTType.png.identifier as CFString,
|
||||
1,
|
||||
nil) else {
|
||||
throw IntegrationError.screenshotWrite
|
||||
}
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
guard CGImageDestinationFinalize(destination) else {
|
||||
throw IntegrationError.screenshotWrite
|
||||
}
|
||||
}
|
||||
|
||||
private static func writeJson<T: Encodable>(_ value: T) throws {
|
||||
let encoder = JSONEncoder()
|
||||
encoder.dateEncodingStrategy = .iso8601
|
||||
encoder.outputFormatting = [.sortedKeys]
|
||||
FileHandle.standardOutput.write(try encoder.encode(value))
|
||||
FileHandle.standardOutput.write(Data([0x0A]))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
namespace MeetingAssistant.Notifications;
|
||||
|
||||
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
|
||||
|
||||
public interface IAgentApprovalPromptService
|
||||
{
|
||||
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
|
||||
{
|
||||
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
|
||||
}
|
||||
|
||||
public sealed class AgentApprovalService
|
||||
{
|
||||
private readonly IAgentApprovalPromptService prompts;
|
||||
private readonly ILogger<AgentApprovalService> logger;
|
||||
private readonly TimeSpan timeout;
|
||||
|
||||
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
|
||||
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
|
||||
|
||||
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
|
||||
{
|
||||
this.prompts = prompts;
|
||||
this.logger = logger;
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
|
||||
{
|
||||
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
deadline.CancelAfter(timeout);
|
||||
try
|
||||
{
|
||||
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
|
||||
.WaitAsync(deadline.Token);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return accepted && !deadline.IsCancellationRequested;
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await deadline.CancelAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
#if WINDOWS
|
||||
using System.Collections.Concurrent;
|
||||
using CommunityToolkit.WinUI.Notifications;
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Notifications;
|
||||
|
||||
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
|
||||
: IAgentApprovalPromptService, IDisposable
|
||||
{
|
||||
private const string Source = "summary-agent-approval";
|
||||
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||
private readonly object gate = new();
|
||||
private bool registered;
|
||||
private bool disposed;
|
||||
|
||||
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = Guid.NewGuid().ToString("N");
|
||||
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
try
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (disposed) return false;
|
||||
if (!registered)
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||
registered = true;
|
||||
}
|
||||
pending[id] = response;
|
||||
new ToastContentBuilder()
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText(prompt.Title)
|
||||
.AddText(prompt.Message)
|
||||
.AddButton(new ToastButton().SetContent("Yes")
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||
.AddButton(new ToastButton().SetContent("No")
|
||||
.AddArgument("source", Source).AddArgument("requestId", id)
|
||||
.AddArgument("response", "no").SetBackgroundActivation())
|
||||
.Show(toast =>
|
||||
{
|
||||
toast.Tag = id;
|
||||
toast.Group = Source;
|
||||
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
|
||||
});
|
||||
}
|
||||
|
||||
return await response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
pending.TryRemove(id, out _);
|
||||
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
|
||||
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
|
||||
}
|
||||
}
|
||||
|
||||
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||
{
|
||||
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||
if (arguments.GetValueOrDefault("source") != Source ||
|
||||
!arguments.TryGetValue("requestId", out var id) ||
|
||||
!pending.TryRemove(id, out var response)) return;
|
||||
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
disposed = true;
|
||||
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||
registered = false;
|
||||
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||
pending.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,52 @@
|
||||
using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Workflow;
|
||||
|
||||
namespace MeetingAssistant;
|
||||
|
||||
public sealed record PlatformCapabilityReport(
|
||||
string Host,
|
||||
string MeetingMetadataProvider,
|
||||
string CalendarMeetingProvider,
|
||||
string MeetingStartPromptProvider,
|
||||
string ScreenshotProvider,
|
||||
string WorkflowEditorProvider,
|
||||
bool WorkflowEditorAvailable,
|
||||
bool CalendarRecordingPromptsEnabled,
|
||||
bool SpeakerIdentificationEnabled,
|
||||
bool ScreenshotOcrEnabled,
|
||||
bool FunAsrBackendEnabled,
|
||||
bool FunAsrDiarizationEnabled,
|
||||
bool WhisperLocalDiarizationEnabled)
|
||||
{
|
||||
public static PlatformCapabilityReport Create(
|
||||
HostOperatingSystem host,
|
||||
MeetingAssistantOptions options,
|
||||
IMeetingMetadataProvider meetingMetadataProvider,
|
||||
ICalendarMeetingProvider calendarMeetingProvider,
|
||||
IMeetingStartPromptService meetingStartPromptProvider,
|
||||
IActiveWindowScreenshotCapture screenshotProvider,
|
||||
IWorkflowRulesEditorWindowService workflowEditorProvider)
|
||||
{
|
||||
return new PlatformCapabilityReport(
|
||||
host.Kind switch
|
||||
{
|
||||
HostOperatingSystemKind.Windows => "Windows",
|
||||
HostOperatingSystemKind.MacOS => "macOS",
|
||||
_ => "Unsupported"
|
||||
},
|
||||
meetingMetadataProvider.GetType().Name,
|
||||
calendarMeetingProvider.GetType().Name,
|
||||
meetingStartPromptProvider.GetType().Name,
|
||||
screenshotProvider.GetType().Name,
|
||||
workflowEditorProvider.GetType().Name,
|
||||
workflowEditorProvider is not NoopWorkflowRulesEditorWindowService,
|
||||
options.CalendarRecordingPrompts.Enabled,
|
||||
options.SpeakerIdentification.Enabled,
|
||||
options.Screenshots.Ocr.Enabled,
|
||||
options.FunAsr.Backend.Enabled,
|
||||
options.FunAsr.Diarization.Enabled,
|
||||
options.WhisperLocal.Diarization.Enabled);
|
||||
}
|
||||
}
|
||||
+90
-13
@@ -3,7 +3,9 @@ using MeetingAssistant.Calendar;
|
||||
using MeetingAssistant.Hotkeys;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Logging;
|
||||
using MeetingAssistant.MacOs;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Notifications;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Speakers;
|
||||
@@ -16,7 +18,11 @@ using Microsoft.Extensions.Options;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
|
||||
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
|
||||
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
|
||||
builder.Services.AddOptions<MeetingAssistantOptions>()
|
||||
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton(HostOperatingSystem.DetectCurrent());
|
||||
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
|
||||
@@ -25,6 +31,9 @@ builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
|
||||
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
|
||||
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
|
||||
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
|
||||
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
|
||||
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
|
||||
builder.Services.AddSingleton<MicrophoneAudioSource>();
|
||||
builder.Services.AddSingleton<SystemAudioSource>();
|
||||
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||
@@ -35,8 +44,11 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
|
||||
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||
PortableMeetingAudioRegistration.Add(builder.Services);
|
||||
MacOsMeetingIntegrationRegistration.Add(builder.Services, OperatingSystem.IsMacOS());
|
||||
builder.Services.AddHostedService<MacOsDesktopControlService>();
|
||||
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
|
||||
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
|
||||
@@ -66,14 +78,8 @@ builder.Services.AddSingleton<IMeetingStartPromptService>(services =>
|
||||
services.GetRequiredService<WindowsMeetingStartPromptService>());
|
||||
builder.Services.AddHostedService(services =>
|
||||
services.GetRequiredService<WindowsMeetingStartPromptService>());
|
||||
#else
|
||||
builder.Services.AddSingleton<ICalendarMeetingProvider, NoopCalendarMeetingProvider>();
|
||||
builder.Services.AddSingleton<IMeetingStartPromptService, NoopMeetingStartPromptService>();
|
||||
#endif
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
builder.Services.AddSingleton<IScreenshotImageCropper, WindowsScreenshotImageCropper>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
|
||||
builder.Services.AddSingleton<MeetingScreenshotService>();
|
||||
@@ -91,8 +97,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
|
||||
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<SpeakerIdentityService>();
|
||||
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
|
||||
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
|
||||
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
|
||||
builder.Services.AddSingleton(services =>
|
||||
{
|
||||
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer;
|
||||
return resemblyzer.Enabled
|
||||
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
|
||||
: SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
});
|
||||
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
|
||||
: services.GetRequiredService<SpeakerIdentityService>());
|
||||
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
|
||||
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
|
||||
.Value.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
|
||||
: services.GetRequiredService<SpeakerIdentityMergeService>());
|
||||
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
|
||||
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
|
||||
@@ -102,7 +135,11 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
|
||||
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
|
||||
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
|
||||
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
|
||||
#if WINDOWS
|
||||
@@ -110,8 +147,12 @@ builder.Services.AddSingleton(services => WorkflowRulesEditorMarkdownLinkResolve
|
||||
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value));
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRulesEditorWindowService>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
|
||||
builder.Services.AddSingleton<MacOsWorkflowRulesEditorSession>();
|
||||
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, MacOsWorkflowRulesEditorWindowService>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<AgentApprovalService>();
|
||||
builder.Services.AddSingleton<WorkflowChangeRequestService>();
|
||||
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
|
||||
builder.Services.AddSingleton<AsrDiagnosticService>();
|
||||
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
|
||||
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
|
||||
@@ -134,6 +175,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
|
||||
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
|
||||
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
|
||||
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
|
||||
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddHostedService<GlobalHotkeyService>();
|
||||
builder.Services.AddHostedService<UnoTaskbarIconService>();
|
||||
@@ -146,11 +188,32 @@ app.MapGet("/health", () => Results.Ok(new
|
||||
service = "meeting-assistant",
|
||||
status = "ok"
|
||||
}));
|
||||
app.MapGet("/diagnostics/platform-capabilities", (
|
||||
HostOperatingSystem host,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
IMeetingMetadataProvider meetingMetadataProvider,
|
||||
ICalendarMeetingProvider calendarMeetingProvider,
|
||||
IMeetingStartPromptService meetingStartPromptProvider,
|
||||
IActiveWindowScreenshotCapture screenshotProvider,
|
||||
IWorkflowRulesEditorWindowService workflowEditorProvider) =>
|
||||
Results.Ok(PlatformCapabilityReport.Create(
|
||||
host,
|
||||
options.Value,
|
||||
meetingMetadataProvider,
|
||||
calendarMeetingProvider,
|
||||
meetingStartPromptProvider,
|
||||
screenshotProvider,
|
||||
workflowEditorProvider)));
|
||||
|
||||
app.MapGet("/recording/status", (MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
|
||||
app.MapGet("/profiles/{launchProfile}/recording/status", (
|
||||
string launchProfile,
|
||||
MeetingRecordingCoordinator coordinator) => Results.Ok(coordinator.CurrentStatus));
|
||||
app.MapPost("/application/exit", (IHostApplicationLifetime lifetime) =>
|
||||
{
|
||||
lifetime.StopApplication();
|
||||
return Results.Accepted();
|
||||
});
|
||||
app.MapGet("/diagnostics/outlook/current-meeting", async (
|
||||
IMeetingMetadataProvider metadataProvider,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
@@ -280,6 +343,15 @@ app.MapPost("/diagnostics/settings-and-logs/show", (
|
||||
rulesEditorWindow.Show();
|
||||
return Results.Accepted();
|
||||
});
|
||||
#if !WINDOWS
|
||||
app.MapGet("/diagnostics/settings-and-logs", () =>
|
||||
Results.Content(MacOsWorkflowRulesEditorPage.Html, "text/html"));
|
||||
app.MapPost("/diagnostics/settings-and-logs/chat", async (
|
||||
MacOsWorkflowRulesEditorChatRequest request,
|
||||
MacOsWorkflowRulesEditorSession session,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await session.SendAsync(request.Message, cancellationToken)));
|
||||
#endif
|
||||
|
||||
static WorkflowConfigurationReloadResponse ReloadWorkflowConfiguration(IConfiguration configuration)
|
||||
{
|
||||
@@ -322,6 +394,11 @@ app.MapPost("/profiles/{launchProfile}/recording/abort", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
|
||||
app.MapPost("/profiles/{launchProfile}/meetings/screenshot/capture", async (
|
||||
string launchProfile,
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.CaptureScreenshotAsync(launchProfile, cancellationToken)));
|
||||
app.MapPost("/meetings/current/summary/run", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
IMeetingSummaryPipeline summaryPipeline,
|
||||
|
||||
@@ -6,6 +6,8 @@ public interface IMeetingInactivityPromptService
|
||||
MeetingInactivityPromptRequest request,
|
||||
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task DismissAllAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingInactivityPromptRequest(
|
||||
@@ -18,9 +20,32 @@ public enum MeetingInactivityPromptResponse
|
||||
{
|
||||
Dismissed,
|
||||
Continue,
|
||||
Pause,
|
||||
Stop
|
||||
}
|
||||
|
||||
internal sealed record MeetingInactivityPromptAction(
|
||||
string Content,
|
||||
string ResponseArgument,
|
||||
MeetingInactivityPromptResponse Response);
|
||||
|
||||
internal static class MeetingInactivityPromptActions
|
||||
{
|
||||
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
|
||||
[
|
||||
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
|
||||
new("No", "continue", MeetingInactivityPromptResponse.Continue),
|
||||
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
|
||||
];
|
||||
|
||||
public static MeetingInactivityPromptResponse ParseResponse(string? value)
|
||||
{
|
||||
return All.FirstOrDefault(action =>
|
||||
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
|
||||
?.Response ?? MeetingInactivityPromptResponse.Continue;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
|
||||
{
|
||||
public Task ShowStopPromptAsync(
|
||||
@@ -30,4 +55,9 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,5 +2,7 @@ namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMicrophoneCaptureSourceFactory
|
||||
{
|
||||
// The caller owns the returned source and disposes it when it implements IDisposable,
|
||||
// including when capture is canceled before enumeration starts.
|
||||
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMicrophoneSwitchPromptService
|
||||
{
|
||||
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
|
||||
{
|
||||
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(false);
|
||||
}
|
||||
@@ -33,6 +33,17 @@ public interface ITranscriptStore
|
||||
string line,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
async Task AppendLinesAsync(
|
||||
TranscriptSession session,
|
||||
IReadOnlyList<string> lines,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var line in lines)
|
||||
{
|
||||
await AppendLineAsync(session, line, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
internal enum MacOsAudioCaptureKind
|
||||
{
|
||||
Microphone,
|
||||
System
|
||||
}
|
||||
|
||||
internal sealed record MacOsAudioCaptureRequest(
|
||||
MacOsAudioCaptureKind Kind,
|
||||
int SampleRate,
|
||||
int Channels);
|
||||
|
||||
internal interface IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request);
|
||||
}
|
||||
|
||||
internal interface IMacOsAudioCaptureProcess : IAsyncDisposable
|
||||
{
|
||||
Stream StandardOutput { get; }
|
||||
|
||||
int ExitCode { get; }
|
||||
|
||||
bool HasExited { get; }
|
||||
|
||||
Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken);
|
||||
|
||||
Task WaitForExitAsync(CancellationToken cancellationToken);
|
||||
|
||||
void Terminate();
|
||||
}
|
||||
|
||||
internal abstract class MacOsMeetingAudioSource : IMeetingAudioSource
|
||||
{
|
||||
private const int ReadBufferSize = 8 * 1024;
|
||||
private readonly MacOsAudioCaptureKind kind;
|
||||
private readonly IMacOsAudioCaptureProcessFactory processFactory;
|
||||
private readonly ILogger logger;
|
||||
|
||||
protected MacOsMeetingAudioSource(
|
||||
MacOsAudioCaptureKind kind,
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger logger)
|
||||
{
|
||||
this.kind = kind;
|
||||
this.processFactory = processFactory;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
MeetingAssistantOptions options,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new MacOsAudioCaptureRequest(
|
||||
kind,
|
||||
options.Recording.SampleRate,
|
||||
options.Recording.Channels);
|
||||
await using var process = processFactory.Start(request);
|
||||
var standardErrorTask = process.ReadStandardErrorAsync(CancellationToken.None);
|
||||
var buffer = new byte[ReadBufferSize];
|
||||
byte? pendingByte = null;
|
||||
var hasLoggedFirstChunk = false;
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting macOS {CaptureKind} audio capture at {SampleRate} Hz/{Channels} channel(s)",
|
||||
kind,
|
||||
request.SampleRate,
|
||||
request.Channels);
|
||||
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var bytesRead = await process.StandardOutput.ReadAsync(buffer, cancellationToken);
|
||||
if (bytesRead == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var pcm = CopyCompleteSamples(buffer, bytesRead, ref pendingByte);
|
||||
if (pcm.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hasLoggedFirstChunk)
|
||||
{
|
||||
hasLoggedFirstChunk = true;
|
||||
logger.LogInformation(
|
||||
"macOS {CaptureKind} audio produced its first PCM chunk: {ByteCount} bytes",
|
||||
kind,
|
||||
pcm.Length);
|
||||
}
|
||||
|
||||
yield return new AudioChunk(pcm, request.SampleRate, request.Channels);
|
||||
}
|
||||
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
var standardError = await standardErrorTask;
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
throw CreateCaptureException(kind, process.ExitCode, standardError);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Terminate();
|
||||
await process.WaitForExitAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] CopyCompleteSamples(byte[] buffer, int bytesRead, ref byte? pendingByte)
|
||||
{
|
||||
var totalBytes = bytesRead + (pendingByte.HasValue ? 1 : 0);
|
||||
var completeBytes = totalBytes - totalBytes % sizeof(short);
|
||||
var pcm = new byte[completeBytes];
|
||||
var targetOffset = 0;
|
||||
|
||||
if (pendingByte is { } firstByte)
|
||||
{
|
||||
pcm[0] = firstByte;
|
||||
targetOffset = 1;
|
||||
pendingByte = null;
|
||||
}
|
||||
|
||||
var copyCount = completeBytes - targetOffset;
|
||||
if (copyCount > 0)
|
||||
{
|
||||
Buffer.BlockCopy(buffer, 0, pcm, targetOffset, copyCount);
|
||||
}
|
||||
|
||||
var consumedFromBuffer = copyCount;
|
||||
if (consumedFromBuffer < bytesRead)
|
||||
{
|
||||
pendingByte = buffer[consumedFromBuffer];
|
||||
}
|
||||
|
||||
return pcm;
|
||||
}
|
||||
|
||||
private static InvalidOperationException CreateCaptureException(
|
||||
MacOsAudioCaptureKind kind,
|
||||
int exitCode,
|
||||
string standardError)
|
||||
{
|
||||
var detail = string.IsNullOrWhiteSpace(standardError)
|
||||
? "The native capture helper did not provide an error message."
|
||||
: standardError.Trim();
|
||||
return new InvalidOperationException(
|
||||
$"macOS {kind.ToString().ToLowerInvariant()} audio capture exited with code {exitCode}. {detail}");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsMicrophoneAudioSource : MacOsMeetingAudioSource
|
||||
{
|
||||
public MacOsMicrophoneAudioSource(
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger<MacOsMicrophoneAudioSource> logger)
|
||||
: base(MacOsAudioCaptureKind.Microphone, processFactory, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsSystemAudioSource : MacOsMeetingAudioSource
|
||||
{
|
||||
public MacOsSystemAudioSource(
|
||||
IMacOsAudioCaptureProcessFactory processFactory,
|
||||
ILogger<MacOsSystemAudioSource> logger)
|
||||
: base(MacOsAudioCaptureKind.System, processFactory, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||
{
|
||||
private readonly ILogger<MacOsAudioCaptureProcessFactory> logger;
|
||||
|
||||
public MacOsAudioCaptureProcessFactory(ILogger<MacOsAudioCaptureProcessFactory> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||
{
|
||||
if (!OperatingSystem.IsMacOS())
|
||||
{
|
||||
throw new PlatformNotSupportedException("The macOS audio capture helper can only run on macOS.");
|
||||
}
|
||||
|
||||
var helperPath = Path.Combine(
|
||||
AppContext.BaseDirectory,
|
||||
"Native",
|
||||
"MeetingAssistantAudioCapture.app",
|
||||
"Contents",
|
||||
"MacOS",
|
||||
"macos-meeting-audio-capture");
|
||||
if (!File.Exists(helperPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"The macOS audio capture helper was not found at '{helperPath}'. Build or publish the application on macOS so the Swift helper is compiled into the Native folder.",
|
||||
helperPath);
|
||||
}
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = helperPath,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
startInfo.ArgumentList.Add(request.Kind == MacOsAudioCaptureKind.Microphone ? "microphone" : "system");
|
||||
startInfo.ArgumentList.Add(request.SampleRate.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
startInfo.ArgumentList.Add(request.Channels.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
|
||||
logger.LogDebug("Starting native macOS audio helper {HelperPath} for {CaptureKind}", helperPath, request.Kind);
|
||||
var process = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException($"Failed to start the macOS audio capture helper '{helperPath}'.");
|
||||
return new MacOsAudioCaptureProcess(process);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed class MacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||
{
|
||||
private readonly Process process;
|
||||
|
||||
public MacOsAudioCaptureProcess(Process process)
|
||||
{
|
||||
this.process = process;
|
||||
}
|
||||
|
||||
public Stream StandardOutput => process.StandardOutput.BaseStream;
|
||||
|
||||
public int ExitCode => process.ExitCode;
|
||||
|
||||
public bool HasExited => process.HasExited;
|
||||
|
||||
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return process.WaitForExitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
process.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly IMeetingInactivityClock inactivityClock;
|
||||
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
|
||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
private RecordingRun? currentRun;
|
||||
@@ -63,7 +64,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
IMeetingRunArtifactCleaner? artifactCleaner = null,
|
||||
IMeetingInactivityPromptService? inactivityPromptService = null,
|
||||
IMeetingInactivityClock? inactivityClock = null,
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
|
||||
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
|
||||
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
|
||||
{
|
||||
this.audioSource = audioSource;
|
||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||
@@ -86,6 +88,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
|
||||
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
|
||||
this.options = options.Value;
|
||||
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
|
||||
?? SpeakerSampleCollectionPolicy.ExistingBackend;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -96,7 +100,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
currentArtifacts?.AssistantContextPath,
|
||||
currentArtifacts?.SummaryPath,
|
||||
GetProcessState(currentRun),
|
||||
currentRun?.LaunchProfileName);
|
||||
currentRun?.LaunchProfileName,
|
||||
IsRecording && currentRun?.IsTranscriptionPaused == true);
|
||||
|
||||
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
|
||||
|
||||
@@ -289,10 +294,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
startedAt,
|
||||
launchProfile.Name,
|
||||
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
|
||||
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
|
||||
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
|
||||
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
|
||||
logger);
|
||||
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
|
||||
if (runOptions.Recording.InactivitySafeguard.Enabled)
|
||||
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
|
||||
{
|
||||
_ = Task.Run(
|
||||
() => RunInactivitySafeguardAsync(run),
|
||||
@@ -334,11 +341,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return await StopAsync(null, cancellationToken);
|
||||
return await StopAsync(null, null, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> StopAsync(
|
||||
DateTimeOffset? inferredEndTime,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
DateTimeOffset? expectedPauseStartedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -347,13 +357,30 @@ public sealed class MeetingRecordingCoordinator
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (currentRun is null || currentRun.IsCaptureStopping)
|
||||
if (currentRun is null ||
|
||||
currentRun.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
run = currentRun;
|
||||
if (inferredEndTime is not null)
|
||||
var inactivityStopClaimed = expectedActivityVersion is not null;
|
||||
if (expectedActivityVersion is not null &&
|
||||
(inferredEndTime is null ||
|
||||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
|
||||
if (expectedPauseStartedAt is not null &&
|
||||
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
|
||||
{
|
||||
run.SetInferredEndTime(inferredEndTime.Value);
|
||||
}
|
||||
@@ -363,7 +390,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
{
|
||||
run.Abort();
|
||||
}
|
||||
else
|
||||
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
|
||||
{
|
||||
run.StopCapture();
|
||||
}
|
||||
@@ -389,6 +416,12 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
|
||||
{
|
||||
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
if (IsAzureSpeechRun(run))
|
||||
{
|
||||
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
|
||||
@@ -409,6 +442,65 @@ public sealed class MeetingRecordingCoordinator
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
|
||||
bool isPaused,
|
||||
RecordingRun? expectedRun,
|
||||
long? expectedActivityVersion,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var run = currentRun;
|
||||
if (run is null ||
|
||||
run.IsCaptureStopping ||
|
||||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
var changed = false;
|
||||
await run.PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var changedAt = inactivityClock.Now;
|
||||
changed = run.TrySetTranscriptionPaused(
|
||||
isPaused,
|
||||
changedAt,
|
||||
expectedActivityVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
run.PipelineGate.Release();
|
||||
}
|
||||
|
||||
if (changed && isPaused)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (changed)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Meeting transcription {PauseState}",
|
||||
isPaused ? "paused" : "unpaused");
|
||||
}
|
||||
|
||||
return CurrentStatus;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
@@ -624,6 +716,24 @@ public sealed class MeetingRecordingCoordinator
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Meeting recording failed");
|
||||
if (IsAzureSpeechRun(run) && !run.IsQueuedForOfflineTranscription)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Azure Speech transcription failed; retaining recorded audio and queueing offline transcription backlog item");
|
||||
run.MarkQueuedForOfflineTranscription();
|
||||
try
|
||||
{
|
||||
await offlineTranscriptionBacklog.EnqueueAsync(
|
||||
CreateOfflineBacklogItem(run),
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception backlogException)
|
||||
{
|
||||
logger.LogError(
|
||||
backlogException,
|
||||
"Could not queue offline transcription backlog item; retaining recorded audio for manual recovery");
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -661,8 +771,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
chunk.Channels);
|
||||
}
|
||||
|
||||
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
|
||||
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
|
||||
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -701,7 +810,6 @@ public sealed class MeetingRecordingCoordinator
|
||||
run.ResetSpeakerIdentification();
|
||||
}
|
||||
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
var sample = run.TryAddSpeakerSample(segment);
|
||||
if (sample is not null)
|
||||
{
|
||||
@@ -718,6 +826,56 @@ public sealed class MeetingRecordingCoordinator
|
||||
var relabeledSegment = run.Relabel(segment);
|
||||
run.AddLiveSegment(relabeledSegment);
|
||||
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
|
||||
run.RecordTranscriptActivity(segment, inactivityClock.Now);
|
||||
if (!string.IsNullOrWhiteSpace(segment.Text))
|
||||
{
|
||||
// Switching profiles holds the coordinator gate while draining this reader.
|
||||
// Notification cleanup must not make transcript draining wait for that gate.
|
||||
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsForRunAsync(
|
||||
RecordingRun run,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await DismissInactivityPromptsAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Pending cleanup is no longer needed after this run stops capturing.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await inactivityPromptService.DismissAllAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -845,7 +1003,10 @@ public sealed class MeetingRecordingCoordinator
|
||||
private async Task RunInactivitySafeguardAsync(RecordingRun run)
|
||||
{
|
||||
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
|
||||
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
|
||||
var promptThresholds = safeguardOptions.Enabled
|
||||
? GetInactivityPromptThresholds(safeguardOptions)
|
||||
: [];
|
||||
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
|
||||
var promptedThresholds = new HashSet<TimeSpan>();
|
||||
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
|
||||
|
||||
@@ -853,7 +1014,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
|
||||
string.Join(", ", promptThresholds),
|
||||
safeguardOptions.AutoStopAfter,
|
||||
safeguardOptions.CheckInterval);
|
||||
checkInterval);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -869,9 +1030,38 @@ public sealed class MeetingRecordingCoordinator
|
||||
snapshot.LastTranscriptActivityAt);
|
||||
}
|
||||
|
||||
var pauseStartedAt = snapshot.TranscriptionPausedAt;
|
||||
if (pauseStartedAt is not null)
|
||||
{
|
||||
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
|
||||
if (maximumPauseDuration > TimeSpan.Zero &&
|
||||
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
|
||||
maximumPauseDuration);
|
||||
var stopStatus = await StopAsync(
|
||||
null,
|
||||
run,
|
||||
null,
|
||||
pauseStartedAt,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = inactivityClock.Now;
|
||||
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
|
||||
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
if (safeguardOptions.Enabled &&
|
||||
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
|
||||
inactivityDuration >= safeguardOptions.AutoStopAfter)
|
||||
{
|
||||
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
|
||||
@@ -879,8 +1069,16 @@ public sealed class MeetingRecordingCoordinator
|
||||
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
|
||||
inactivityDuration,
|
||||
inferredEndTime);
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
return;
|
||||
var stopStatus = await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
snapshot.ActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
if (!stopStatus.IsRecording)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var threshold in promptThresholds)
|
||||
@@ -897,6 +1095,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
inactivityDuration,
|
||||
threshold,
|
||||
inferredEndTime);
|
||||
var promptActivityVersion = snapshot.ActivityVersion;
|
||||
await inactivityPromptService.ShowStopPromptAsync(
|
||||
new MeetingInactivityPromptRequest(
|
||||
inactivityDuration,
|
||||
@@ -912,16 +1111,36 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
if (response == MeetingInactivityPromptResponse.Stop)
|
||||
{
|
||||
await StopAsync(inferredEndTime, CancellationToken.None);
|
||||
await StopAsync(
|
||||
inferredEndTime,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
null,
|
||||
CancellationToken.None);
|
||||
}
|
||||
else if (response == MeetingInactivityPromptResponse.Pause)
|
||||
{
|
||||
await SetTranscriptionPausedAsync(
|
||||
true,
|
||||
run,
|
||||
promptActivityVersion,
|
||||
CancellationToken.None);
|
||||
}
|
||||
},
|
||||
run.CaptureCancellation);
|
||||
|
||||
var currentActivity = run.GetTranscriptActivitySnapshot();
|
||||
if (currentActivity.IsTranscriptionPaused ||
|
||||
currentActivity.ActivityVersion != promptActivityVersion)
|
||||
{
|
||||
await DismissInactivityPromptsAsync(run.CaptureCancellation);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
await inactivityClock.DelayAsync(
|
||||
GetInactivitySafeguardCheckInterval(safeguardOptions),
|
||||
checkInterval,
|
||||
run.CaptureCancellation);
|
||||
}
|
||||
}
|
||||
@@ -989,11 +1208,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
try
|
||||
{
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
|
||||
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
|
||||
if (checkpoint is null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
|
||||
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1821,6 +2040,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
|
||||
}
|
||||
|
||||
private static TimeSpan GetInactivitySafeguardCheckInterval(
|
||||
RecordingInactivitySafeguardOptions options)
|
||||
{
|
||||
@@ -1877,6 +2101,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
private TimeSpan? lastTranscriptSegmentEnd;
|
||||
private long transcriptActivityVersion;
|
||||
private DateTimeOffset? inferredEndTime;
|
||||
private bool isTranscriptionPaused;
|
||||
private DateTimeOffset? transcriptionPausedAt;
|
||||
|
||||
public RecordingRun(
|
||||
CancellationTokenSource captureCancellation,
|
||||
@@ -1892,6 +2118,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
string launchProfileName,
|
||||
TimeSpan liveSampleBufferDuration,
|
||||
int maxSpeakerSamples,
|
||||
bool requireNonOverlappingSpeakerSamples,
|
||||
ILogger logger)
|
||||
{
|
||||
CaptureCancellationSource = captureCancellation;
|
||||
@@ -1912,8 +2139,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
liveSampleBufferDuration,
|
||||
maxSpeakerSamples,
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleSegmentGap,
|
||||
logger);
|
||||
options.SpeakerIdentification.MaximumSampleDuration,
|
||||
logger,
|
||||
requireNonOverlappingSpeakerSamples);
|
||||
}
|
||||
|
||||
public CancellationTokenSource CaptureCancellationSource { get; }
|
||||
@@ -1960,6 +2188,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public bool HasAttachedPromptMetadata { get; private set; }
|
||||
|
||||
public bool IsTranscriptionPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
return isTranscriptionPaused;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||
|
||||
public DateTimeOffset? InferredEndTime
|
||||
@@ -1983,6 +2222,37 @@ public sealed class MeetingRecordingCoordinator
|
||||
HasAttachedPromptMetadata = true;
|
||||
}
|
||||
|
||||
public bool TrySetTranscriptionPaused(
|
||||
bool isPaused,
|
||||
DateTimeOffset changedAt,
|
||||
long? expectedActivityVersion)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (expectedActivityVersion is not null &&
|
||||
transcriptActivityVersion != expectedActivityVersion.Value)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isTranscriptionPaused == isPaused)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
isTranscriptionPaused = isPaused;
|
||||
transcriptionPausedAt = isPaused ? changedAt : null;
|
||||
|
||||
if (!isPaused)
|
||||
{
|
||||
lastTranscriptActivityAt = changedAt;
|
||||
}
|
||||
|
||||
transcriptActivityVersion++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void Abort()
|
||||
{
|
||||
IsAborted = true;
|
||||
@@ -2011,13 +2281,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||
{
|
||||
await PipelineGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
AppendAudio(chunk);
|
||||
await Pipeline.WriteAsync(chunk, cancellationToken);
|
||||
var routedChunk = IsTranscriptionPaused
|
||||
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
|
||||
: chunk;
|
||||
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
|
||||
AppendAudio(routedChunk);
|
||||
await Pipeline.WriteAsync(routedChunk, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -2201,7 +2475,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
return new TranscriptActivitySnapshot(
|
||||
lastTranscriptActivityAt,
|
||||
lastTranscriptSegmentEnd,
|
||||
transcriptActivityVersion);
|
||||
transcriptActivityVersion,
|
||||
isTranscriptionPaused,
|
||||
transcriptionPausedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2221,6 +2497,35 @@ public sealed class MeetingRecordingCoordinator
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
inferredEndTime = endTime;
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
|
||||
{
|
||||
lock (transcriptActivityGate)
|
||||
{
|
||||
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
StopCapture();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void AppendAudio(AudioChunk chunk)
|
||||
{
|
||||
speakerSampleCollector.AppendAudio(chunk);
|
||||
@@ -2244,9 +2549,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
return speakerSampleCollector.Snapshot();
|
||||
}
|
||||
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
|
||||
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
var speakers = GetUnmappedSampleSpeakers();
|
||||
var speakers = GetUnmappedSampleSpeakers(samples);
|
||||
if (speakers.Count == 0)
|
||||
{
|
||||
return null;
|
||||
@@ -2254,7 +2561,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
var checkpoint = new LiveIdentificationCheckpoint(
|
||||
speakers,
|
||||
BuildAttendeeSignature(meetingNote));
|
||||
BuildAttendeeSignature(meetingNote),
|
||||
Options.SpeakerIdentification.Resemblyzer.Enabled
|
||||
? speakers.Select(speaker => samples.Count(sample =>
|
||||
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
|
||||
: []);
|
||||
lock (liveIdentificationGate)
|
||||
{
|
||||
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
|
||||
@@ -2312,9 +2623,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
|
||||
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
|
||||
{
|
||||
return GetSpeakerSamplesSnapshot()
|
||||
return samples
|
||||
.Select(sample => sample.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -2350,7 +2661,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
public sealed record TranscriptActivitySnapshot(
|
||||
DateTimeOffset LastTranscriptActivityAt,
|
||||
TimeSpan? LastTranscriptSegmentEnd,
|
||||
long ActivityVersion);
|
||||
long ActivityVersion,
|
||||
bool IsTranscriptionPaused,
|
||||
DateTimeOffset? TranscriptionPausedAt);
|
||||
|
||||
private static int StateRank(AssistantContextState state)
|
||||
{
|
||||
@@ -2368,11 +2681,13 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public sealed record LiveIdentificationCheckpoint(
|
||||
IReadOnlyList<string> Speakers,
|
||||
string AttendeeSignature)
|
||||
string AttendeeSignature,
|
||||
IReadOnlyList<int> SampleCounts)
|
||||
{
|
||||
public bool Matches(LiveIdentificationCheckpoint other)
|
||||
{
|
||||
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
|
||||
SampleCounts.SequenceEqual(other.SampleCounts) &&
|
||||
Speakers.Count == other.Speakers.Count &&
|
||||
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -2405,7 +2720,8 @@ public sealed record RecordingStatus(
|
||||
string? AssistantContextPath,
|
||||
string? SummaryPath,
|
||||
RecordingProcessState State = RecordingProcessState.Idle,
|
||||
string? LaunchProfile = null);
|
||||
string? LaunchProfile = null,
|
||||
bool IsPaused = false);
|
||||
|
||||
public enum RecordingProcessState
|
||||
{
|
||||
|
||||
@@ -8,22 +8,30 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
private readonly IMicrophoneCaptureSourceFactory captureSources;
|
||||
private readonly ILogger<MicrophoneAudioSource> logger;
|
||||
private readonly TimeSpan recoveryDelay;
|
||||
private readonly MicrophoneSwitchMonitor? switchMonitor;
|
||||
private readonly MicrophoneDeviceSelection? selection;
|
||||
|
||||
public MicrophoneAudioSource(
|
||||
IMicrophoneCaptureSourceFactory captureSources,
|
||||
ILogger<MicrophoneAudioSource> logger)
|
||||
: this(captureSources, logger, DefaultRecoveryDelay)
|
||||
ILogger<MicrophoneAudioSource> logger,
|
||||
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||
MicrophoneDeviceSelection? selection = null)
|
||||
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
|
||||
{
|
||||
}
|
||||
|
||||
internal MicrophoneAudioSource(
|
||||
IMicrophoneCaptureSourceFactory captureSources,
|
||||
ILogger<MicrophoneAudioSource> logger,
|
||||
TimeSpan recoveryDelay)
|
||||
TimeSpan recoveryDelay,
|
||||
MicrophoneSwitchMonitor? switchMonitor = null,
|
||||
MicrophoneDeviceSelection? selection = null)
|
||||
{
|
||||
this.captureSources = captureSources;
|
||||
this.logger = logger;
|
||||
this.recoveryDelay = recoveryDelay;
|
||||
this.switchMonitor = switchMonitor;
|
||||
this.selection = selection;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
@@ -34,21 +42,44 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||
MeetingAssistantOptions options,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
using var session = new CaptureSession(selection, cancellationToken);
|
||||
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
|
||||
try
|
||||
{
|
||||
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
|
||||
{
|
||||
yield return chunk;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
session.Stop();
|
||||
await monitoring;
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
|
||||
MeetingAssistantOptions options,
|
||||
CaptureSession session,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var failedAttempts = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var captureToken = session.BeginCapture();
|
||||
IMeetingAudioSource? captureSource = null;
|
||||
IAsyncEnumerator<AudioChunk>? capture = null;
|
||||
Exception? failure = null;
|
||||
|
||||
try
|
||||
{
|
||||
capture = captureSources
|
||||
.CreateCapture(options)
|
||||
.CaptureAsync(options, cancellationToken)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
captureSource = captureSources.CreateCapture(options);
|
||||
capture = captureSource
|
||||
.CaptureAsync(options, captureToken)
|
||||
.GetAsyncEnumerator(captureToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
@@ -56,6 +87,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
failure = exception;
|
||||
}
|
||||
|
||||
using var captureOwner = captureSource as IDisposable;
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
yield break;
|
||||
@@ -65,14 +97,14 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
{
|
||||
try
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
while (!captureToken.IsCancellationRequested)
|
||||
{
|
||||
var hasNext = false;
|
||||
try
|
||||
{
|
||||
hasNext = await capture.MoveNextAsync();
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
@@ -80,7 +112,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
failure = exception;
|
||||
}
|
||||
|
||||
if (cancellationToken.IsCancellationRequested || failure is not null || !hasNext)
|
||||
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
|
||||
{
|
||||
break;
|
||||
}
|
||||
@@ -102,7 +134,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
{
|
||||
await capture.DisposeAsync();
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
@@ -112,11 +144,19 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
}
|
||||
}
|
||||
|
||||
var switchRequested = captureToken.IsCancellationRequested;
|
||||
session.EndCapture();
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (switchRequested)
|
||||
{
|
||||
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
|
||||
continue;
|
||||
}
|
||||
|
||||
failedAttempts++;
|
||||
logger.LogWarning(
|
||||
failure,
|
||||
@@ -142,4 +182,53 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
|
||||
{
|
||||
private readonly object gate = new();
|
||||
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
private CancellationTokenSource? capture;
|
||||
|
||||
public CancellationToken Token => lifetime.Token;
|
||||
|
||||
public CancellationToken BeginCapture()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
|
||||
return capture.Token;
|
||||
}
|
||||
}
|
||||
|
||||
public void EndCapture()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
capture?.Dispose();
|
||||
capture = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
|
||||
selection.Select(device.Id);
|
||||
capture?.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
lock (gate) lifetime.Cancel();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Stop();
|
||||
EndCapture();
|
||||
lifetime.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class MicrophoneSwitchMonitor
|
||||
{
|
||||
private readonly IMicrophoneDeviceProvider devices;
|
||||
private readonly IMicrophoneSwitchPromptService prompts;
|
||||
private readonly ILogger<MicrophoneSwitchMonitor> logger;
|
||||
private readonly TimeSpan pollInterval;
|
||||
private readonly TimeSpan promptLifetime;
|
||||
|
||||
public MicrophoneSwitchMonitor(
|
||||
IMicrophoneDeviceProvider devices,
|
||||
IMicrophoneSwitchPromptService prompts,
|
||||
ILogger<MicrophoneSwitchMonitor> logger)
|
||||
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
|
||||
{
|
||||
}
|
||||
|
||||
internal MicrophoneSwitchMonitor(
|
||||
IMicrophoneDeviceProvider devices,
|
||||
IMicrophoneSwitchPromptService prompts,
|
||||
ILogger<MicrophoneSwitchMonitor> logger,
|
||||
TimeSpan pollInterval,
|
||||
TimeSpan promptLifetime)
|
||||
{
|
||||
this.devices = devices;
|
||||
this.prompts = prompts;
|
||||
this.logger = logger;
|
||||
this.pollInterval = pollInterval;
|
||||
this.promptLifetime = promptLifetime;
|
||||
}
|
||||
|
||||
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
await Task.Delay(pollInterval, cancellationToken);
|
||||
var available = ReadDevices();
|
||||
if (available is null) continue;
|
||||
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var (id, prompt) in pending.ToArray())
|
||||
{
|
||||
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
|
||||
if (prompt.Task.IsCompleted)
|
||||
{
|
||||
prompt.Cancellation.Dispose();
|
||||
pending.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var device in available)
|
||||
{
|
||||
if (known.Add(device.Id))
|
||||
{
|
||||
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
promptCancellation.CancelAfter(promptLifetime);
|
||||
pending.Add(device.Id, new PendingPrompt(promptCancellation,
|
||||
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
|
||||
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
|
||||
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
|
||||
&& !cancellationToken.IsCancellationRequested
|
||||
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
|
||||
&& !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
switchMicrophone(device, cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
|
||||
}
|
||||
}
|
||||
|
||||
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
|
||||
{
|
||||
try
|
||||
{
|
||||
return devices.GetAvailableMicrophones();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not check for newly available microphones");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
|
||||
}
|
||||
@@ -4,11 +4,12 @@ using NAudio.Wave;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource
|
||||
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
|
||||
{
|
||||
private readonly IWaveIn capture;
|
||||
private readonly string sourceName;
|
||||
private readonly ILogger logger;
|
||||
private int disposed;
|
||||
|
||||
public NaudioCaptureAudioSource(
|
||||
IWaveIn capture,
|
||||
@@ -22,13 +23,15 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource
|
||||
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureWith(capture, sourceName, logger, cancellationToken);
|
||||
return CaptureWith(cancellationToken);
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<AudioChunk> CaptureWith(
|
||||
IWaveIn capture,
|
||||
string sourceName,
|
||||
ILogger logger,
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<AudioChunk> CaptureWith(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var channel = Channel.CreateUnbounded<AudioChunk>();
|
||||
@@ -83,7 +86,7 @@ internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource
|
||||
}
|
||||
finally
|
||||
{
|
||||
capture.Dispose();
|
||||
Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,8 +117,25 @@ public sealed class OfflineTranscriptionBacklogProcessor
|
||||
pipelineOptions,
|
||||
cancellationToken);
|
||||
|
||||
if (!finishedSegments.Any(segment =>
|
||||
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
|
||||
}
|
||||
|
||||
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
|
||||
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
|
||||
await transcriptStore.AppendLinesAsync(session,
|
||||
[
|
||||
"",
|
||||
"## Offline transcription recovery",
|
||||
"",
|
||||
"> The following transcript was recovered by replaying the entire recording. "
|
||||
+ "It may duplicate earlier transcript passages. "
|
||||
+ "Treat repeated passages as the same discussion when summarizing.",
|
||||
"",
|
||||
.. lines
|
||||
], cancellationToken);
|
||||
|
||||
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
|
||||
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
internal static class PortableMeetingAudioRegistration
|
||||
{
|
||||
public static void Add(IServiceCollection services)
|
||||
{
|
||||
Add(services, OperatingSystem.IsMacOS());
|
||||
}
|
||||
|
||||
internal static void Add(IServiceCollection services, bool isMacOs)
|
||||
{
|
||||
if (!isMacOs)
|
||||
{
|
||||
services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||
return;
|
||||
}
|
||||
|
||||
services.AddSingleton<IMacOsAudioCaptureProcessFactory, MacOsAudioCaptureProcessFactory>();
|
||||
services.AddSingleton<MacOsMicrophoneAudioSource>();
|
||||
services.AddSingleton<MacOsSystemAudioSource>();
|
||||
services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||
services.AddSingleton<IMeetingAudioSource>(serviceProvider => new CompositeMeetingAudioSource(
|
||||
serviceProvider.GetRequiredService<MacOsMicrophoneAudioSource>(),
|
||||
serviceProvider.GetRequiredService<MacOsSystemAudioSource>(),
|
||||
serviceProvider.GetRequiredService<IAcousticEchoCancellerFactory>(),
|
||||
serviceProvider.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||
}
|
||||
}
|
||||
@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
private readonly object gate = new();
|
||||
private readonly RollingAudioBuffer audioBuffer;
|
||||
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly int maxSamplesPerSpeaker;
|
||||
private readonly TimeSpan minimumUninterruptedSpeechDuration;
|
||||
private readonly TimeSpan maximumSegmentGap;
|
||||
private readonly TimeSpan minimumSampleSpeechDuration;
|
||||
private readonly TimeSpan maximumSampleDuration;
|
||||
private readonly bool requireNonOverlappingSamples;
|
||||
private readonly ILogger? logger;
|
||||
private PendingSpeakerSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
|
||||
: this(
|
||||
bufferDuration,
|
||||
maxSamplesPerSpeaker,
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromSeconds(1),
|
||||
logger: null)
|
||||
{
|
||||
}
|
||||
private SpeakerSampleSpan? pendingSpan;
|
||||
|
||||
public SpeakerAudioSampleCollector(
|
||||
TimeSpan bufferDuration,
|
||||
int maxSamplesPerSpeaker,
|
||||
TimeSpan minimumUninterruptedSpeechDuration,
|
||||
TimeSpan maximumSegmentGap,
|
||||
ILogger? logger = null)
|
||||
TimeSpan minimumSampleSpeechDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
ILogger? logger = null,
|
||||
bool requireNonOverlappingSamples = false)
|
||||
{
|
||||
SpeakerSampleDurationConfiguration.ValidateOrThrow(
|
||||
minimumSampleSpeechDuration,
|
||||
maximumSampleDuration,
|
||||
"Speaker sample collector configuration is invalid");
|
||||
audioBuffer = new RollingAudioBuffer(bufferDuration);
|
||||
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
|
||||
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
|
||||
? minimumUninterruptedSpeechDuration
|
||||
: TimeSpan.Zero;
|
||||
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
|
||||
? maximumSegmentGap
|
||||
: TimeSpan.Zero;
|
||||
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
|
||||
this.maximumSampleDuration = maximumSampleDuration;
|
||||
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
lock (gate)
|
||||
{
|
||||
samplesBySpeaker.Clear();
|
||||
lastAcceptedEndBySpeaker.Clear();
|
||||
pendingSpan = null;
|
||||
audioBuffer.Reset();
|
||||
}
|
||||
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return null;
|
||||
}
|
||||
|
||||
TranscriptionSegment sampleSegment;
|
||||
PendingSpanReset? reset;
|
||||
SpeakerSampleSpan? sampleSpan;
|
||||
SpeakerSampleSpan? previousSpan;
|
||||
lock (gate)
|
||||
{
|
||||
(sampleSegment, reset) = ExtendPendingSpan(segment);
|
||||
if (requireNonOverlappingSamples &&
|
||||
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
|
||||
segment.Start < lastAcceptedEnd)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
|
||||
segment.Speaker,
|
||||
lastAcceptedEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
|
||||
}
|
||||
|
||||
if (reset is not null)
|
||||
if (sampleSpan is null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
|
||||
reset.PreviousSpeaker,
|
||||
segment.Speaker,
|
||||
reset.PreviousEnd,
|
||||
segment.Start,
|
||||
reset.Gap,
|
||||
maximumSegmentGap);
|
||||
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
|
||||
segment.Speaker);
|
||||
return null;
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
|
||||
var sampleSegment = sampleSpan.ToSegment();
|
||||
|
||||
if (previousSpan is not null)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
|
||||
previousSpan.Speaker,
|
||||
segment.Speaker,
|
||||
previousSpan.End,
|
||||
segment.Start);
|
||||
}
|
||||
|
||||
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
|
||||
if (!score.Accepted)
|
||||
{
|
||||
logger?.LogInformation(
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
|
||||
sampleSegment.Speaker,
|
||||
score.Reason,
|
||||
sampleSpan.SpeechDuration,
|
||||
sampleSegment.End - sampleSegment.Start,
|
||||
minimumUninterruptedSpeechDuration,
|
||||
minimumSampleSpeechDuration,
|
||||
score.WordCount);
|
||||
return null;
|
||||
}
|
||||
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
|
||||
lock (gate)
|
||||
{
|
||||
if (requireNonOverlappingSamples &&
|
||||
ReferenceEquals(pendingSpan, sampleSpan))
|
||||
{
|
||||
pendingSpan = null;
|
||||
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
|
||||
}
|
||||
|
||||
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
|
||||
{
|
||||
samples = [];
|
||||
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
|
||||
{
|
||||
if (pendingSpan is null ||
|
||||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
|
||||
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
|
||||
{
|
||||
var reset = pendingSpan is null
|
||||
? null
|
||||
: new PendingSpanReset(
|
||||
pendingSpan.Speaker,
|
||||
pendingSpan.End,
|
||||
segment.Start - pendingSpan.End);
|
||||
pendingSpan = new PendingSpeakerSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
segment.End,
|
||||
[segment.Text]);
|
||||
return (pendingSpan.ToSegment(), reset);
|
||||
pendingSpan = extended;
|
||||
return (pendingSpan, null);
|
||||
}
|
||||
|
||||
pendingSpan = pendingSpan.Extend(segment);
|
||||
return (pendingSpan.ToSegment(), null);
|
||||
var previousSpan = pendingSpan;
|
||||
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
return (pendingSpan, previousSpan);
|
||||
}
|
||||
|
||||
private static SampleScore Score(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan minimumUninterruptedSpeechDuration)
|
||||
TimeSpan speechDuration,
|
||||
TimeSpan minimumSampleSpeechDuration)
|
||||
{
|
||||
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
|
||||
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
|
||||
var durationSeconds = speechDuration.TotalSeconds;
|
||||
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
|
||||
{
|
||||
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
|
||||
}
|
||||
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
|
||||
}
|
||||
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
|
||||
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
|
||||
var wordScore = Math.Min(words / 60.0, 1);
|
||||
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
|
||||
segment.Text.TrimEnd().EndsWith('?') ||
|
||||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
|
||||
|
||||
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
|
||||
|
||||
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
|
||||
|
||||
private sealed record PendingSpeakerSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
IReadOnlyList<string> TextParts)
|
||||
{
|
||||
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
|
||||
{
|
||||
return this with
|
||||
{
|
||||
End = segment.End > End ? segment.End : End,
|
||||
TextParts = TextParts.Append(segment.Text).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,6 @@ public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource
|
||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
throw new PlatformNotSupportedException(
|
||||
"Meeting audio capture is only implemented for the Windows target in this version.");
|
||||
"Meeting audio capture is supported on Windows and macOS, but is unavailable on this host.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,25 @@ public sealed class VaultTranscriptStore : ITranscriptStore
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public Task AppendLinesAsync(
|
||||
TranscriptSession session,
|
||||
IReadOnlyList<string> lines,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (lines.Count == 0)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
return WithFileGateAsync(
|
||||
session.TranscriptPath,
|
||||
() => File.AppendAllTextAsync(
|
||||
session.TranscriptPath,
|
||||
string.Concat(lines.Select(line => line + Environment.NewLine)),
|
||||
cancellationToken),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ReplaceLineAsync(
|
||||
TranscriptSession session,
|
||||
TranscriptLineReference lineReference,
|
||||
|
||||
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
private const string NotificationGroup = "meeting-inactivity";
|
||||
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
|
||||
private readonly object registrationGate = new();
|
||||
private readonly object notificationGate = new();
|
||||
private bool registered;
|
||||
|
||||
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
|
||||
@@ -37,24 +37,27 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
try
|
||||
{
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
lock (notificationGate)
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
EnsureRegistered();
|
||||
var promptId = Guid.NewGuid().ToString("N");
|
||||
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
|
||||
logger.LogInformation(
|
||||
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
var notification = BuildNotification(promptId, request);
|
||||
notification.Show(toast =>
|
||||
{
|
||||
toast.Group = NotificationGroup;
|
||||
toast.Tag = promptId;
|
||||
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
|
||||
});
|
||||
logger.LogInformation(
|
||||
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
|
||||
promptId,
|
||||
request.Threshold);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -76,6 +79,36 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DismissAllAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (pendingPrompts.IsEmpty)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
pendingPrompts.Clear();
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
|
||||
logger.LogInformation("Dismissed all native Windows inactivity notifications");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
@@ -83,29 +116,28 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (!registered)
|
||||
lock (notificationGate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (!registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
|
||||
registered = false;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void EnsureRegistered()
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (registrationGate)
|
||||
lock (notificationGate)
|
||||
{
|
||||
if (registered)
|
||||
{
|
||||
@@ -122,29 +154,25 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
string promptId,
|
||||
MeetingInactivityPromptRequest request)
|
||||
{
|
||||
var yesButton = new ToastButton()
|
||||
.SetContent("Yes")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "stop")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
var noButton = new ToastButton()
|
||||
.SetContent("No")
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", "continue")
|
||||
.SetBackgroundActivation();
|
||||
|
||||
return new ToastContentBuilder()
|
||||
var builder = new ToastContentBuilder()
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText("Stop meeting?")
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
|
||||
.AddButton(yesButton)
|
||||
.AddButton(noButton);
|
||||
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
|
||||
|
||||
foreach (var action in MeetingInactivityPromptActions.All)
|
||||
{
|
||||
builder.AddButton(new ToastButton()
|
||||
.SetContent(action.Content)
|
||||
.AddArgument("source", NotificationSource)
|
||||
.AddArgument("promptId", promptId)
|
||||
.AddArgument("response", action.ResponseArgument)
|
||||
.SetBackgroundActivation());
|
||||
}
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
|
||||
@@ -177,10 +205,9 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
|
||||
return;
|
||||
}
|
||||
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue) &&
|
||||
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
|
||||
? MeetingInactivityPromptResponse.Stop
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
var response = TryGetArgument(arguments, "response", out var responseValue)
|
||||
? MeetingInactivityPromptActions.ParseResponse(responseValue)
|
||||
: MeetingInactivityPromptResponse.Continue;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#if WINDOWS
|
||||
using System.Collections.Concurrent;
|
||||
using CommunityToolkit.WinUI.Notifications;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
|
||||
{
|
||||
private const string Source = "meeting-microphone-switch";
|
||||
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
|
||||
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
|
||||
private readonly object gate = new();
|
||||
private bool registered;
|
||||
|
||||
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var id = Guid.NewGuid().ToString("N");
|
||||
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
try
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (!registered)
|
||||
{
|
||||
ToastNotificationManagerCompat.OnActivated += OnActivated;
|
||||
registered = true;
|
||||
}
|
||||
|
||||
pending[id] = response;
|
||||
new ToastContentBuilder()
|
||||
.AddArgument("source", Source)
|
||||
.AddArgument("promptId", id)
|
||||
.SetToastScenario(ToastScenario.Reminder)
|
||||
.SetToastDuration(ToastDuration.Long)
|
||||
.AddText("New microphone available")
|
||||
.AddText($"Switch this meeting to {microphone.Name}?")
|
||||
.AddButton(new ToastButton().SetContent("Yes")
|
||||
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||
.AddArgument("response", "yes").SetBackgroundActivation())
|
||||
.AddButton(new ToastButton().SetContent("No")
|
||||
.AddArgument("source", Source).AddArgument("promptId", id)
|
||||
.AddArgument("response", "no").SetBackgroundActivation())
|
||||
.Show(toast =>
|
||||
{
|
||||
toast.Tag = id;
|
||||
toast.Group = Source;
|
||||
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
|
||||
});
|
||||
}
|
||||
|
||||
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
|
||||
return await response.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
pending.TryRemove(id, out _);
|
||||
try
|
||||
{
|
||||
ToastNotificationManagerCompat.History.Remove(id, Source);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
|
||||
{
|
||||
var arguments = NotificationActivationArguments.Parse(args.Argument);
|
||||
if (arguments.GetValueOrDefault("source") != Source ||
|
||||
!arguments.TryGetValue("promptId", out var id) ||
|
||||
!pending.TryRemove(id, out var response)) return;
|
||||
|
||||
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
|
||||
registered = false;
|
||||
foreach (var response in pending.Values) response.TrySetResult(false);
|
||||
pending.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -1,7 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
@@ -46,6 +43,25 @@ public sealed record ScreenshotOcrResult
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
public interface IScreenshotImageCropper
|
||||
{
|
||||
Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class UnavailableScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
public interface IMeetingScreenshotService
|
||||
{
|
||||
Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
@@ -147,6 +163,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
IMeetingNoteImageOcrService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IScreenshotImageCropper screenshotImageCropper;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IMeetingNoteStore meetingNoteStore;
|
||||
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
|
||||
@@ -162,6 +179,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
IMeetingNoteStore meetingNoteStore,
|
||||
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
IScreenshotImageCropper screenshotImageCropper,
|
||||
ILogger<MeetingScreenshotService> logger,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
{
|
||||
@@ -171,6 +189,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
this.attendeeCanonicalizer = attendeeCanonicalizer;
|
||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||
this.ocrClient = ocrClient;
|
||||
this.screenshotImageCropper = screenshotImageCropper;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
@@ -639,17 +658,20 @@ public sealed partial class MeetingScreenshotService :
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
|
||||
var croppedPath = await screenshotImageCropper.SaveCroppedScreenshotAsync(
|
||||
screenshotPath,
|
||||
crop,
|
||||
cancellationToken);
|
||||
if (croppedPath is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because no image cropper is available on this host",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
croppedPath));
|
||||
@@ -657,7 +679,7 @@ public sealed partial class MeetingScreenshotService :
|
||||
Environment.NewLine +
|
||||
Environment.NewLine;
|
||||
}
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
@@ -667,39 +689,6 @@ public sealed partial class MeetingScreenshotService :
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CA1416
|
||||
private static Task<string> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult(croppedPath);
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackOcr(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationTokenSource cancellation,
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#if WINDOWS
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
internal sealed class WindowsScreenshotImageCropper : IScreenshotImageCropper
|
||||
{
|
||||
#pragma warning disable CA1416
|
||||
public Task<string?> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult<string?>(croppedPath);
|
||||
}
|
||||
catch (ExternalException exception)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"The screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} could not be applied.",
|
||||
exception);
|
||||
}
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(
|
||||
ScreenshotCropCoordinates crop,
|
||||
int imageWidth,
|
||||
int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
|
||||
return [];
|
||||
}
|
||||
|
||||
return await finalizer.FinalizeAsync(
|
||||
return await finalizer.FinalizeEnabledAsync(
|
||||
wavPath,
|
||||
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
|
||||
options.Diarization,
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerSnippetExtractor snippetExtractor;
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentificationService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerSnippetExtractor snippetExtractor,
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
ResemblyzerVoiceClusterMatcher clusterMatcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentificationService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.snippetExtractor = snippetExtractor;
|
||||
this.encoder = encoder;
|
||||
this.clusterMatcher = clusterMatcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task ApplySpeakerOverrideAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string sourceSpeaker,
|
||||
string targetSpeaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled ||
|
||||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
|
||||
string.IsNullOrWhiteSpace(targetSpeaker) ||
|
||||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = sourceSpeaker.Trim();
|
||||
var targetName = targetSpeaker.Trim();
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var sourceCandidate = identities
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
|
||||
.OrderBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
|
||||
if (target is null && sourceCandidate is null && vectors.Count == 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
|
||||
sourceLabel,
|
||||
targetName);
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (target is null)
|
||||
{
|
||||
target = sourceCandidate ?? new SpeakerIdentity
|
||||
{
|
||||
CreatedAt = now
|
||||
};
|
||||
if (target.Id == 0)
|
||||
{
|
||||
context.SpeakerIdentities.Add(target);
|
||||
}
|
||||
}
|
||||
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
|
||||
{
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
sourceCandidate,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
context.SpeakerIdentities.Remove(sourceCandidate);
|
||||
}
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
|
||||
outlierPruner.AddAndPrune(
|
||||
target,
|
||||
vectors,
|
||||
now);
|
||||
target.UpdatedAt = now;
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
target.References,
|
||||
sourceLabel,
|
||||
targetName,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteSpeakerIdentityAsync(
|
||||
string identity,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
var target = identities.FirstOrDefault(candidate =>
|
||||
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
|
||||
if (target is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
context.SpeakerIdentities.Remove(target);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
bool final,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled || request.Segments.Count == 0)
|
||||
{
|
||||
return EmptyResult(request.Segments);
|
||||
}
|
||||
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var knownMappings = request.KnownSpeakerMappings ??
|
||||
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
if (final && knownMappings.Count > 0)
|
||||
{
|
||||
await PersistMappedSpeakerEvidenceAsync(
|
||||
context,
|
||||
request,
|
||||
knownMappings,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
var identifiedNames = knownMappings.Values
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var segmentSpeaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
identifiedNames.Add(segmentSpeaker.Trim());
|
||||
}
|
||||
|
||||
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
var pendingAudits = new List<PendingIdentificationAudit>();
|
||||
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAutomaticVectorsAsync(
|
||||
request,
|
||||
speaker,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
|
||||
speaker,
|
||||
vectors.Count,
|
||||
resemblyzerOptions.RequiredVectorsPerSpeaker);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (identity, decision) = await FindMatchAsync(
|
||||
context,
|
||||
attendees,
|
||||
identifiedNames,
|
||||
vectors,
|
||||
cancellationToken);
|
||||
if (identity is null)
|
||||
{
|
||||
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
|
||||
{
|
||||
unmatchedSpeakers.Add((speaker, vectors));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousCanonicalName = identity.CanonicalName;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
UpdateMatchedIdentity(identity, attendees);
|
||||
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
|
||||
!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
}
|
||||
|
||||
var displayName = identity.GetDisplayName();
|
||||
if (!string.IsNullOrWhiteSpace(displayName))
|
||||
{
|
||||
mappings[speaker] = displayName;
|
||||
identifiedNames.Add(displayName);
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
identifiedNames.Add(acceptedName);
|
||||
}
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
displayName,
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
if (final)
|
||||
{
|
||||
LearnUnmatchedSpeakers(
|
||||
context,
|
||||
request.MeetingNote,
|
||||
attendees,
|
||||
matchedAcceptedNames,
|
||||
unmatchedSpeakers,
|
||||
pendingAudits);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
await AppendAuditsAsync(pendingAudits, cancellationToken);
|
||||
var relabeled = request.Segments
|
||||
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
|
||||
? segment with { Speaker = name }
|
||||
: segment)
|
||||
.ToList();
|
||||
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
|
||||
}
|
||||
|
||||
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> identifiedNames,
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
|
||||
var identities = await LoadIdentities(context)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var candidates = identities
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
|
||||
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.OrderByDescending(candidate => candidate.IsAttendee)
|
||||
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
|
||||
.ThenBy(candidate => candidate.Identity.Id)
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
|
||||
candidate.Identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
candidate.Identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
var match = clusterMatcher.Match(queryVectors, candidates);
|
||||
return (
|
||||
match.IdentityId is { } identityId
|
||||
? identities.Single(identity => identity.Id == identityId)
|
||||
: null,
|
||||
match);
|
||||
}
|
||||
|
||||
private void LearnUnmatchedSpeakers(
|
||||
SpeakerIdentityDbContext context,
|
||||
MeetingNote meetingNote,
|
||||
IReadOnlyList<string> attendees,
|
||||
IReadOnlySet<string> matchedAcceptedNames,
|
||||
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
|
||||
ICollection<PendingIdentificationAudit> pendingAudits)
|
||||
{
|
||||
var candidates = attendees
|
||||
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var (speaker, vectors) in unmatchedSpeakers)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var identity = new SpeakerIdentity
|
||||
{
|
||||
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
CandidateNames = candidates
|
||||
.Select(name => new SpeakerCandidateName { Name = name })
|
||||
.ToList(),
|
||||
References = [CreateReference(meetingNote, now)]
|
||||
};
|
||||
outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
context.SpeakerIdentities.Add(identity);
|
||||
logger.LogInformation(
|
||||
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
|
||||
speaker,
|
||||
identity.VoiceVectors.Count,
|
||||
string.Join(", ", candidates));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
pendingAudits.Add(new PendingIdentificationAudit(
|
||||
identity,
|
||||
speaker,
|
||||
identity.CanonicalName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AppendAuditsAsync(
|
||||
IEnumerable<PendingIdentificationAudit> pendingAudits,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
|
||||
audit.Identity.References,
|
||||
audit.Speaker,
|
||||
audit.Name,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback,
|
||||
cancellationToken);
|
||||
if (samples.Count < requiredCount)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task PersistMappedSpeakerEvidenceAsync(
|
||||
SpeakerIdentityDbContext context,
|
||||
SpeakerIdentificationRequest request,
|
||||
IReadOnlyDictionary<string, string> knownMappings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
|
||||
foreach (var (speaker, mappedName) in knownMappings)
|
||||
{
|
||||
var identity = identities
|
||||
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
|
||||
.OrderBy(candidate =>
|
||||
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(candidate => candidate.Id)
|
||||
.FirstOrDefault();
|
||||
if (identity is null)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
|
||||
speaker,
|
||||
mappedName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
|
||||
if (vectors.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var previousReferenceCount = identity.References.Count;
|
||||
var vectorUpdate = outlierPruner.AddAndPrune(
|
||||
identity,
|
||||
vectors,
|
||||
now);
|
||||
SpeakerIdentityReferences.AddIfMissing(
|
||||
identity,
|
||||
CreateReference(request.MeetingNote, now),
|
||||
now);
|
||||
if (identity.References.Count != previousReferenceCount)
|
||||
{
|
||||
identity.UpdatedAt = now;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
|
||||
vectorUpdate.AddedCount,
|
||||
speaker,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = await ResolveWavSamplesAsync(
|
||||
request,
|
||||
speaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
allowAudioFallback: true,
|
||||
cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return await encoder.EncodeAsync(samples, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
|
||||
SpeakerIdentificationRequest request,
|
||||
string speaker,
|
||||
int maxSamples,
|
||||
bool allowAudioFallback,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var suppliedSamples = request.Samples?
|
||||
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(sample => sample.WavBytes.Length > 0)
|
||||
.OrderByDescending(sample => sample.Score)
|
||||
.Take(maxSamples)
|
||||
.ToList()
|
||||
?? [];
|
||||
var wavSamples = suppliedSamples
|
||||
.Select(sample => sample.WavBytes)
|
||||
.ToList();
|
||||
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
|
||||
{
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
request.Segments.Count);
|
||||
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
|
||||
{
|
||||
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
|
||||
request.AudioPath,
|
||||
span,
|
||||
cancellationToken);
|
||||
if (wavBytes.Length > 0)
|
||||
{
|
||||
wavSamples.Add(wavBytes);
|
||||
}
|
||||
|
||||
if (wavSamples.Count >= maxSamples)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
|
||||
wavSamples.Count,
|
||||
maxSamples,
|
||||
speaker,
|
||||
suppliedSamples.Count,
|
||||
wavSamples.Count - suppliedSamples.Count);
|
||||
return wavSamples;
|
||||
}
|
||||
|
||||
private static bool OverlapsSuppliedSample(
|
||||
IReadOnlyList<TranscriptionSegment> span,
|
||||
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
|
||||
{
|
||||
if (span.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var start = span[0].Start;
|
||||
var end = span[^1].End;
|
||||
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
|
||||
}
|
||||
|
||||
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
|
||||
{
|
||||
return context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References);
|
||||
}
|
||||
|
||||
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
|
||||
}
|
||||
|
||||
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
|
||||
}
|
||||
|
||||
private sealed record PendingIdentificationAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string Speaker,
|
||||
string Name);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ResemblyzerVoiceClusterMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
|
||||
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
|
||||
|
||||
public ResemblyzerSpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ResemblyzerVoiceClusterMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
resemblyzerOptions = this.options.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
|
||||
TimeSpan? recentIdentityAge,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var recentIds = identities
|
||||
.Where(identity => identity.CreatedAt >= cutoff)
|
||||
.Select(identity => identity.Id)
|
||||
.ToList();
|
||||
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
|
||||
var attempts = 0;
|
||||
var mergedPairs = 0;
|
||||
var pendingAudits = new List<PendingMergeAudit>();
|
||||
|
||||
foreach (var sourceId in recentIds)
|
||||
{
|
||||
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
|
||||
if (source is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
|
||||
source,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)
|
||||
.Take(required * 2)
|
||||
.ToList();
|
||||
if (sourceVectors.Count < required * 2)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
|
||||
source.Id,
|
||||
sourceVectors.Count,
|
||||
required * 2);
|
||||
continue;
|
||||
}
|
||||
|
||||
var candidates = identities
|
||||
.Where(identity => identity.Id != source.Id)
|
||||
.Where(identity => identity.VoiceVectors.Any(vector =>
|
||||
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
|
||||
.Take(Math.Max(1, options.MaxMatchCandidates))
|
||||
.Select(identity => new ResemblyzerVoiceVectorCandidate(
|
||||
identity.Id,
|
||||
SpeakerVoiceVectors.DecodeCompatible(
|
||||
identity,
|
||||
resemblyzerOptions.ModelId,
|
||||
logger)))
|
||||
.Where(candidate => candidate.Vectors.Count > 0)
|
||||
.ToList();
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
|
||||
if (!first.Accepted || first.IdentityId is not { } targetId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
attempts++;
|
||||
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
|
||||
if (!second.Accepted || second.IdentityId != targetId)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
|
||||
source.Id,
|
||||
first.IdentityId,
|
||||
second.IdentityId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
|
||||
if (target is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
resemblyzerOptions.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
pendingAudits.Add(new PendingMergeAudit(
|
||||
target,
|
||||
targetName,
|
||||
sourceName));
|
||||
context.SpeakerIdentities.Remove(source);
|
||||
identities.Remove(source);
|
||||
mergedPairs++;
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
foreach (var audit in pendingAudits)
|
||||
{
|
||||
try
|
||||
{
|
||||
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
|
||||
audit.Identity.References,
|
||||
audit.TargetName,
|
||||
audit.SourceName,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
|
||||
audit.Identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return new SpeakerIdentityMergeResult(
|
||||
recentIds.Count,
|
||||
identities.Count,
|
||||
attempts,
|
||||
mergedPairs);
|
||||
}
|
||||
|
||||
private sealed record PendingMergeAudit(
|
||||
SpeakerIdentity Identity,
|
||||
string TargetName,
|
||||
string SourceName);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorCandidate(
|
||||
int IdentityId,
|
||||
IReadOnlyList<float[]> Vectors);
|
||||
|
||||
public sealed record ResemblyzerVoiceClusterMatchResult(
|
||||
int? IdentityId,
|
||||
string Reason,
|
||||
double Cohesion,
|
||||
double? BestSimilarity,
|
||||
double? RunnerUpSimilarity)
|
||||
{
|
||||
public bool Accepted => IdentityId.HasValue;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceClusterMatcher
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
|
||||
|
||||
public ResemblyzerVoiceClusterMatcher(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceClusterMatcher> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceClusterMatchResult Match(
|
||||
IReadOnlyList<float[]> queryVectors,
|
||||
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
|
||||
{
|
||||
var requiredCount = options.RequiredVectorsPerSpeaker;
|
||||
if (queryVectors.Count < requiredCount)
|
||||
{
|
||||
return Reject(
|
||||
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
|
||||
cohesion: 0,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var normalizedQuery = queryVectors
|
||||
.Take(requiredCount)
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var cohesion = MeanPairwiseCosine(normalizedQuery);
|
||||
if (cohesion < options.MinimumClusterCohesion)
|
||||
{
|
||||
return Reject(
|
||||
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
|
||||
cohesion,
|
||||
bestSimilarity: null,
|
||||
runnerUpSimilarity: null);
|
||||
}
|
||||
|
||||
var scored = candidates
|
||||
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
|
||||
.Where(candidate => candidate is not null)
|
||||
.Select(candidate => candidate!.Value)
|
||||
.OrderByDescending(candidate => candidate.Similarity)
|
||||
.ThenBy(candidate => candidate.IdentityId)
|
||||
.ToList();
|
||||
if (scored.Count == 0)
|
||||
{
|
||||
return Reject("no compatible identity vectors were available", cohesion, null, null);
|
||||
}
|
||||
|
||||
var best = scored[0];
|
||||
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
|
||||
if (best.Similarity < options.MinimumIdentitySimilarity)
|
||||
{
|
||||
return Reject(
|
||||
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
if (runnerUp is { } runnerUpSimilarity &&
|
||||
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
|
||||
{
|
||||
return Reject(
|
||||
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
|
||||
best.IdentityId,
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
best.IdentityId,
|
||||
"accepted",
|
||||
cohesion,
|
||||
best.Similarity,
|
||||
runnerUp);
|
||||
}
|
||||
|
||||
private ResemblyzerVoiceClusterMatchResult Reject(
|
||||
string reason,
|
||||
double cohesion,
|
||||
double? bestSimilarity,
|
||||
double? runnerUpSimilarity)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
return new ResemblyzerVoiceClusterMatchResult(
|
||||
null,
|
||||
reason,
|
||||
cohesion,
|
||||
bestSimilarity,
|
||||
runnerUpSimilarity);
|
||||
}
|
||||
|
||||
private (int IdentityId, double Similarity)? ScoreCandidate(
|
||||
IReadOnlyList<float[]> normalizedQuery,
|
||||
ResemblyzerVoiceVectorCandidate candidate)
|
||||
{
|
||||
if (candidate.Vectors.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var normalizedCandidate = candidate.Vectors
|
||||
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
|
||||
.ToList();
|
||||
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
|
||||
var similarities = normalizedQuery
|
||||
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
|
||||
.Order()
|
||||
.ToList();
|
||||
return (candidate.IdentityId, Median(similarities));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
|
||||
candidate.IdentityId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] Centroid(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var centroid = new float[ResemblyzerVectorContract.Dimensions];
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] += vector[index];
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = 0; index < centroid.Length; index++)
|
||||
{
|
||||
centroid[index] /= vectors.Count;
|
||||
}
|
||||
|
||||
return centroid;
|
||||
}
|
||||
|
||||
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
if (vectors.Count < 2)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
double total = 0;
|
||||
var pairs = 0;
|
||||
for (var first = 0; first < vectors.Count - 1; first++)
|
||||
{
|
||||
for (var second = first + 1; second < vectors.Count; second++)
|
||||
{
|
||||
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
|
||||
pairs++;
|
||||
}
|
||||
}
|
||||
|
||||
return total / pairs;
|
||||
}
|
||||
|
||||
private static double Median(IReadOnlyList<double> sortedValues)
|
||||
{
|
||||
var middle = sortedValues.Count / 2;
|
||||
return sortedValues.Count % 2 == 0
|
||||
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
|
||||
: sortedValues[middle];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record ResemblyzerVoiceVectorUpdateResult(
|
||||
int AddedCount,
|
||||
int RemovedCount)
|
||||
{
|
||||
public bool Changed => AddedCount > 0 || RemovedCount > 0;
|
||||
}
|
||||
|
||||
public sealed class ResemblyzerVoiceVectorOutlierPruner
|
||||
{
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
|
||||
|
||||
public ResemblyzerVoiceVectorOutlierPruner(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
|
||||
{
|
||||
this.options = options;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public int Prune(SpeakerIdentity identity)
|
||||
{
|
||||
var compatible = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
|
||||
identity,
|
||||
options.ModelId,
|
||||
logger))
|
||||
{
|
||||
try
|
||||
{
|
||||
compatible.Add(new DecodedSpeakerVoiceVector(
|
||||
entry.Stored,
|
||||
SpeakerVoiceVectors.Normalize(
|
||||
entry.Vector,
|
||||
$"Stored voice vector {entry.Stored.Id}")));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
|
||||
entry.Stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
var minimumVectors = options.OutlierPruningMinimumVectors;
|
||||
if (compatible.Count < minimumVectors)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
|
||||
var ranked = clusters
|
||||
.GroupBy(cluster => cluster)
|
||||
.Where(group => group.Key > 0)
|
||||
.Select(group => new { Id = group.Key, Count = group.Count() })
|
||||
.OrderByDescending(group => group.Count)
|
||||
.ThenBy(group => group.Id)
|
||||
.ToList();
|
||||
if (ranked.Count == 0)
|
||||
{
|
||||
return Skip(identity, "no dense cluster was found");
|
||||
}
|
||||
|
||||
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
|
||||
{
|
||||
return Skip(identity, "no uniquely largest dense cluster was found");
|
||||
}
|
||||
|
||||
var dominant = ranked[0];
|
||||
var ratio = (double)dominant.Count / compatible.Count;
|
||||
if (ratio < options.OutlierPruningMinimumClusterRatio)
|
||||
{
|
||||
return Skip(
|
||||
identity,
|
||||
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
|
||||
}
|
||||
|
||||
var removed = compatible
|
||||
.Where((_, index) => clusters[index] != dominant.Id)
|
||||
.Select(item => item.Stored)
|
||||
.ToList();
|
||||
foreach (var vector in removed)
|
||||
{
|
||||
identity.VoiceVectors.Remove(vector);
|
||||
}
|
||||
|
||||
if (removed.Count > 0)
|
||||
{
|
||||
identity.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
|
||||
removed.Count,
|
||||
identity.Id,
|
||||
dominant.Count,
|
||||
compatible.Count);
|
||||
return removed.Count;
|
||||
}
|
||||
|
||||
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<float[]> vectors,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var removedBeforeAdding = Prune(identity);
|
||||
var added = SpeakerVoiceVectors.AddDistinct(
|
||||
identity,
|
||||
vectors,
|
||||
options.ModelId,
|
||||
options.MaxVectorsPerIdentity,
|
||||
createdAt);
|
||||
var removedAfterAdding = Prune(identity);
|
||||
var result = new ResemblyzerVoiceVectorUpdateResult(
|
||||
added,
|
||||
removedBeforeAdding + removedAfterAdding);
|
||||
if (result.Changed)
|
||||
{
|
||||
identity.UpdatedAt = createdAt;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
|
||||
{
|
||||
var neighbors = Enumerable.Range(0, vectors.Count)
|
||||
.Select(index => Enumerable.Range(0, vectors.Count)
|
||||
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
|
||||
.ToList())
|
||||
.ToList();
|
||||
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
|
||||
var labels = new int[vectors.Count];
|
||||
var clusterId = 0;
|
||||
|
||||
for (var point = 0; point < vectors.Count; point++)
|
||||
{
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
labels[point] = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
clusterId++;
|
||||
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
|
||||
}
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
private static void ExpandCluster(
|
||||
int seed,
|
||||
int clusterId,
|
||||
IReadOnlyList<List<int>> neighbors,
|
||||
int[] labels,
|
||||
int minimumNeighbors)
|
||||
{
|
||||
labels[seed] = clusterId;
|
||||
var pending = new Queue<int>(neighbors[seed]);
|
||||
var queued = neighbors[seed].ToHashSet();
|
||||
while (pending.TryDequeue(out var point))
|
||||
{
|
||||
if (labels[point] == -1)
|
||||
{
|
||||
labels[point] = clusterId;
|
||||
}
|
||||
|
||||
if (labels[point] != 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
labels[point] = clusterId;
|
||||
if (neighbors[point].Count < minimumNeighbors)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var neighbor in neighbors[point])
|
||||
{
|
||||
if (queued.Add(neighbor))
|
||||
{
|
||||
pending.Enqueue(neighbor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int Skip(
|
||||
SpeakerIdentity identity,
|
||||
string reason)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
|
||||
identity.Id,
|
||||
reason);
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed class ResemblyzerWarmupHostedService : BackgroundService
|
||||
{
|
||||
private readonly IResemblyzerVoiceEncoder encoder;
|
||||
private readonly bool speakerIdentificationEnabled;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
|
||||
public ResemblyzerWarmupHostedService(
|
||||
IResemblyzerVoiceEncoder encoder,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<ResemblyzerWarmupHostedService> logger)
|
||||
{
|
||||
this.encoder = encoder;
|
||||
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!speakerIdentificationEnabled || !options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Yield();
|
||||
try
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
await encoder.WarmUpAsync(stoppingToken);
|
||||
logger.LogInformation(
|
||||
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
|
||||
VaultPath.Resolve(options.RuntimeFolder),
|
||||
options.ModelId);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
|
||||
VaultPath.Resolve(options.RuntimeFolder));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
|
||||
|
||||
public List<SpeakerSnippet> Snippets { get; set; } = [];
|
||||
|
||||
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
|
||||
|
||||
public List<SpeakerIdentityReference> References { get; set; } = [];
|
||||
|
||||
[NotMapped]
|
||||
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
|
||||
string.IsNullOrWhiteSpace(reference.TranscriptPath);
|
||||
}
|
||||
|
||||
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerVoiceVector
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public int SpeakerIdentityId { get; set; }
|
||||
|
||||
public SpeakerIdentity? SpeakerIdentity { get; set; }
|
||||
|
||||
public string ModelId { get; set; } = "";
|
||||
|
||||
public int Dimensions { get; set; }
|
||||
|
||||
public byte[] VectorBytes { get; set; } = [];
|
||||
|
||||
public string Fingerprint { get; set; } = "";
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class SpeakerAlias
|
||||
{
|
||||
public int Id { get; set; }
|
||||
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
|
||||
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
|
||||
|
||||
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
|
||||
|
||||
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.VoiceVectors)
|
||||
.WithOne(vector => vector.SpeakerIdentity)
|
||||
.HasForeignKey(vector => vector.SpeakerIdentityId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
|
||||
entity.HasMany(identity => identity.References)
|
||||
.WithOne(reference => reference.SpeakerIdentity)
|
||||
.HasForeignKey(reference => reference.SpeakerIdentityId)
|
||||
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
|
||||
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
|
||||
.IsUnique();
|
||||
|
||||
modelBuilder.Entity<SpeakerVoiceVector>()
|
||||
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
|
||||
|
||||
modelBuilder.Entity<SpeakerIdentityReference>()
|
||||
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
|
||||
.IsUnique();
|
||||
|
||||
@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
{
|
||||
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
|
||||
private readonly ISpeakerIdentityMatcher matcher;
|
||||
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
|
||||
private readonly SpeakerIdentificationOptions options;
|
||||
private readonly ILogger<SpeakerIdentityMergeService> logger;
|
||||
|
||||
public SpeakerIdentityMergeService(
|
||||
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
|
||||
ISpeakerIdentityMatcher matcher,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<SpeakerIdentityMergeService> logger)
|
||||
{
|
||||
this.dbContextFactory = dbContextFactory;
|
||||
this.matcher = matcher;
|
||||
this.outlierPruner = outlierPruner;
|
||||
this.options = options.Value.SpeakerIdentification;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
|
||||
var identities = await context.SpeakerIdentities
|
||||
.AsSplitQuery()
|
||||
.Include(identity => identity.Aliases)
|
||||
.Include(identity => identity.CandidateNames)
|
||||
.Include(identity => identity.Snippets)
|
||||
.Include(identity => identity.VoiceVectors)
|
||||
.Include(identity => identity.References)
|
||||
.OrderByDescending(identity => identity.References.Count)
|
||||
.ThenBy(identity => identity.Id)
|
||||
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
|
||||
|
||||
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
|
||||
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
|
||||
SpeakerIdentityMerger.MergeInto(
|
||||
SpeakerIdentityMerger.MergeIntoAndPrune(
|
||||
target,
|
||||
source,
|
||||
options.MaxSnippetsPerSpeaker);
|
||||
options.MaxSnippetsPerSpeaker,
|
||||
options.Resemblyzer.MaxVectorsPerIdentity,
|
||||
outlierPruner);
|
||||
logger.LogInformation(
|
||||
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
|
||||
source.Id,
|
||||
|
||||
@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityMerger
|
||||
{
|
||||
public static void MergeIntoAndPrune(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors,
|
||||
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
|
||||
{
|
||||
outlierPruner.Prune(target);
|
||||
outlierPruner.Prune(source);
|
||||
MergeInto(target, source, maxSnippets, maxVoiceVectors);
|
||||
outlierPruner.Prune(target);
|
||||
}
|
||||
|
||||
public static void MergeInto(
|
||||
SpeakerIdentity target,
|
||||
SpeakerIdentity source,
|
||||
int maxSnippets)
|
||||
int maxSnippets,
|
||||
int maxVoiceVectors = int.MaxValue)
|
||||
{
|
||||
AddAlias(target, source.CanonicalName);
|
||||
foreach (var alias in source.Aliases)
|
||||
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
|
||||
.ToList();
|
||||
target.Snippets.Clear();
|
||||
target.Snippets.AddRange(retainedSnippets);
|
||||
|
||||
var retainedVectors = target.VoiceVectors
|
||||
.Concat(source.VoiceVectors)
|
||||
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
|
||||
.OrderByDescending(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
|
||||
.Take(Math.Max(1, maxVoiceVectors))
|
||||
.Select(vector => new SpeakerVoiceVector
|
||||
{
|
||||
SpeakerIdentity = target,
|
||||
ModelId = vector.ModelId,
|
||||
Dimensions = vector.Dimensions,
|
||||
VectorBytes = vector.VectorBytes.ToArray(),
|
||||
Fingerprint = vector.Fingerprint,
|
||||
CreatedAt = vector.CreatedAt
|
||||
})
|
||||
.ToList();
|
||||
target.VoiceVectors.Clear();
|
||||
target.VoiceVectors.AddRange(retainedVectors);
|
||||
target.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerIdentityNaming
|
||||
{
|
||||
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[] { identity.CanonicalName }
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool MatchesAnyAcceptedName(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
public static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
}
|
||||
|
||||
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(MeetingAttendeeNames.NormalizeDisplayName)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool UpdateCandidateNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
var resetToAttendees = intersection.Count == 0;
|
||||
SetCandidates(identity, resetToAttendees ? attendees : intersection);
|
||||
if (intersection.Count == 1)
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
}
|
||||
|
||||
return resetToAttendees;
|
||||
}
|
||||
|
||||
public static void SetCandidates(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
|
||||
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
|
||||
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
|
||||
"SpeakerIdentityId" INTEGER NOT NULL,
|
||||
"ModelId" TEXT NOT NULL,
|
||||
"Dimensions" INTEGER NOT NULL,
|
||||
"VectorBytes" BLOB NOT NULL,
|
||||
"Fingerprint" TEXT NOT NULL,
|
||||
"CreatedAt" TEXT NOT NULL,
|
||||
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
|
||||
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
|
||||
);
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
|
||||
""",
|
||||
cancellationToken);
|
||||
await context.Database.ExecuteSqlRawAsync(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
|
||||
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
|
||||
""",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
|
||||
|
||||
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
|
||||
target.CanonicalName = targetName;
|
||||
target.UpdatedAt = now;
|
||||
ResetCandidates(target, [targetName]);
|
||||
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
|
||||
AddMeetingReference(target, meetingReference);
|
||||
var snippetAdded = AddSnippetIfNeeded(target, snippet);
|
||||
await context.SaveChangesAsync(cancellationToken);
|
||||
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
|
||||
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
|
||||
|
||||
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
|
||||
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
|
||||
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
|
||||
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var speaker in request.Segments
|
||||
.Select(segment => segment.Speaker)
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
|
||||
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(speaker.Trim());
|
||||
}
|
||||
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
matchedAcceptedNames.Add(acceptedName);
|
||||
}
|
||||
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
{
|
||||
speakerMappings[speaker] = speakerName;
|
||||
alreadyIdentifiedNames.Add(speakerName);
|
||||
foreach (var acceptedName in GetAcceptedNames(identity))
|
||||
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
|
||||
{
|
||||
alreadyIdentifiedNames.Add(acceptedName);
|
||||
}
|
||||
|
||||
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
|
||||
speakerName,
|
||||
GetAcceptedNames(identity).ToList()));
|
||||
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Select(identity => new
|
||||
{
|
||||
Identity = identity,
|
||||
IsAttendee = MatchesAttendees(identity, attendees),
|
||||
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
|
||||
IsActive = identity.UpdatedAt >= activeCutoff
|
||||
})
|
||||
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
|
||||
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
|
||||
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
|
||||
request.Segments,
|
||||
speaker,
|
||||
options.MaximumSampleSegmentGap,
|
||||
options.MinimumSampleSpeechDuration);
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration);
|
||||
logger.LogInformation(
|
||||
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
|
||||
operation,
|
||||
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Include(identity => identity.References)
|
||||
.ToListAsync(cancellationToken);
|
||||
return identities
|
||||
.Where(identity => GetAcceptedNames(identity).Contains(name))
|
||||
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
|
||||
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
|
||||
.ThenBy(identity => identity.Id)
|
||||
.FirstOrDefault();
|
||||
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
.ToListAsync(cancellationToken);
|
||||
return candidates
|
||||
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
|
||||
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
|
||||
.Where(identity => identity.CandidateNames.Any(candidate =>
|
||||
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
|
||||
identity.Aliases.Any(alias =>
|
||||
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
|
||||
}
|
||||
|
||||
private static bool IsSameReference(
|
||||
SpeakerIdentityReference first,
|
||||
SpeakerIdentityReference second)
|
||||
{
|
||||
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool MatchesAcceptedNames(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlySet<string> names)
|
||||
{
|
||||
return GetAcceptedNames(identity).Any(names.Contains);
|
||||
}
|
||||
|
||||
private static bool MatchesAttendees(
|
||||
SpeakerIdentity identity,
|
||||
IReadOnlyList<string> attendees)
|
||||
{
|
||||
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
|
||||
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
|
||||
}
|
||||
|
||||
private static Task<SpeakerIdentity?> LoadIdentityAsync(
|
||||
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
var currentCandidates = identity.CandidateNames
|
||||
.Select(candidate => candidate.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
|
||||
var aliasToCandidate = identity.Aliases
|
||||
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
|
||||
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
|
||||
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
|
||||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
|
||||
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
|
||||
var intersection = attendees
|
||||
.Select(attendee => currentCandidates.Contains(attendee)
|
||||
? attendee
|
||||
: aliasToCandidate.GetValueOrDefault(attendee) ??
|
||||
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
|
||||
? fallbackAliasCandidate
|
||||
: null))
|
||||
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
|
||||
.Select(candidate => candidate!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (intersection.Count == 0)
|
||||
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
|
||||
if (resetToAttendees)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
|
||||
identity.Id,
|
||||
FormatNames(attendees));
|
||||
ResetCandidates(identity, attendees);
|
||||
ReplaceOldestSnippet(identity, snippet);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
|
||||
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
|
||||
identity.Id,
|
||||
FormatNames(currentCandidates),
|
||||
FormatNames(attendees),
|
||||
FormatNames(intersection));
|
||||
ResetCandidates(identity, intersection);
|
||||
if (intersection.Count == 1)
|
||||
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
|
||||
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
|
||||
{
|
||||
identity.CanonicalName = intersection[0];
|
||||
logger.LogInformation(
|
||||
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
|
||||
identity.Id,
|
||||
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
|
||||
}
|
||||
}
|
||||
|
||||
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
|
||||
{
|
||||
identity.CandidateNames.Clear();
|
||||
identity.CandidateNames.AddRange(candidates
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
|
||||
}
|
||||
|
||||
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
identity.CanonicalName
|
||||
}
|
||||
.Concat(identity.Aliases.Select(alias => alias.Name))
|
||||
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
|
||||
.Where(name => !string.IsNullOrWhiteSpace(name))
|
||||
.Select(name => name!.Trim())
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
|
||||
{
|
||||
return attendees
|
||||
.Select(NormalizeAttendee)
|
||||
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Order(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeAttendee(string attendee)
|
||||
{
|
||||
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
|
||||
}
|
||||
|
||||
private static bool IsDiarizedSpeakerLabel(string speaker)
|
||||
{
|
||||
var normalized = speaker.Trim();
|
||||
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
|
||||
{
|
||||
return SpeakerIdentityReferences.Create(
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public sealed record SpeakerSampleCollectionPolicy(
|
||||
int MinimumRetainedSamples,
|
||||
bool RequireNonOverlappingSamples)
|
||||
{
|
||||
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
|
||||
|
||||
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
|
||||
{
|
||||
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
|
||||
}
|
||||
|
||||
public int ResolveRetainedSampleLimit(int configuredLimit)
|
||||
{
|
||||
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SpeakerSampleDurationConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
SpeakerIdentificationOptions options,
|
||||
string context)
|
||||
{
|
||||
ValidateOrThrow(
|
||||
options.MinimumSampleSpeechDuration,
|
||||
options.MaximumSampleDuration,
|
||||
context);
|
||||
}
|
||||
|
||||
public static void ValidateOrThrow(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(minimumDuration, maximumDuration);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (minimumDuration < TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
|
||||
}
|
||||
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
|
||||
}
|
||||
|
||||
return minimumDuration > maximumDuration
|
||||
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ResemblyzerSpeakerRecognitionConfiguration
|
||||
{
|
||||
public static void ValidateOrThrow(
|
||||
ResemblyzerSpeakerRecognitionOptions options,
|
||||
string context)
|
||||
{
|
||||
var error = GetError(options);
|
||||
if (error is not null)
|
||||
{
|
||||
throw new InvalidOperationException($"{context}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
|
||||
{
|
||||
if (options.RequiredVectorsPerSpeaker < 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
|
||||
}
|
||||
|
||||
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
|
||||
}
|
||||
|
||||
if (options.OutlierPruningMinimumNeighbors < 1 ||
|
||||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
|
||||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumClusterCohesion))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
if (!IsCosine(options.MinimumIdentitySimilarity))
|
||||
{
|
||||
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
|
||||
}
|
||||
|
||||
return !double.IsFinite(options.MinimumSimilarityMargin) ||
|
||||
options.MinimumSimilarityMargin is < 0 or > 2
|
||||
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
|
||||
: null;
|
||||
}
|
||||
|
||||
private static bool IsCosine(double value)
|
||||
{
|
||||
return double.IsFinite(value) && value is >= -1 and <= 1;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
|
||||
: IValidateOptions<MeetingAssistantOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
|
||||
{
|
||||
var error = SpeakerSampleDurationConfiguration.GetError(
|
||||
options.SpeakerIdentification.MinimumSampleSpeechDuration,
|
||||
options.SpeakerIdentification.MaximumSampleDuration) ??
|
||||
ResemblyzerSpeakerRecognitionConfiguration.GetError(
|
||||
options.SpeakerIdentification.Resemblyzer);
|
||||
return error is null
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using MeetingAssistant.Transcription;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record SpeakerSampleSpan(
|
||||
string Speaker,
|
||||
TimeSpan Start,
|
||||
TimeSpan End,
|
||||
TimeSpan SpeechDuration,
|
||||
TimeSpan MaximumDuration,
|
||||
IReadOnlyList<TranscriptionSegment> Segments)
|
||||
{
|
||||
public static SpeakerSampleSpan? Create(
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumDuration)
|
||||
{
|
||||
if (maximumDuration <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(maximumDuration),
|
||||
maximumDuration,
|
||||
"Speaker sample maximum duration must be greater than zero.");
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
|
||||
return boundedEnd > segment.Start
|
||||
? new SpeakerSampleSpan(
|
||||
segment.Speaker,
|
||||
segment.Start,
|
||||
boundedEnd,
|
||||
boundedEnd - segment.Start,
|
||||
maximumDuration,
|
||||
[segment with { End = boundedEnd }])
|
||||
: null;
|
||||
}
|
||||
|
||||
public bool TryExtend(
|
||||
TranscriptionSegment segment,
|
||||
out SpeakerSampleSpan extended)
|
||||
{
|
||||
extended = this;
|
||||
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
|
||||
segment.Start >= Start + MaximumDuration)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var boundedEnd = Min(segment.End, Start + MaximumDuration);
|
||||
if (boundedEnd <= segment.Start)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var uncoveredStart = segment.Start > End ? segment.Start : End;
|
||||
var additionalSpeechDuration = boundedEnd > uncoveredStart
|
||||
? boundedEnd - uncoveredStart
|
||||
: TimeSpan.Zero;
|
||||
extended = this with
|
||||
{
|
||||
End = boundedEnd > End ? boundedEnd : End,
|
||||
SpeechDuration = SpeechDuration + additionalSpeechDuration,
|
||||
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
public TranscriptionSegment ToSegment()
|
||||
{
|
||||
return new TranscriptionSegment(
|
||||
Start,
|
||||
End,
|
||||
Speaker,
|
||||
string.Join(' ', Segments
|
||||
.Select(segment => segment.Text)
|
||||
.Where(text => !string.IsNullOrWhiteSpace(text))));
|
||||
}
|
||||
|
||||
private static TimeSpan Min(TimeSpan left, TimeSpan right)
|
||||
{
|
||||
return left < right ? left : right;
|
||||
}
|
||||
}
|
||||
@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal static class SpeakerSampleSpanSelector
|
||||
{
|
||||
public static bool CanExtend(
|
||||
string currentSpeaker,
|
||||
TimeSpan currentEnd,
|
||||
TranscriptionSegment nextSegment,
|
||||
TimeSpan maximumSegmentGap)
|
||||
{
|
||||
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
|
||||
nextSegment.Start - currentEnd <= maximumSegmentGap;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
|
||||
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan maximumSegmentGap,
|
||||
TimeSpan minimumDuration)
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
var best = new List<TranscriptionSegment>();
|
||||
var current = new List<TranscriptionSegment>();
|
||||
|
||||
SpeakerSampleSpan? best = null;
|
||||
SpeakerSampleSpan? current = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (current.Count == 0)
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
if (IsSpeaker(segment, speaker))
|
||||
{
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is not null &&
|
||||
(best is null || current.SpeechDuration > best.SpeechDuration))
|
||||
{
|
||||
current.Clear();
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
best = current;
|
||||
}
|
||||
|
||||
current.Add(segment);
|
||||
best = LongerSpan(current, best);
|
||||
}
|
||||
|
||||
return SpanDuration(best) >= minimumDuration
|
||||
? best
|
||||
return best is not null && best.SpeechDuration >= minimumDuration
|
||||
? best.Segments
|
||||
: [];
|
||||
}
|
||||
|
||||
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
|
||||
IReadOnlyList<TranscriptionSegment> segments,
|
||||
string speaker,
|
||||
TimeSpan minimumDuration,
|
||||
TimeSpan maximumSampleDuration,
|
||||
int maxSpans)
|
||||
{
|
||||
if (maxSpans <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var completed = new List<SpeakerSampleSpan>();
|
||||
SpeakerSampleSpan? current = null;
|
||||
TimeSpan? lastCompletedEnd = null;
|
||||
foreach (var segment in segments.OrderBy(segment => segment.Start))
|
||||
{
|
||||
if (!IsSpeaker(segment, speaker))
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (lastCompletedEnd is { } end && segment.Start < end)
|
||||
{
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
current = ExtendOrStart(current, segment, maximumSampleDuration);
|
||||
if (current is null || current.SpeechDuration < minimumDuration)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
completed.Add(current);
|
||||
lastCompletedEnd = current.End;
|
||||
current = null;
|
||||
}
|
||||
|
||||
return completed
|
||||
.OrderByDescending(span => span.SpeechDuration)
|
||||
.ThenBy(span => span.Start)
|
||||
.Take(maxSpans)
|
||||
.OrderBy(span => span.Start)
|
||||
.Select(span => span.Segments)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
|
||||
{
|
||||
return segments.Count == 0
|
||||
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
|
||||
: segments[^1].End - segments[0].Start;
|
||||
}
|
||||
|
||||
private static SpeakerSampleSpan? ExtendOrStart(
|
||||
SpeakerSampleSpan? current,
|
||||
TranscriptionSegment segment,
|
||||
TimeSpan maximumSampleDuration)
|
||||
{
|
||||
return current is not null && current.TryExtend(segment, out var extended)
|
||||
? extended
|
||||
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
|
||||
}
|
||||
|
||||
private static bool IsSpeaker(
|
||||
TranscriptionSegment segment,
|
||||
string speaker)
|
||||
{
|
||||
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static List<TranscriptionSegment> LongerSpan(
|
||||
List<TranscriptionSegment> current,
|
||||
List<TranscriptionSegment> best)
|
||||
{
|
||||
return SpanDuration(current) > SpanDuration(best)
|
||||
? current.ToList()
|
||||
: best;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
internal sealed record DecodedSpeakerVoiceVector(
|
||||
SpeakerVoiceVector Stored,
|
||||
float[] Vector);
|
||||
|
||||
internal static class ResemblyzerVectorContract
|
||||
{
|
||||
public const int Dimensions = 256;
|
||||
}
|
||||
|
||||
internal static class SpeakerVoiceVectors
|
||||
{
|
||||
public static float[] Decode(SpeakerVoiceVector stored)
|
||||
{
|
||||
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
|
||||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
|
||||
}
|
||||
|
||||
var vector = new float[stored.Dimensions];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
|
||||
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
|
||||
}
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
public static int AddDistinct(
|
||||
SpeakerIdentity identity,
|
||||
IEnumerable<float[]> vectors,
|
||||
string modelId,
|
||||
int maximumCount,
|
||||
DateTimeOffset createdAt)
|
||||
{
|
||||
var limit = Math.Max(1, maximumCount);
|
||||
var fingerprints = identity.VoiceVectors
|
||||
.Select(vector => vector.Fingerprint)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var added = 0;
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
if (identity.VoiceVectors.Count >= limit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var bytes = Encode(vector);
|
||||
var fingerprint = Fingerprint(modelId, bytes);
|
||||
if (!fingerprints.Add(fingerprint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
identity.VoiceVectors.Add(new SpeakerVoiceVector
|
||||
{
|
||||
ModelId = modelId,
|
||||
Dimensions = vector.Length,
|
||||
VectorBytes = bytes,
|
||||
Fingerprint = fingerprint,
|
||||
CreatedAt = createdAt
|
||||
});
|
||||
added++;
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
public static IReadOnlyList<float[]> DecodeCompatible(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
return DecodeCompatibleEntries(identity, modelId, logger)
|
||||
.Select(entry => entry.Vector)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
|
||||
SpeakerIdentity identity,
|
||||
string modelId,
|
||||
ILogger logger)
|
||||
{
|
||||
var vectors = new List<DecodedSpeakerVoiceVector>();
|
||||
foreach (var stored in identity.VoiceVectors
|
||||
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
|
||||
.OrderBy(vector => vector.CreatedAt)
|
||||
.ThenBy(vector => vector.Id))
|
||||
{
|
||||
try
|
||||
{
|
||||
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
|
||||
}
|
||||
catch (InvalidDataException exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
|
||||
stored.Id,
|
||||
identity.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
public static double Cosine(float[] first, float[] second)
|
||||
{
|
||||
double dot = 0;
|
||||
for (var index = 0; index < first.Length; index++)
|
||||
{
|
||||
dot += first[index] * second[index];
|
||||
}
|
||||
|
||||
return dot;
|
||||
}
|
||||
|
||||
public static float[] Normalize(float[]? vector, string description)
|
||||
{
|
||||
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
double magnitudeSquared = 0;
|
||||
foreach (var value in vector)
|
||||
{
|
||||
if (!float.IsFinite(value))
|
||||
{
|
||||
throw new InvalidDataException($"{description} contains a non-finite value.");
|
||||
}
|
||||
|
||||
magnitudeSquared += value * value;
|
||||
}
|
||||
|
||||
var magnitude = Math.Sqrt(magnitudeSquared);
|
||||
if (magnitude <= double.Epsilon)
|
||||
{
|
||||
throw new InvalidDataException($"{description} has zero magnitude.");
|
||||
}
|
||||
|
||||
return vector.Select(value => (float)(value / magnitude)).ToArray();
|
||||
}
|
||||
|
||||
private static byte[] Encode(float[] vector)
|
||||
{
|
||||
if (vector.Length != ResemblyzerVectorContract.Dimensions)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
|
||||
}
|
||||
|
||||
var bytes = new byte[vector.Length * sizeof(float)];
|
||||
for (var index = 0; index < vector.Length; index++)
|
||||
{
|
||||
if (!float.IsFinite(vector[index]))
|
||||
{
|
||||
throw new InvalidDataException("Voice vector contains a non-finite value.");
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteSingleLittleEndian(
|
||||
bytes.AsSpan(index * sizeof(float), sizeof(float)),
|
||||
vector[index]);
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static string Fingerprint(string modelId, byte[] vectorBytes)
|
||||
{
|
||||
var modelBytes = Encoding.UTF8.GetBytes(modelId);
|
||||
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
|
||||
modelBytes.CopyTo(input, 0);
|
||||
input[modelBytes.Length] = 0;
|
||||
vectorBytes.CopyTo(input, modelBytes.Length + 1);
|
||||
return Convert.ToHexString(SHA256.HashData(input));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.Transcription;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace MeetingAssistant.Speakers;
|
||||
|
||||
public interface IResemblyzerVoiceEncoder
|
||||
{
|
||||
Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task WarmUpAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
|
||||
{
|
||||
private const string EnvironmentSchemaVersion = "venv-v2";
|
||||
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
|
||||
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
|
||||
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
|
||||
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
|
||||
private const string LibrosaRequirement = "librosa>=0.9.1";
|
||||
private const string ScipyRequirement = "scipy>=1.2.1";
|
||||
private readonly SemaphoreSlim commandLock = new(1, 1);
|
||||
private readonly ICommandRunner commandRunner;
|
||||
private readonly ResemblyzerSpeakerRecognitionOptions options;
|
||||
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
|
||||
private string? verifiedEnvironmentPythonPath;
|
||||
|
||||
public VenvResemblyzerVoiceEncoder(
|
||||
ICommandRunner commandRunner,
|
||||
IOptions<MeetingAssistantOptions> options,
|
||||
ILogger<VenvResemblyzerVoiceEncoder> logger)
|
||||
{
|
||||
this.commandRunner = commandRunner;
|
||||
this.options = options.Value.SpeakerIdentification.Resemblyzer;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<float[]>> EncodeAsync(
|
||||
IReadOnlyList<byte[]> wavSamples,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (wavSamples.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
if (wavSamples.Any(sample => sample.Length == 0))
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
|
||||
}
|
||||
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(inputFolder);
|
||||
try
|
||||
{
|
||||
for (var index = 0; index < wavSamples.Count; index++)
|
||||
{
|
||||
await File.WriteAllBytesAsync(
|
||||
Path.Combine(inputFolder, $"{index:D4}.wav"),
|
||||
wavSamples[index],
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(inputFolder, recursive: true);
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
catch (UnauthorizedAccessException exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
|
||||
string inputFolder,
|
||||
int expectedCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
var pythonPath = await EnsureEnvironmentAsync(token);
|
||||
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
|
||||
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
|
||||
var result = await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[scriptPath, Path.GetFullPath(inputFolder)],
|
||||
"encoding",
|
||||
token);
|
||||
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task WarmUpAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await commandLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await RunWithTimeoutAsync(async token =>
|
||||
{
|
||||
await EnsureEnvironmentAsync(token);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
commandLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateDependencySettings();
|
||||
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
|
||||
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
|
||||
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
|
||||
var readyPath = Path.Combine(environmentFolder, ".ready");
|
||||
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
|
||||
&& File.Exists(pythonPath)
|
||||
&& File.Exists(readyPath))
|
||||
{
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
if (File.Exists(pythonPath) && File.Exists(readyPath))
|
||||
{
|
||||
var verification = await commandRunner.RunAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
cancellationToken);
|
||||
if (verification.ExitCode == 0)
|
||||
{
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
|
||||
verification.ExitCode,
|
||||
verification.StandardError);
|
||||
File.Delete(readyPath);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(environmentFolder);
|
||||
await RunRequiredAsync(
|
||||
options.PythonCommand,
|
||||
["-m", "venv", "--clear", environmentFolder],
|
||||
"virtual environment creation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
["-m", "pip", "install", "--upgrade", "pip"],
|
||||
"pip upgrade",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
"--index-url", options.TorchIndexUrl,
|
||||
TorchRequirement
|
||||
],
|
||||
"CPU PyTorch installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install",
|
||||
.. RuntimeDependencyRequirements
|
||||
],
|
||||
"Resemblyzer dependency installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
[
|
||||
"-m", "pip", "install", "--no-deps",
|
||||
ResemblyzerRequirement
|
||||
],
|
||||
"Resemblyzer installation",
|
||||
cancellationToken);
|
||||
await RunRequiredAsync(
|
||||
pythonPath,
|
||||
WarmupArguments,
|
||||
"environment verification",
|
||||
cancellationToken);
|
||||
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
|
||||
verifiedEnvironmentPythonPath = pythonPath;
|
||||
return pythonPath;
|
||||
}
|
||||
|
||||
private async Task<CommandResult> RunRequiredAsync(
|
||||
string fileName,
|
||||
IReadOnlyList<string> arguments,
|
||||
string operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
|
||||
ThrowIfFailed(result, operation);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string BuildEncodingScript()
|
||||
{
|
||||
return
|
||||
"import json\n"
|
||||
+ "import sys\n"
|
||||
+ "import numpy as np\n"
|
||||
+ "from pathlib import Path\n"
|
||||
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
|
||||
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
|
||||
+ "vectors = []\n"
|
||||
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
|
||||
+ " wav = preprocess_wav(path)\n"
|
||||
+ " vector = encoder.embed_utterance(wav)\n"
|
||||
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
|
||||
+ $"print('{JsonStart}')\n"
|
||||
+ "print(json.dumps(vectors, allow_nan=False))\n"
|
||||
+ $"print('{JsonEnd}')\n";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
|
||||
{
|
||||
var json = ExtractJson(output);
|
||||
float[][]? vectors;
|
||||
try
|
||||
{
|
||||
vectors = JsonSerializer.Deserialize<float[][]>(json);
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
|
||||
}
|
||||
|
||||
if (vectors is null || vectors.Length != expectedCount)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
|
||||
}
|
||||
|
||||
return vectors
|
||||
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string ExtractJson(string output)
|
||||
{
|
||||
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
|
||||
if (start < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
|
||||
}
|
||||
|
||||
start += JsonStart.Length;
|
||||
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
|
||||
if (end < 0)
|
||||
{
|
||||
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
|
||||
}
|
||||
|
||||
return output[start..end].Trim();
|
||||
}
|
||||
|
||||
private string BuildEnvironmentKey()
|
||||
{
|
||||
var settings = string.Join('\n', EnvironmentManifest);
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
|
||||
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
|
||||
}
|
||||
|
||||
private string TorchRequirement => $"torch=={options.TorchVersion}";
|
||||
|
||||
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
|
||||
|
||||
private static string[] WarmupArguments =>
|
||||
[
|
||||
"-c",
|
||||
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
|
||||
];
|
||||
|
||||
private string[] RuntimeDependencyRequirements =>
|
||||
[
|
||||
NumpyBeforePython313Requirement,
|
||||
NumpyFromPython313Requirement,
|
||||
LibrosaRequirement,
|
||||
ScipyRequirement,
|
||||
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
|
||||
];
|
||||
|
||||
private IEnumerable<string> EnvironmentManifest =>
|
||||
[
|
||||
EnvironmentSchemaVersion,
|
||||
options.PythonCommand,
|
||||
options.TorchIndexUrl,
|
||||
TorchRequirement,
|
||||
.. RuntimeDependencyRequirements,
|
||||
ResemblyzerRequirement
|
||||
];
|
||||
|
||||
private static string GetEnvironmentPythonPath(string environmentFolder)
|
||||
{
|
||||
return OperatingSystem.IsWindows()
|
||||
? Path.Combine(environmentFolder, "Scripts", "python.exe")
|
||||
: Path.Combine(environmentFolder, "bin", "python");
|
||||
}
|
||||
|
||||
private void ValidateDependencySettings()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.PythonCommand))
|
||||
{
|
||||
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
|
||||
}
|
||||
|
||||
ValidateVersion(options.PackageVersion, "Resemblyzer package");
|
||||
ValidateVersion(options.TorchVersion, "PyTorch");
|
||||
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
|
||||
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|
||||
|| indexUri.Scheme != Uri.UriSchemeHttps)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateVersion(string version, string dependency)
|
||||
{
|
||||
if (!PackageVersionPattern().IsMatch(version))
|
||||
{
|
||||
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<T> RunWithTimeoutAsync<T>(
|
||||
Func<CancellationToken, Task<T>> operation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
|
||||
? new CancellationTokenSource(options.CommandTimeout)
|
||||
: null;
|
||||
using var linkedSource = timeoutSource is null
|
||||
? null
|
||||
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
try
|
||||
{
|
||||
return await operation(linkedSource?.Token ?? cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (
|
||||
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
|
||||
{
|
||||
throw new TimeoutException(
|
||||
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ThrowIfFailed(CommandResult result, string operation)
|
||||
{
|
||||
if (result.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
|
||||
}
|
||||
}
|
||||
|
||||
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex PackageVersionPattern();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user